"""
Gremlin Chat: CLI-like AI assistant for Google Sheets.

Implements the "Phantom Stream" protocol:
1. POST /chat -> starts job, returns job_id
2. GET /chat/{job_id} -> polls for tokens/actions from Redis buffer
3. Background: LLM generates, pushes to Redis list

"Brain in the Cloud, Hands in the Sheet" - backend never touches Sheets API.
"""

from __future__ import annotations

import asyncio
import hashlib
import json
import logging
import re
from pathlib import Path
from datetime import datetime, timedelta, timezone
from enum import Enum
from typing import Any, Dict, List, Literal, Optional, Tuple, TYPE_CHECKING
from uuid import uuid4

from fastapi import APIRouter, Depends, HTTPException, status, BackgroundTasks
from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession

from ...db import get_session, AsyncSessionLocal
from ...auth_security import require_account
from ..progress import get_redis
from ...services.gremlin_ai import GremlinAI, get_thinking_level, select_model_for_task
from ...services.gremlin_intent_router import (
    ContextLevel,
    GremlinMode,
    IntentSignals,
    IntentResult,
    ENRICHMENT_TOOLS,
    ResolutionTarget,
    detect_template,
    check_resolution_prep_needed,
    classify_intent,
    should_use_bulk_match,
    sniff_key_columns,
)

if TYPE_CHECKING:
    from ...services.gremlin_tools import UndoStrategy
from ...services.salesforce_gateway import SalesforceGateway
from ...services.gremlin_tool_registry import ToolSet, get_tools_for_mode
from ...services.resolution_prep import ResolutionPrepRecommendation
from ...services.usage_gateway import UsageGateway

# Note: get_gremlin_tier imported inside functions to avoid circular import

logger = logging.getLogger(__name__)

# Safe import - chat boots even if tool registry fails
try:
    from ...services.gremlin_tools import (
        get_registry_safe,
        get_tool_prompt_descriptions,
        validate_tool_params,
        validate_tool_available,
        is_execution_enabled,
    )

    TOOL_REGISTRY = get_registry_safe()
    TOOLS_AVAILABLE = bool(TOOL_REGISTRY)
except ImportError as e:
    logger.warning("gremlin_tools import failed, using legacy chat: %s", e)
    TOOL_REGISTRY = {}
    TOOLS_AVAILABLE = False

    def get_tool_prompt_descriptions(*args, **kwargs) -> str:
        return ""

    def validate_tool_params(*args, **kwargs) -> list:
        return []

    def validate_tool_available(*args, **kwargs) -> Tuple[bool, Optional[str]]:
        return True, None

    def is_execution_enabled() -> bool:
        return True  # Legacy mode always allows execution

router = APIRouter(tags=["gremlin-chat"])  # No prefix - included under gremlin.py's /api/v2/gremlin

# Redis key patterns
CHAT_STREAM_KEY = "gremlin:chat:{job_id}:stream"
CHAT_STATE_KEY = "gremlin:chat:{job_id}:state"
SNAPSHOT_KEY = "gremlin:snapshot:{snapshot_id}"
CHAT_TTL_SECONDS = 300  # 5 minutes
SNAPSHOT_TTL_SECONDS = 86400  # 24 hours for undo snapshots

# Conversation history settings
CHAT_HISTORY_KEY = "gremlin:chat:{conversation_id}:history"
CHAT_HISTORY_LIMIT = 10  # Keep last 10 turns to save tokens
CHAT_HISTORY_TTL_SECONDS = 86400  # 24 hours retention

# Model pinning for multi-turn conversations (thought signature compatibility)
# If a conversation falls back to 2.5 Pro, it stays pinned to avoid state-mismatch errors
CHAT_MODEL_PIN_KEY = "gremlin:chat:{conversation_id}:model_pin"

# Resolution prep confirmation patterns
PREP_CONFIRM_PATTERNS = [
    r"^(yes|y|ok|sure|proceed|go ahead|do it|sounds good|let'?s do it)\.?$",
    r"^(run|start|execute) (the )?(prep|steps|resolution)",
]

PREP_DECLINE_PATTERNS = [
    r"^(no|n|skip|cancel|nevermind|no thanks)\.?$",
    r"^(just|only) (run|do|execute)",
]

PROMPT_DIR = Path(__file__).resolve().parent.parent / "prompts" / "gremlin"

RESOLUTION_HINT_KEYWORDS = [
    "resolve",
    "match",
    "matching",
    "salesforce",
    "sfdc",
    "crm",
    "account id",
    "canonical",
    "foundrygraph",
    "dedupe",
    "duplicate",
    "duplicates",
    "company",
    "companies",
    "domain",
    "domains",
    "enrich",
]
BASE_RULES_PATH = PROMPT_DIR / "base_rules.txt"
MODE_PROMPTS = {
    GremlinMode.DATA_QUALITY: PROMPT_DIR / "data_quality_system.txt",
    GremlinMode.MODEL: PROMPT_DIR / "model_system.txt",
    GremlinMode.RESEARCH: PROMPT_DIR / "research_system.txt",
}
_PROMPT_CACHE: Dict[str, str] = {}


# ---------------------------------------------------------------------------
# Conversation History Helpers
# ---------------------------------------------------------------------------

async def _get_chat_history(redis, conversation_id: str) -> str:
    """Retrieve formatted chat history from Redis."""
    if not conversation_id:
        return ""
    key = CHAT_HISTORY_KEY.format(conversation_id=conversation_id)
    items = await redis.lrange(key, 0, -1)
    if not items:
        return ""

    # Format for prompt (User/Assistant turns)
    history_str = "\n## Conversation History (most recent messages)\n"
    for item in items:
        # Decode bytes if needed
        if isinstance(item, bytes):
            item = item.decode("utf-8")
        try:
            msg = json.loads(item)
            role = msg.get("role", "user").upper()
            content = msg.get("content", "")
            # Truncate long messages in history
            if len(content) > 500:
                content = content[:500] + "..."
            history_str += f"{role}: {content}\n"
        except json.JSONDecodeError:
            continue
    return history_str


async def _append_chat_history(redis, conversation_id: str, role: str, content: str) -> None:
    """Append a message to conversation history."""
    if not conversation_id or not content:
        return
    key = CHAT_HISTORY_KEY.format(conversation_id=conversation_id)
    entry = json.dumps({"role": role, "content": content})
    await redis.rpush(key, entry)
    await redis.ltrim(key, -CHAT_HISTORY_LIMIT, -1)  # Keep size manageable
    await redis.expire(key, CHAT_HISTORY_TTL_SECONDS)


def parse_prep_response(user_message: str, context: Any) -> Optional[str]:
    """
    Check if user message is responding to a prep suggestion.

    Returns:
    - "confirm" if user wants to run prep
    - "decline" if user wants to skip prep
    - None if message is unrelated
    """
    if not getattr(context, "awaiting_prep_confirmation", False):
        return None

    msg = user_message.strip().lower()

    for pattern in PREP_CONFIRM_PATTERNS:
        if re.match(pattern, msg, re.IGNORECASE):
            return "confirm"

    for pattern in PREP_DECLINE_PATTERNS:
        if re.match(pattern, msg, re.IGNORECASE):
            return "decline"

    return None


async def _get_pinned_model(redis, conversation_id: str) -> Optional[str]:
    """Get the pinned model for a conversation (for thought signature compatibility)."""
    if not conversation_id:
        return None
    key = CHAT_MODEL_PIN_KEY.format(conversation_id=conversation_id)
    pinned = await redis.get(key)
    if pinned:
        return pinned.decode("utf-8") if isinstance(pinned, bytes) else pinned
    return None


async def _pin_model(redis, conversation_id: str, model: str) -> None:
    """Pin a conversation to a specific model (after fallback, to avoid thought signature issues)."""
    if not conversation_id or not model:
        return
    key = CHAT_MODEL_PIN_KEY.format(conversation_id=conversation_id)
    await redis.set(key, model)
    await redis.expire(key, CHAT_HISTORY_TTL_SECONDS)


# ---------------------------------------------------------------------------
# Gremlin Protocol: Action Types
# ---------------------------------------------------------------------------

class UndoHint(BaseModel):
    """Structured undo information for reversible actions."""
    type: Literal["KEYBOARD", "RESTORE_ROWS", "RESTORE_RANGE", "SNAPSHOT", "IRREVERSIBLE"]
    sheet_name: str
    range_a1: Optional[str] = None  # For RESTORE_RANGE
    rows: Optional[List[int]] = None  # For RESTORE_ROWS
    snapshot_id: Optional[str] = None  # For RESTORE_SNAPSHOT (Redis key)
    message: str  # Human-readable undo instructions


class ConfirmationTier(str, Enum):
    AUTO = "auto"
    PREVIEW = "preview"
    EXPLICIT = "explicit"


class GremlinAction(BaseModel):
    """A proposed action the client can execute."""
    action_id: str = Field(default_factory=lambda: str(uuid4()))  # Unique ID for this action
    type: str  # UPDATE_CELLS, STYLE_RANGE, HIGHLIGHT_ROWS, ADD_COLUMN, etc.
    label: str  # Human-readable label
    description: str  # What this action will do
    payload: Dict[str, Any]  # Type-specific payload
    affected_range: Optional[str] = None  # A1 notation of affected area
    confirm_required: bool = True  # Show proposal card before executing
    confirmation_tier: Optional[str] = None  # auto | preview | explicit
    destructive: bool = False  # Warn user this action cannot be easily undone
    undo_hint: Optional[UndoHint] = None  # Structured undo information
    context_hash: Optional[str] = None  # Hash of sheet state when proposed
    proposed_at: Optional[str] = None  # ISO timestamp when proposed
    auto_execute_allowed: bool = False  # Server policy: can client auto-apply?
    tool_version: Optional[str] = None
    schema_version: Optional[int] = None
    validation_errors: Optional[List[str]] = None
    high_impact: Optional[bool] = None
    high_impact_warning: Optional[str] = None
    requires_double_confirm: Optional[bool] = None
    execution_disabled: Optional[bool] = Field(None, alias="_execution_disabled")
    # Note: draft field removed - server enforces all actions as drafts regardless of LLM output

    class Config:
        populate_by_name = True
        extra = "allow"


_SUPPORTED_ACTIONS = {
    "UPDATE_CELLS",
    "STYLE_RANGE",
    "HIGHLIGHT_ROWS",
    "ADD_COLUMN",
    "DELETE_ROWS",
    "DEDUPE",
    "RUN_TOOL",
    "OPEN_WIZARD",
    "READ_SHEET",
}


def _filter_supported_actions(
    actions: List[Any],
) -> Tuple[List[Any], List[str]]:
    filtered: List[Any] = []
    removed: List[str] = []
    for action in actions:
        action_type = None
        if isinstance(action, dict):
            action_type = action.get("type")
        else:
            action_type = getattr(action, "type", None)
        if action_type in _SUPPORTED_ACTIONS:
            filtered.append(action)
        else:
            if action_type:
                removed.append(action_type)
    return filtered, sorted(set(removed))


class GremlinProtocol:
    """Factory for creating Gremlin Protocol actions."""

    @staticmethod
    def update_cells(range_a1: str, values: List[List[Any]], description: str) -> GremlinAction:
        return GremlinAction(
            type="UPDATE_CELLS",
            label=f"Update {range_a1}",
            description=description,
            payload={"range": range_a1, "values": values},
        )

    @staticmethod
    def style_range(range_a1: str, style: Dict[str, Any], description: str) -> GremlinAction:
        return GremlinAction(
            type="STYLE_RANGE",
            label=f"Style {range_a1}",
            description=description,
            payload={"range": range_a1, "style": style},
        )

    @staticmethod
    def highlight_rows(rows: List[int], color: str, description: str) -> GremlinAction:
        return GremlinAction(
            type="HIGHLIGHT_ROWS",
            label=f"Highlight {len(rows)} rows",
            description=description,
            payload={"rows": rows, "color": color},
        )

    @staticmethod
    def add_column(header: str, position: int, values: Optional[List[Any]] = None) -> GremlinAction:
        return GremlinAction(
            type="ADD_COLUMN",
            label=f"Add column '{header}'",
            description=f"Insert new column at position {position}",
            payload={"header": header, "position": position, "values": values or []},
        )

    @staticmethod
    def delete_rows(rows: List[int], description: str, sheet_name: str = "") -> GremlinAction:
        # Generate typed undo hint with row numbers
        row_list = ", ".join(str(r) for r in rows[:5])
        if len(rows) > 5:
            row_list += f"... and {len(rows) - 5} more"

        return GremlinAction(
            type="DELETE_ROWS",
            label=f"Delete {len(rows)} rows",
            description=description,
            payload={"rows": rows},
            affected_range=f"Rows: {row_list}",
            confirm_required=True,
            destructive=True,
            undo_hint=UndoHint(
                type="RESTORE_ROWS",
                sheet_name=sheet_name,
                rows=rows,
                message=f"Use Ctrl+Z immediately to restore. Rows: {row_list}",
            ),
            proposed_at=datetime.now(timezone.utc).isoformat(),
        )

    @staticmethod
    def dedupe(key_columns: List[str], strategy: str, description: str, sheet_name: str = "") -> GremlinAction:
        cols_str = ", ".join(key_columns[:3])
        if len(key_columns) > 3:
            cols_str += f"... +{len(key_columns) - 3} more"

        return GremlinAction(
            type="DEDUPE",
            label="Remove duplicates",
            description=description,
            payload={"key_columns": key_columns, "strategy": strategy},
            confirm_required=True,
            destructive=True,
            undo_hint=UndoHint(
                type="RESTORE_SNAPSHOT",
                sheet_name=sheet_name,
                message=f"Snapshot saved. Key columns: {cols_str}",
            ),
            proposed_at=datetime.now(timezone.utc).isoformat(),
        )

    @staticmethod
    def run_tool(tool_name: str, args: Dict[str, Any], description: str) -> GremlinAction:
        """Request to run an existing Gremlin tool (scan, fix, enrich, etc.)."""
        return GremlinAction(
            type="RUN_TOOL",
            label=f"Run {tool_name}",
            description=description,
            payload={"tool": tool_name, "args": args},
            confirm_required=False,  # Tools show their own confirmation
        )

    @staticmethod
    def open_wizard(
        template_type: str,
        label: Optional[str] = None,
        suggested_inputs: Optional[Dict[str, Any]] = None,
    ) -> GremlinAction:
        template = template_type or "model"
        return GremlinAction(
            type="OPEN_WIZARD",
            label=label or f"Open {template.title()} Wizard",
            description=f"Open the {template} template wizard",
            payload={
                "template_type": template,
                "suggested_inputs": suggested_inputs or {},
            },
            confirm_required=False,
            destructive=False,
        )

    @staticmethod
    def read_sheet(sheet_name: str) -> GremlinAction:
        """Tool call: request client to read and send more sheet data."""
        return GremlinAction(
            type="READ_SHEET",
            label=f"Reading '{sheet_name}'...",
            description=f"Need to examine the {sheet_name} tab",
            payload={"sheet_name": sheet_name},
            confirm_required=False,  # Silent tool call
        )


# ---------------------------------------------------------------------------
# Request/Response Models
# ---------------------------------------------------------------------------

class AdditionalSheetData(BaseModel):
    """Data from a sheet fetched via READ_SHEET tool call."""
    sheet_name: str
    headers: List[str] = Field(default_factory=list)
    sample_rows: List[List[Any]] = Field(default_factory=list)
    row_count: int = 0
    column_count: int = 0


class SheetContext(BaseModel):
    """The "Viewport" - only what the AI needs to see."""
    spreadsheet_id: str
    sheet_name: str
    headers: List[str] = Field(default_factory=list)
    sample_rows: List[List[Any]] = Field(default_factory=list, description="First 20-50 rows")
    row_count: int = 0
    column_count: int = 0
    total_row_count: Optional[int] = None
    total_rows: Optional[int] = None
    total_cols: Optional[int] = None
    chat_preview_truncated: bool = False
    chat_preview_rows: int = 0
    chat_preview_cols: int = 0
    audit_scan_truncated: bool = False
    audit_scan_rows: int = 0
    audit_scan_cols: int = 0
    profile_available: bool = False
    profile_timestamp: Optional[datetime] = None
    profile_key: Optional[str] = None
    profile_truncated: bool = False
    column_types: Dict[str, Any] = Field(default_factory=dict, description="Inferred column types (LIGHT/MEDIUM modes)")
    context_level: Optional[ContextLevel] = None
    salesforce_connected: bool = False
    active_range: Optional[str] = None  # User's current selection
    other_sheets: List[str] = Field(default_factory=list, description="Names of other tabs")
    additional_sheets: Dict[str, Any] = Field(default_factory=dict, description="Data from READ_SHEET tool calls")
    is_truncated: Optional[bool] = None
    data_rows: List[Dict[str, Any]] = Field(default_factory=list, description="Optional raw rows keyed by header name")
    resolution_prep_offered: bool = False
    awaiting_prep_confirmation: bool = False
    prep_recommendation: Optional[Any] = None
    prep_confirmed: bool = False
    prep_completed: bool = False


def compute_context_hash(context: SheetContext) -> str:
    """
    Compute a hash of the sheet context to detect stale proposals.

    Explicitly includes:
    - sheet_name: Ensures action runs on correct sheet
    - row_count: Detects rows added/removed since proposal
    - column_count: Detects columns added/removed
    - headers: Detects column renames/reorders
    - spreadsheet_id: Prevents cross-spreadsheet execution

    Does NOT include:
    - sample_rows: Cell values can change without invalidating structure
    - active_range: Selection changes frequently, not structural
    - other_sheets: Other tabs don't affect current sheet actions
    """

    hash_data = {
        "spreadsheet_id": context.spreadsheet_id,
        "sheet_name": context.sheet_name,
        "row_count": context.row_count,
        "column_count": context.column_count,
        "headers": context.headers,  # Order matters
    }

    json_str = json.dumps(hash_data, sort_keys=True, separators=(",", ":"))
    return hashlib.sha256(json_str.encode("utf-8")).hexdigest()[:16]


class ChatStartRequest(BaseModel):
    """Start a new chat turn."""
    context: Optional[SheetContext] = None  # Required but Optional for better error messages
    message: str = Field(..., min_length=1, max_length=2000)
    conversation_id: Optional[str] = None  # For multi-turn conversations
    freestyle_mode: bool = False  # Enable freestyle execution
    mode: Optional[GremlinMode] = None  # User override
    mode_locked: bool = False  # If true, don't auto-switch on later turns
    context_level: Optional[ContextLevel] = None  # Override context depth
    resolution_target: Optional[ResolutionTarget] = None  # User override for company resolution
    resolution_target_set: bool = False  # User explicitly chose a source


class ChatStartResponse(BaseModel):
    """Response with job_id for polling."""
    job_id: str
    conversation_id: str
    context_hash: str  # Client should validate this before applying actions
    freestyle_enabled: bool = False  # Server confirms freestyle availability
    detected_mode: Optional[GremlinMode] = None
    mode_confidence: Optional[float] = None
    mode_signals: Dict[str, Any] = Field(default_factory=dict)
    suggested_context_level: Optional[ContextLevel] = None
    has_secondary_intent: bool = False
    secondary_mode: Optional[GremlinMode] = None
    resolution_target: Optional[ResolutionTarget] = None


class ChatPollResponse(BaseModel):
    """Polled response with accumulated tokens and actions."""
    status: str  # "generating", "complete", "error"
    tokens: str  # New text since last poll
    actions: List[GremlinAction] = Field(default_factory=list)
    error: Optional[str] = None
    thinking: Optional[str] = None  # Optional "thinking" display


async def _maybe_offer_resolution_prep(
    parsed: Dict[str, Any],
    context: SheetContext,
) -> Dict[str, Any]:
    """If LLM wants to run enrichment, intercept and suggest resolution prep first."""
    actions = parsed.get("actions") or []
    target_tool = None
    for action in actions:
        if not isinstance(action, dict):
            continue
        if action.get("type") != "RUN_TOOL":
            continue
        payload = action.get("payload", {}) or {}
        tool_name = payload.get("tool")
        if tool_name in ENRICHMENT_TOOLS:
            target_tool = tool_name
            break

    if not target_tool:
        return parsed

    try:
        recommendation = await check_resolution_prep_needed(
            intent=parsed.get("intent"),
            tool_name=target_tool,
            context=context,
        )
    except Exception as exc:  # Defensive: don't block chat on prep failure
        logger.debug("Resolution prep check failed: %s", exc)
        return parsed

    if not recommendation or not recommendation.prep_needed:
        return parsed

    updated = dict(parsed)
    updated["message"] = recommendation.user_message or parsed.get("message", "")
    updated["actions"] = []  # Don't proceed with enrichment until user confirms
    updated["prep_recommendation"] = {
        "data_type": recommendation.data_type.value,
        "confidence": recommendation.confidence,
        "suggested_steps": recommendation.suggested_steps,
        "missing_identifiers": recommendation.missing_identifiers,
        "has_identifiers": recommendation.has_identifiers,
        "column_mapping": recommendation.column_mapping,
        "prep_needed": recommendation.prep_needed,
        "awaiting_confirmation": True,
    }

    # Mark context so downstream handlers know we're awaiting confirmation
    try:
        setattr(context, "awaiting_prep_confirmation", True)
        setattr(context, "prep_recommendation", recommendation)
    except Exception:
        logger.debug("Unable to mark context for resolution prep offer", exc_info=True)

    return updated


# ---------------------------------------------------------------------------
# Template Wizard Routing
# ---------------------------------------------------------------------------

WIZARD_LABELS = {
    "waterfall": "Open Waterfall Builder",
    "capacity": "Open Capacity Builder",
    "coverage": "Open Coverage Builder",
    "sdr": "Open SDR Builder",
    "market_finder": "Open Market Finder",
    "partner_overlap": "Open Partner Overlap",
    "territory_planner": "Open Territory Planner",
}


async def _try_template_shortcut(message: str) -> Optional[Dict[str, Any]]:
    """Skip LLM entirely for clear template requests."""
    template_type, confidence = detect_template(message)
    if template_type and confidence >= 0.80:
        label = WIZARD_LABELS.get(
            template_type,
            f"Open {template_type.replace('_', ' ').title()} Builder",
        )
        action = GremlinProtocol.open_wizard(template_type, label=label)
        return {
            "message": f"Opening the {template_type.replace('_', ' ')} wizard.",
            "actions": [action.dict()],
            "skipped_llm": True,
        }
    return None


def _maybe_force_template_action(
    parsed: Dict[str, Any],
    intent: Optional[IntentResult],
) -> Dict[str, Any]:
    if not intent or not intent.signals.template_target:
        return parsed
    target = intent.signals.template_target
    actions = parsed.get("actions") or []
    for action in actions:
        if isinstance(action, dict) and action.get("type") == "OPEN_WIZARD":
            return parsed
    label = WIZARD_LABELS.get(target, f"Open {target.title()} Builder")
    wizard_action = GremlinProtocol.open_wizard(target, label=label)
    parsed["actions"] = [wizard_action.dict()]
    if not parsed.get("message"):
        parsed["message"] = f"Opening the {target.replace('_', ' ')} wizard."
    return parsed


# ---------------------------------------------------------------------------
# Background Task: LLM Generation
# ---------------------------------------------------------------------------

async def _generate_chat_response(
    job_id: str,
    context: SheetContext,
    message: str,
    tier: str,
    account_id: str,
    freestyle_mode: bool,
    db: AsyncSession,
    conversation_id: Optional[str] = None,
    sfdc_gateway: Optional[Any] = None,
    intent: Optional[IntentResult] = None,
    tool_set: Optional[ToolSet] = None,
    context_level: Optional[ContextLevel] = None,
) -> None:
    """Background task: generate LLM response and push tokens to Redis."""
    redis = await get_redis()
    if not redis:
        logger.error("Redis not available for chat job %s", job_id)
        await _set_chat_state(job_id, "error", "Redis not available")
        return

    stream_key = CHAT_STREAM_KEY.format(job_id=job_id)
    state_key = CHAT_STATE_KEY.format(job_id=job_id)

    try:
        # Set state to generating
        await redis.set(state_key, "generating", ex=CHAT_TTL_SECONDS)

        # Fetch conversation history and append user message
        history_str = ""
        if conversation_id:
            history_str = await _get_chat_history(redis, conversation_id)
            await _append_chat_history(redis, conversation_id, "user", message)

        profile = None
        if context.profile_key:
            profile = await _load_profile_from_cache(redis, context.profile_key)

        template_shortcut = await _try_template_shortcut(message)
        if template_shortcut:
            if template_shortcut.get("message"):
                await _push_token(
                    redis,
                    stream_key,
                    json.dumps({"type": "text", "content": template_shortcut["message"]}),
                )
            for action in template_shortcut.get("actions", []):
                enriched = _enrich_action(action, context, freestyle_mode=freestyle_mode)
                await _push_token(
                    redis,
                    stream_key,
                    json.dumps({"type": "action", "content": enriched}),
                )
            if conversation_id and template_shortcut.get("message"):
                await _append_chat_history(
                    redis, conversation_id, "assistant", template_shortcut["message"]
                )
            await redis.set(state_key, "complete", ex=CHAT_TTL_SECONDS)
            return

        # Build prompt from context (including history)
        prompt = _build_chat_prompt(
            context,
            message,
            intent=intent,
            tool_set=tool_set,
            context_level=context_level,
            freestyle_mode=freestyle_mode,
            history_str=history_str,
            profile=profile,
        )

        # Use GremlinAI for generation (mode-aware model selection)
        detected_mode = intent.mode.value if intent else None
        gremlin_ai = GremlinAI(tier, mode=detected_mode)
        if not gremlin_ai.model:
            # Free tier: return canned response
            # Parse and push in the same format as paid tier
            free_response = json.loads(_free_tier_response(context, message))
            if free_response.get("message"):
                await _push_token(redis, stream_key, json.dumps({
                    "type": "text",
                    "content": free_response["message"]
                }))
            for action in free_response.get("actions", []):
                enriched = _enrich_action(action, context, freestyle_mode=freestyle_mode)
                await _push_token(redis, stream_key, json.dumps({
                    "type": "action",
                    "content": enriched
                }))
            await redis.set(state_key, "complete", ex=CHAT_TTL_SECONDS)
            return

        # Check for pinned model (for thought signature compatibility in multi-turn)
        # If a previous turn fell back to 2.5 Pro, stay on it to avoid state-mismatch errors
        pinned_model = await _get_pinned_model(redis, conversation_id) if conversation_id else None
        if pinned_model:
            logger.info("Chat %s using pinned model %s for conversation %s", job_id, pinned_model, conversation_id)

        # Generate with Vertex AI
        try:
            task_type = _infer_task_type(message, intent)
            complexity = {
                "multi_intent": bool(intent and intent.signals.multi_intent),
                "ambiguous_range": bool(intent and intent.signals.key_missing),
                "large_dataset": bool(
                    (context.total_rows or context.row_count or 0) > 2000
                ),
                "confidence_below_threshold": bool(intent and intent.confidence < 0.6),
            }
            thinking_level = get_thinking_level(task_type, complexity)
            model_override = None
            if detected_mode:
                model_override = select_model_for_task(detected_mode, task_type, complexity)
            if pinned_model:
                model_override = pinned_model
            raw_response, usage = await gremlin_ai._generate(
                prompt,
                response_mime_type="application/json",
                max_tokens=4096,
                model_override=model_override,
                thinking_level=thinking_level,
            )
        except asyncio.TimeoutError:
            await _push_token(redis, stream_key, "I'm taking too long to think. Let me try a simpler approach...")
            await redis.set(state_key, "complete", ex=CHAT_TTL_SECONDS)
            return
        except Exception as exc:
            logger.exception("Chat generation failed: %s", exc)
            await _push_token(redis, stream_key, f"Something went wrong: {exc}")
            await redis.set(state_key, "error", ex=CHAT_TTL_SECONDS)
            return

        # If fallback was used, pin this conversation to the fallback model
        # This prevents thought signature issues when switching back and forth
        if conversation_id and usage.get("fallback_used"):
            fallback_model = usage.get("model_used")
            if fallback_model:
                await _pin_model(redis, conversation_id, fallback_model)
                logger.info(
                    "Chat %s pinned conversation %s to fallback model %s",
                    job_id, conversation_id, fallback_model
                )

        # Parse response
        if raw_response:
            logger.info("Chat %s raw LLM response (%d chars): %s", job_id, len(raw_response), raw_response[:500])
            parsed = _parse_llm_response(raw_response, message)
            logger.info("Chat %s parsed: message=%s, actions=%d", job_id, bool(parsed.get("message")), len(parsed.get("actions", [])))
            parsed = await _maybe_offer_resolution_prep(parsed, context)
            parsed = _maybe_force_template_action(parsed, intent)
            actions_raw = parsed.get("actions", [])
            actions, removed_actions = _filter_supported_actions(actions_raw)
            if removed_actions:
                parsed["actions"] = actions
                if not actions:
                    warning = (
                        f"I can't execute '{removed_actions[0]}' right now. "
                        "Available actions: Audit, Clean, Resolve, Dedupe, Build Model."
                    )
                    if parsed.get("message"):
                        parsed["message"] = f"{parsed['message']}\n\n{warning}"
                    else:
                        parsed["message"] = warning

            # Push thinking (if any)
            if parsed.get("thinking"):
                await _push_token(redis, stream_key, json.dumps({
                    "type": "thinking",
                    "content": parsed["thinking"]
                }))

            # Push message text
            if parsed.get("message"):
                await _push_token(redis, stream_key, json.dumps({
                    "type": "text",
                    "content": parsed["message"]
                }))

            # Execute inline tools (resolve_company, match_salesforce_account) before pushing
            # These tools need server-side execution with SFDC gateway
            actions = parsed.get("actions", [])
            if actions:
                sfdc_gateway = await _ensure_sfdc_gateway(account_id, context, db, sfdc_gateway)
                actions = await _execute_inline_tools(actions, context, sfdc_gateway)

            # Push actions (enriched with integrity fields)
            for action in actions:
                enriched = _enrich_action(action, context, freestyle_mode=freestyle_mode)
                await _push_token(redis, stream_key, json.dumps({
                    "type": "action",
                    "content": enriched
                }))

            # Save assistant response to history for multi-turn conversations
            if conversation_id and parsed.get("message"):
                await _append_chat_history(redis, conversation_id, "assistant", parsed["message"])

        await redis.set(state_key, "complete", ex=CHAT_TTL_SECONDS)

    except Exception as exc:
        logger.exception("Chat generation error for job %s: %s", job_id, exc)
        await redis.set(state_key, "error", ex=CHAT_TTL_SECONDS)


async def _push_token(redis, stream_key: str, content: str) -> None:
    """Push a token/message to the Redis stream."""
    await redis.rpush(stream_key, content)
    await redis.expire(stream_key, CHAT_TTL_SECONDS)


CURRENT_TOOL_VERSION = "1.0.0"
CURRENT_SCHEMA_VERSION = 1
HIGH_IMPACT_ROW_THRESHOLD = 100  # Rows affected that trigger extra confirmation


def _enrich_action(action: Dict[str, Any], context: SheetContext, freestyle_mode: bool = False) -> Dict[str, Any]:
    """
    Server-side enforcement: add integrity fields and validate parameters.

    Key policy: Server COMPUTES destructive from registry, ignores model output.
    """
    # KILL SWITCH: If execution disabled, force all actions to require confirmation
    if not is_execution_enabled():
        action["auto_execute_allowed"] = False
        action["confirm_required"] = True
        action["_execution_disabled"] = True  # Client can show "execution paused" message

    ctx_hash = compute_context_hash(context)
    now = datetime.now(timezone.utc).isoformat()

    # Ensure action_id exists
    if "action_id" not in action:
        action["action_id"] = str(uuid4())

    # Always set context_hash, proposed_at, and version fields
    action["context_hash"] = ctx_hash
    action["proposed_at"] = now
    action["tool_version"] = CURRENT_TOOL_VERSION
    action["schema_version"] = CURRENT_SCHEMA_VERSION

    if action.get("type") == "OPEN_WIZARD":
        action["destructive"] = False
        action["confirm_required"] = False
        action["confirmation_tier"] = ConfirmationTier.AUTO.value
        action["auto_execute_allowed"] = not action.get("_execution_disabled", False)
        return action

    tool_def = None
    # For RUN_TOOL actions, get tool definition and enforce policy
    if action.get("type") == "RUN_TOOL":
        payload = action.get("payload", {})
        tool_name = payload.get("tool")
        tool_args = payload.get("args", {})

        if tool_name:
            available, reason = validate_tool_available(tool_name)
            if not available:
                action["validation_errors"] = [reason]
                action["confirm_required"] = True
                action["_execution_disabled"] = True
                return action

            tool_def = TOOL_REGISTRY.get(tool_name)

            if tool_def:
                # SERVER computes destructive from registry - ignore model output
                action["destructive"] = tool_def.destructive

                # Start with tool's default confirm_required
                confirm_required = tool_def.confirm_required

                # Validate parameters
                ctx_dict = {
                    "headers": context.headers,
                    "row_count": context.row_count,
                    "column_count": context.column_count,
                    "other_sheets": context.other_sheets,
                }
                errors = validate_tool_params(tool_name, tool_args, ctx_dict)
                if errors:
                    # Validation errors ALWAYS require confirmation
                    action["validation_errors"] = errors
                    confirm_required = True

                # Destructive actions ALWAYS require confirmation
                if tool_def.destructive:
                    confirm_required = True

                action["confirm_required"] = confirm_required

                # Add undo_hint from tool definition
                if not action.get("undo_hint"):
                    action["undo_hint"] = {
                        "type": tool_def.undo_strategy.value.upper(),
                        "sheet_name": context.sheet_name,
                        "message": _get_undo_message(tool_def.undo_strategy),
                    }
            else:
                action["validation_errors"] = [f"Unknown tool: {tool_name}"]
                action["confirm_required"] = True
    else:
        # Legacy action types (DELETE_ROWS, DEDUPE) - treat as destructive
        if action.get("type") in ("DELETE_ROWS", "DEDUPE"):
            action["destructive"] = True
            action["confirm_required"] = True

        # Add default undo_hint for legacy actions
        if action.get("destructive") and not action.get("undo_hint"):
            action["undo_hint"] = {
                "type": "KEYBOARD",
                "sheet_name": context.sheet_name,
                "message": "Use Ctrl+Z (Cmd+Z on Mac) immediately to undo.",
            }

    # HIGH-IMPACT WARNING: If action affects many rows, add warning
    affected_range = action.get("affected_range", "")
    if affected_range:
        row_count = _estimate_affected_rows(affected_range, context.row_count)
        if row_count >= HIGH_IMPACT_ROW_THRESHOLD:
            action["high_impact"] = True
            action["high_impact_warning"] = f"This will affect {row_count} rows"
            action["bulk_mutation"] = True
            action["confirm_required"] = True
            # For "Apply Anyway" on high-impact, force second confirmation
            if action.get("validation_errors"):
                action["requires_double_confirm"] = True
        elif (
            row_count > 0
            and tool_def
            and not action.get("destructive")
            and not action.get("validation_errors")
        ):
            # Low-stakes, small-scope change: allow auto-exec by default
            action["confirm_required"] = False

    confirmation_tier = ConfirmationTier.PREVIEW
    if action.get("destructive") or action.get("high_impact") or action.get("requires_double_confirm"):
        confirmation_tier = ConfirmationTier.EXPLICIT
    elif not action.get("confirm_required"):
        confirmation_tier = ConfirmationTier.AUTO

    action["confirmation_tier"] = confirmation_tier.value

    can_auto_execute = (
        confirmation_tier == ConfirmationTier.AUTO
        and not action.get("_execution_disabled", False)
    )
    action["auto_execute_allowed"] = can_auto_execute
    if confirmation_tier == ConfirmationTier.AUTO:
        action["confirm_required"] = False

    logger.info(
        "Action enriched: type=%s tool=%s freestyle=%s destructive=%s confirm_req=%s auto_exec=%s tier=%s",
        action.get("type"),
        action.get("payload", {}).get("tool"),
        freestyle_mode,
        action.get("destructive"),
        action.get("confirm_required"),
        can_auto_execute,
        confirmation_tier.value,
    )

    return action


def _estimate_affected_rows(range_str: str, sheet_row_count: int) -> int:
    """Estimate rows affected by an A1 range string."""
    import re

    # Match patterns like A2:A500, A:A, A2:B500
    match = re.search(r":.*?(\\d+)$", range_str)
    if match:
        end_row = int(match.group(1))
        # Find start row
        start_match = re.search(r"(\\d+):", range_str)
        start_row = int(start_match.group(1)) if start_match else 1
        return end_row - start_row + 1
    # Full column reference (A:A) = all rows
    if re.match(r"^[A-Z]+:[A-Z]+$", range_str, re.IGNORECASE):
        return sheet_row_count
    return 0  # Unknown


def _estimate_bulk_time(row_count: int) -> int:
    """Rough ETA in minutes for bulk jobs."""
    try:
        rc = int(row_count)
    except Exception:
        rc = 0
    if rc <= 0:
        return 1
    return max(1, int(rc / 4000) + 1)


def _infer_task_type(message: str, intent: Optional[IntentResult]) -> str:
    msg = (message or "").lower()
    if intent and intent.signals.template_target:
        return "template_suggestion"
    if intent and intent.signals.multi_intent:
        return "multi_step_plan"
    if intent and intent.mode == GremlinMode.MODEL:
        if any(term in msg for term in ("formula", "vlookup", "pivot", "xlookup", "index")):
            return "complex_formula"
    if intent and intent.mode == GremlinMode.DATA_QUALITY:
        if any(term in msg for term in ("normalize", "clean", "format", "trim", "split")):
            return "quick_transform"
    return "simple_chat"


def _get_undo_message(strategy: "UndoStrategy") -> str:
    """Get user-friendly undo message for a strategy."""
    from ...services.gremlin_tools import UndoStrategy

    messages = {
        UndoStrategy.KEYBOARD: "Use Ctrl+Z (Cmd+Z on Mac) immediately to undo.",
        UndoStrategy.SNAPSHOT: "Pre-image saved. Use 'Undo last Gremlin action' to restore.",
        UndoStrategy.RESTORE_ROWS: "Deleted rows can be restored from the action log.",
        UndoStrategy.IRREVERSIBLE: "This action cannot be automatically undone.",
    }
    return messages.get(strategy, "Use Ctrl+Z immediately to undo.")


# -----------------------------------------------------------------------------
# INLINE TOOL EXECUTION
# Tools that gather data (resolve_company, match_salesforce_account) are executed
# server-side before returning actions to client. Results are added to payload.
#
# IMPORTANT: Inline execution only handles single-record operations (company_name arg).
# Range-based batch operations (range arg) are marked for client-side execution via
# the jobs API - the backend doesn't have access to sheet data (Brain/Hands principle).
# -----------------------------------------------------------------------------

# Tools that may require server-side execution with SFDC gateway
# enrich_company_name added to use smart waterfall (SFDC first, then FoundryGraph)
INLINE_EXECUTION_TOOLS = {"resolve_company", "match_salesforce_account", "enrich_company_name"}


def _needs_inline_execution(action: Dict[str, Any]) -> bool:
    """
    Check if an action requires server-side inline execution.

    Only single-record operations (with company_name arg) are executed inline.
    Range-based batch operations are handled by the client via jobs API.
    """
    if action.get("type") != "RUN_TOOL":
        return False
    payload = action.get("payload", {})
    tool_name = payload.get("tool", "")
    if tool_name not in INLINE_EXECUTION_TOOLS:
        return False

    # Only execute inline if we have single-record args, not range-based
    args = payload.get("args", {})
    has_single_record_args = bool(args.get("company_name"))
    has_range_args = bool(args.get("range"))

    if has_range_args and not has_single_record_args:
        # Range-based operation - mark for client execution, don't run inline
        return False

    return has_single_record_args


async def _execute_tool_inline(
    action: Dict[str, Any],
    context: "SheetContext",
    sfdc_gateway: Optional[Any] = None,
) -> Dict[str, Any]:
    """
    Execute a tool inline and add results to the action payload.

    For resolve_company/match_salesforce_account, this runs the operator
    with the SFDC gateway and returns enriched results.

    Only handles single-record operations (company_name arg).
    Range-based batch operations are marked for client-side execution.
    """
    from ...operators.resolve_company import resolve_company
    from ...operators.match_salesforce_account import match_salesforce_account

    payload = action.get("payload", {})
    tool_name = payload.get("tool", "")
    args = payload.get("args", {})

    result: Dict[str, Any] = {
        "executed": False,
        "tool": tool_name,
        "error": None,
        "data": None,
    }

    # Check for range-based args - these need client-side execution
    if args.get("range") and not args.get("company_name"):
        result["needs_client_execution"] = True
        result["reason"] = "Range-based operation requires client-side batch execution via jobs API"
        logger.debug(
            "Tool %s has range arg - marking for client execution: %s",
            tool_name,
            args.get("range"),
        )
        return result

    try:
        if tool_name == "resolve_company":
            # Get resolution target from args
            target_str = args.get("target", "smart")
            try:
                target = ResolutionTarget(target_str)
            except ValueError:
                target = ResolutionTarget.SMART

            # If no gateway and target needs SFDC, degrade gracefully to FoundryGraph
            # Note: match_salesforce_account does NOT degrade - it's SFDC-only by design
            if target in (ResolutionTarget.SALESFORCE, ResolutionTarget.SMART) and not sfdc_gateway:
                target = ResolutionTarget.FOUNDRYGRAPH
                result["degraded"] = True
                result["degraded_reason"] = "Salesforce not connected, using FoundryGraph"

            company_name = args.get("company_name", "")
            domain = args.get("domain")

            if company_name:
                resolution = await resolve_company(
                    company_name=company_name,
                    domain=domain,
                    target=target,
                    sfdc_connection=sfdc_gateway,
                    sfdc_threshold=args.get("threshold", 80),
                )
                result["executed"] = True
                result["data"] = resolution
            else:
                logger.debug("Tool %s skipped: no company_name in args", tool_name)

        elif tool_name == "match_salesforce_account":
            # match_salesforce_account is SFDC-only - no fallback to FoundryGraph
            # This is intentional: if user explicitly asks for SFDC match, we fail
            # loudly rather than silently returning different data source
            if not sfdc_gateway:
                result["error"] = "Salesforce not connected"
                result["data"] = {
                    "matched": False,
                    "reason": "Salesforce not connected",
                    "source": "salesforce",
                }
            else:
                company_column = args.get("company_column")
                domain_column = args.get("domain_column")
                data_rows = getattr(context, "data_rows", None)

                if company_column and data_rows:
                    rows: list[dict[str, Any]] = []
                    for row in data_rows:
                        if not isinstance(row, dict):
                            continue
                        company = row.get(company_column, "") if row else ""
                        domain_val = row.get(domain_column, "") if (row and domain_column) else ""
                        if not company and not domain_val:
                            continue
                        rows.append({"company_name": company, "domain": domain_val})

                    if rows:
                        from ...operators.match_salesforce_account import (
                            match_salesforce_accounts_batch,
                        )

                        batch_results = await match_salesforce_accounts_batch(
                            rows=rows,
                            sfdc_connection=sfdc_gateway,
                            threshold=args.get("threshold", 80),
                        )
                        result["executed"] = True
                        result["data"] = batch_results
                    else:
                        result["error"] = "No rows to match"
                else:
                    company_name = args.get("company_name", "")
                    domain = args.get("domain")

                    if company_name:
                        match_result = await match_salesforce_account(
                            company_name=company_name,
                            domain=domain,
                            sfdc_connection=sfdc_gateway,
                            threshold=args.get("threshold", 80),
                        )
                        result["executed"] = True
                        result["data"] = match_result
                    else:
                        logger.debug("Tool %s skipped: no company_name in args", tool_name)

        elif tool_name == "enrich_company_name":
            # enrich_company_name now uses SMART waterfall: SFDC first, then FoundryGraph
            # This ensures CRM data is preferred when Salesforce is connected
            company_name = args.get("company_name", args.get("company", ""))
            domain = args.get("domain", args.get("domain_range", ""))

            if company_name or domain:
                # Use resolve_company with SMART target for waterfall behavior
                target = ResolutionTarget.SMART if sfdc_gateway else ResolutionTarget.FOUNDRYGRAPH
                resolution = await resolve_company(
                    company_name=company_name or "",
                    domain=domain,
                    target=target,
                    sfdc_connection=sfdc_gateway,
                    sfdc_threshold=args.get("threshold", 80),
                )
                result["executed"] = True
                result["data"] = {
                    "company_name": resolution.get("resolved_name", ""),
                    "source": resolution.get("resolved_source", "foundrygraph"),
                    "confidence": resolution.get("resolved_confidence", 0.0),
                    "account_id": resolution.get("account_id"),
                    "resolution_notes": resolution.get("resolution_notes", ""),
                }
                # Track if we used SFDC vs FoundryGraph
                if resolution.get("resolved_source") == "salesforce":
                    result["data"]["used_crm"] = True
            else:
                logger.debug("Tool %s skipped: no company_name or domain in args", tool_name)

    except Exception as exc:
        logger.warning("Inline tool execution failed for %s: %s", tool_name, exc)
        result["error"] = str(exc)

    return result


async def _execute_inline_tools(
    actions: List[Dict[str, Any]],
    context: "SheetContext",
    sfdc_gateway: Optional[Any] = None,
) -> List[Dict[str, Any]]:
    """
    Process actions, executing inline tools and adding results to payloads.

    Returns the actions with execution_result added to those that were executed.
    For range-based batch operations, adds needs_client_execution marker.
    """
    # Clear SFDC cache at start of batch to ensure fresh queries
    from ...operators.match_salesforce_account import clear_cache
    clear_cache()

    processed = []
    for action in actions:
        if action.get("type") == "RUN_TOOL":
            payload = action.get("payload", {}) or {}
            tool_name = payload.get("tool", "")
            available, reason = validate_tool_available(tool_name)
            if not available:
                action = dict(action)
                action["validation_errors"] = [reason]
                action["execution_result"] = {"error": reason}
                processed.append(action)
                continue

        # If SFDC match is requested on large ranges, route to bulk endpoint instead of inline execution
        if action.get("type") == "RUN_TOOL":
            payload = action.get("payload", {}) or {}
            tool_name = payload.get("tool", "")
            args = payload.get("args", {}) or {}
            if tool_name == "match_salesforce_account":
                row_count_guess = args.get("row_count")
                if not row_count_guess and isinstance(args.get("range"), list):
                    row_count_guess = len(args.get("range") or [])
                if not row_count_guess:
                    row_count_guess = context.total_rows or context.total_row_count or context.row_count or 0
                if should_use_bulk_match(row_count_guess or 0) and getattr(
                    context, "salesforce_connected", False
                ):
                    processed.append(
                        {
                            "type": "BULK_JOB_REQUIRED",
                            "payload": {
                                "job_type": "sfdc_bulk_match",
                                "endpoint": "/api/v2/salesforce/match/bulk",
                                "estimated_minutes": _estimate_bulk_time(
                                    row_count_guess or 0
                                ),
                                "row_count": row_count_guess or 0,
                                "message": f"Matching {row_count_guess} rows requires a bulk job. Estimated time: {_estimate_bulk_time(row_count_guess or 0)} minutes.",
                            },
                            "confirm_required": True,
                        }
                    )
                    continue

        if _needs_inline_execution(action):
            # Single-record operation - execute inline
            exec_result = await _execute_tool_inline(action, context, sfdc_gateway)
            action = dict(action)  # Copy to avoid mutating original
            action["execution_result"] = exec_result
        elif _is_range_based_resolution_tool(action):
            # Range-based batch operation - mark for client execution via jobs API
            action = dict(action)
            action["execution_result"] = {
                "executed": False,
                "needs_client_execution": True,
                "reason": "Range-based batch operation - execute via jobs API",
                "tool": action.get("payload", {}).get("tool", ""),
            }
            logger.debug(
                "Marking action for client execution: %s range=%s",
                action.get("payload", {}).get("tool"),
                action.get("payload", {}).get("args", {}).get("range"),
            )
        elif action.get("type") == "RUN_TOOL":
            # RUN_TOOL for resolution with missing args (no company_name or range)
            payload = action.get("payload", {})
            tool_name = payload.get("tool", "")
            if tool_name in INLINE_EXECUTION_TOOLS:
                args = payload.get("args", {}) or {}
                if not args.get("company_name") and not args.get("range"):
                    action = dict(action)
                    action["execution_result"] = {
                        "executed": False,
                        "error": "Missing company_name or range for resolution tool",
                        "tool": tool_name,
                    }
                    logger.debug("Marking action as invalid args for tool %s: %s", tool_name, args)
        processed.append(action)

    return processed


async def _ensure_sfdc_gateway(
    account_id: str,
    context: "SheetContext",
    db: Optional[AsyncSession],
    existing_gateway: Optional[Any],
) -> Optional[Any]:
    """
    Lazily build a Salesforce gateway inside the background task.

    Avoids using a closed request-scoped session and enables inline SFDC tools
    even when the resolution target wasn't Salesforce but the sheet is connected.
    """
    if existing_gateway or not getattr(context, "salesforce_connected", False):
        return existing_gateway

    if db and not db.closed and getattr(db, "is_active", True):
        try:
            return await SalesforceGateway.for_tenant(account_id, db)
        except Exception as exc:  # noqa: BLE001
            logger.warning("Inline gateway build (primary) failed for %s: %s", account_id, exc)

    try:
        async with AsyncSessionLocal() as session:
            return await SalesforceGateway.for_tenant(account_id, session)
    except Exception as exc:  # noqa: BLE001
        logger.warning("Inline gateway build (fallback) failed for %s: %s", account_id, exc)
        return existing_gateway


def _is_range_based_resolution_tool(action: Dict[str, Any]) -> bool:
    """Check if action is a range-based resolution tool that needs client execution."""
    if action.get("type") != "RUN_TOOL":
        return False
    payload = action.get("payload", {})
    tool_name = payload.get("tool", "")
    if tool_name not in INLINE_EXECUTION_TOOLS:
        return False
    args = payload.get("args", {})
    return bool(args.get("range")) and not bool(args.get("company_name"))


async def _maybe_run_resolution_prep(
    context: SheetContext,
    recommendation: ResolutionPrepRecommendation,
    sfdc_gateway: Optional[Any],
) -> Dict[str, Any]:
    """
    Execute resolution prep workflow and return results summary.

    Returns dict with:
    - steps_completed: List of completed step names
    - results: {step_name: {matched: N, unmatched: N, new_columns: [...]}}
    - columns_added: List of new column names added to sheet
    """
    results: Dict[str, Any] = {"steps_completed": [], "results": {}, "columns_added": []}
    if not recommendation or not recommendation.suggested_steps:
        return results

    for step in recommendation.suggested_steps:
        if step == "match_sfdc_contact":
            step_result = await _run_contact_matching(context, sfdc_gateway, recommendation)
        elif step == "match_sfdc_account":
            step_result = await _run_account_matching(context, sfdc_gateway, recommendation)
        elif step == "company_to_domain":
            step_result = await _run_domain_resolution(context, recommendation)
        else:
            continue

        results["steps_completed"].append(step)
        results["results"][step] = step_result
        results["columns_added"].extend(step_result.get("new_columns", []))

    return results


async def _run_contact_matching(
    context: SheetContext,
    sfdc_gateway: Any,
    recommendation: ResolutionPrepRecommendation,
) -> Dict[str, Any]:
    """
    Match contacts to SFDC by email.

    1. Extract email column from context.data_rows
    2. Query SFDC Contact where Email IN (...)
    3. For matches, populate contact_id and account_id columns
    4. Return summary
    """
    email_column = recommendation.column_mapping.get("email")
    rows = getattr(context, "data_rows", None) or []
    if not email_column or not rows:
        return {"matched": 0, "unmatched": len(rows) if isinstance(rows, list) else 0, "new_columns": []}

    if not sfdc_gateway:
        return {"matched": 0, "unmatched": len(rows), "new_columns": []}

    emails: List[str] = []
    for row in rows:
        if not isinstance(row, dict):
            continue
        email_val = str(row.get(email_column, "") or "").strip().lower()
        if email_val and "@" in email_val:
            emails.append(email_val)

    if not emails:
        return {"matched": 0, "unmatched": len(rows), "new_columns": []}

    unique_emails = list(dict.fromkeys(emails))
    contact_map: Dict[str, Dict[str, Any]] = {}
    try:
        chunk_size = 100
        for idx in range(0, len(unique_emails), chunk_size):
            chunk = unique_emails[idx : idx + chunk_size]
            escaped = [email.replace("'", "\\'") for email in chunk]
            query_values = ",".join(f"'{email}'" for email in escaped)
            soql = f"SELECT Id, Email, AccountId FROM Contact WHERE Email IN ({query_values})"
            result = await sfdc_gateway.soql(soql)
            records = result.get("records", []) if isinstance(result, dict) else []
            for record in records:
                email_val = str(record.get("Email", "")).strip().lower()
                if not email_val:
                    continue
                contact_map[email_val] = {
                    "contact_id": record.get("Id"),
                    "account_id": record.get("AccountId"),
                }
    except Exception as exc:  # noqa: BLE001
        logger.debug("Contact matching failed: %s", exc)

    contact_id_column = recommendation.column_mapping.get("contact_id") or "sfdc_contact_id"
    account_id_column = recommendation.column_mapping.get("account_id") or "sfdc_account_id"

    matched_count = 0
    new_columns: List[str] = []
    has_contact_column = bool(recommendation.column_mapping.get("contact_id"))
    has_account_column = bool(recommendation.column_mapping.get("account_id"))

    for row in rows:
        if not isinstance(row, dict):
            continue
        email_val = str(row.get(email_column, "") or "").strip().lower()
        match = contact_map.get(email_val)
        if not match:
            continue
        if match.get("contact_id"):
            if not row.get(contact_id_column):
                row[contact_id_column] = match["contact_id"]
            matched_count += 1
        if match.get("account_id") and not row.get(account_id_column):
            row[account_id_column] = match["account_id"]

    if matched_count and not has_contact_column:
        new_columns.append(contact_id_column)
    if matched_count and not has_account_column and account_id_column != contact_id_column:
        new_columns.append(account_id_column)

    total_rows = len([row for row in rows if isinstance(row, dict)])
    return {
        "matched": matched_count,
        "unmatched": max(total_rows - matched_count, 0),
        "new_columns": list(dict.fromkeys(new_columns)),
    }


async def _run_account_matching(
    context: SheetContext,
    sfdc_gateway: Any,
    recommendation: ResolutionPrepRecommendation,
) -> Dict[str, Any]:
    """
    Match companies to SFDC Accounts.

    For rows without account_id:
    1. Try domain match (if domain column exists or can extract from email)
    2. Try company name fuzzy match
    3. Populate account_id column
    """
    from ...operators.match_salesforce_account import match_salesforce_accounts_batch

    rows = getattr(context, "data_rows", None) or []
    if not rows:
        return {"matched": 0, "unmatched": 0, "new_columns": []}
    if not sfdc_gateway:
        return {"matched": 0, "unmatched": len(rows), "new_columns": []}

    account_id_column = recommendation.column_mapping.get("account_id") or "sfdc_account_id"
    domain_column = recommendation.column_mapping.get("domain")
    company_column = recommendation.column_mapping.get("company") or recommendation.column_mapping.get("fullname")

    rows_to_match: List[Tuple[Dict[str, Any], Dict[str, str]]] = []
    for row in rows:
        if not isinstance(row, dict):
            continue
        if row.get(account_id_column):
            continue
        company_val = str(row.get(company_column, "") or "").strip() if company_column else ""
        domain_val = str(row.get(domain_column, "") or "").strip() if domain_column else ""
        if not company_val and not domain_val:
            continue
        rows_to_match.append((row, {"company_name": company_val, "domain": domain_val or None}))

    if not rows_to_match:
        return {"matched": 0, "unmatched": 0, "new_columns": []}

    matched_count = 0
    try:
        match_results = await match_salesforce_accounts_batch(
            rows=[payload for (_, payload) in rows_to_match],
            sfdc_connection=sfdc_gateway,
            threshold=80,
        )
    except Exception as exc:  # noqa: BLE001
        logger.debug("Account matching failed: %s", exc)
        match_results = []

    for (row, _), match in zip(rows_to_match, match_results or []):
        if not isinstance(match, dict):
            continue
        account_id = match.get("account_id")
        if match.get("matched") and account_id and not row.get(account_id_column):
            row[account_id_column] = account_id
            matched_count += 1

    new_columns: List[str] = []
    if matched_count and not recommendation.column_mapping.get("account_id"):
        new_columns.append(account_id_column)

    return {
        "matched": matched_count,
        "unmatched": max(len(rows_to_match) - matched_count, 0),
        "new_columns": new_columns,
    }


async def _run_domain_resolution(
    context: SheetContext,
    recommendation: ResolutionPrepRecommendation,
) -> Dict[str, Any]:
    """
    Resolve domains for companies without domains.

    For rows without domain:
    1. If has email, extract domain from email
    2. If has company name, run company_to_domain
    3. Populate domain column
    """
    from ...operators.company_to_domain import resolve_company_to_domain_batch

    rows = getattr(context, "data_rows", None) or []
    if not rows:
        return {"resolved": 0, "unresolved": 0, "new_columns": []}

    domain_column = recommendation.column_mapping.get("domain") or "domain"
    company_column = recommendation.column_mapping.get("company") or recommendation.column_mapping.get("fullname")
    email_column = recommendation.column_mapping.get("email")

    resolved_count = 0
    rows_to_resolve: List[Tuple[Dict[str, Any], str]] = []

    for row in rows:
        if not isinstance(row, dict):
            continue
        if row.get(domain_column):
            continue

        email_val = str(row.get(email_column, "") or "") if email_column else ""
        domain_from_email = ""
        if "@" in email_val:
            domain_from_email = email_val.split("@", 1)[1].strip().lower()
        if domain_from_email:
            row[domain_column] = domain_from_email
            resolved_count += 1
            continue

        if company_column:
            company_val = str(row.get(company_column, "") or "").strip()
            if company_val:
                rows_to_resolve.append((row, company_val))

    metrics: Dict[str, Any] = {}
    if rows_to_resolve:
        try:
            company_names = [item[1] for item in rows_to_resolve]
            resolution = resolve_company_to_domain_batch(company_names)
            values = resolution.get("values") if isinstance(resolution, dict) else []
            metrics = resolution.get("metrics") if isinstance(resolution, dict) else {}
            for (row, _), domain_val in zip(rows_to_resolve, values or []):
                if domain_val:
                    row[domain_column] = domain_val
                    resolved_count += 1
        except Exception as exc:  # noqa: BLE001
            logger.debug("Domain resolution failed: %s", exc)

    unresolved = 0
    for row in rows:
        if not isinstance(row, dict):
            continue
        if not row.get(domain_column):
            unresolved += 1

    new_columns: List[str] = []
    if resolved_count and not recommendation.column_mapping.get("domain"):
        new_columns.append(domain_column)

    return {
        "resolved": resolved_count,
        "unresolved": unresolved,
        "new_columns": new_columns,
        "metrics": metrics,
    }


async def _set_chat_state(job_id: str, state: str, error: Optional[str] = None) -> None:
    """Set chat job state in Redis."""
    redis = await get_redis()
    if redis:
        state_key = CHAT_STATE_KEY.format(job_id=job_id)
        await redis.set(state_key, state, ex=CHAT_TTL_SECONDS)


class _SafeFormatDict(dict):
    def __missing__(self, key: str) -> str:  # pragma: no cover - defensive
        return ""


def _load_prompt_template(path: Path) -> str:
    """Load prompt template with simple caching."""
    if not path:
        return ""
    cache_key = str(path)
    if cache_key in _PROMPT_CACHE:
        return _PROMPT_CACHE[cache_key]
    try:
        text = path.read_text(encoding="utf-8")
    except FileNotFoundError:
        return ""
    _PROMPT_CACHE[cache_key] = text
    return text


def _render_prompt_template(template: str, variables: Dict[str, Any]) -> str:
    """Render prompt template supporting {include: file} directives."""

    def _include(match: re.Match[str]) -> str:
        fname = match.group(1).strip()
        include_path = PROMPT_DIR / fname
        return _load_prompt_template(include_path)

    with_includes = re.sub(r"{include:\s*([^}]+)}", _include, template)
    try:
        safe_vars = _SafeFormatDict(**{k: ("" if v is None else v) for k, v in variables.items()})
        return with_includes.format_map(safe_vars)
    except Exception as exc:  # pragma: no cover - defensive
        logger.debug("Prompt formatting issue: %s", exc)
        return with_includes


def _format_toolset_for_prompt(tool_set: Optional[ToolSet]) -> str:
    if not tool_set:
        return ""
    lines = ["## Mode Tooling"]
    for tool in tool_set.tools:
        markers = []
        if tool.is_read_only:
            markers.append("read-only")
        if tool.is_write:
            markers.append("writes")
        marker_text = f" ({', '.join(markers)})" if markers else ""
        lines.append(f"- {tool.name}{marker_text}: {tool.description}")

    constraints = tool_set.constraints
    lines.append(
        "\nConstraints: "
        f"max_cells_per_action={constraints.max_cells_per_action or 'n/a'}, "
        f"max_rows_per_batch={constraints.max_rows_per_batch}, "
        f"require_preview={constraints.require_preview}, "
        f"allow_web_search={constraints.allow_web_search}"
    )
    return "\n".join(lines)


def _preview_rows_for_level(context: SheetContext, context_level: ContextLevel) -> List[List[Any]]:
    if context_level == ContextLevel.LIGHT:
        return []
    max_rows = 10 if context_level == ContextLevel.MEDIUM else 50
    return (context.sample_rows or [])[:max_rows]


def _format_sample_preview(preview_rows: List[List[Any]]) -> str:
    if not preview_rows:
        return "  (no data)"
    lines = []
    for i, row in enumerate(preview_rows[:5]):
        row_str = " | ".join(str(cell)[:30] for cell in row[:10])
        lines.append(f"  Row {i + 2}: {row_str}")
    return "\n".join(lines)


def _format_column_types(column_types: Dict[str, Any]) -> str:
    if not column_types:
        return "(unknown)"
    pairs = [f"{header}: {col_type}" for header, col_type in list(column_types.items())[:15]]
    return ", ".join(pairs)


def _format_additional_sheets(context: SheetContext) -> str:
    if not context.additional_sheets:
        return ""

    section = "\n## Additional Sheet Data (from READ_SHEET requests)\n"
    for sheet_name, sheet_data in context.additional_sheets.items():
        if isinstance(sheet_data, dict):
            add_headers = sheet_data.get("headers", [])
            add_rows = sheet_data.get("sample_rows", [])
            add_row_count = sheet_data.get("row_count", 0)
            add_col_count = sheet_data.get("column_count", 0)

            add_headers_str = ", ".join(str(h)[:30] for h in add_headers[:15]) if add_headers else "(no headers)"
            section += f'\n### Sheet: "{sheet_name}" ({add_row_count} rows x {add_col_count} columns)\n'
            section += f"Headers: {add_headers_str}\n"

            if add_rows:
                section += "Sample rows:\n"
                for i, row in enumerate(add_rows[:5]):
                    row_str = " | ".join(str(cell)[:30] for cell in row[:10])
                    section += f"  Row {i + 2}: {row_str}\n"
    return section


async def _load_profile_from_cache(redis, profile_key: Optional[str]) -> Optional[Dict[str, Any]]:
    if not redis or not profile_key:
        return None
    raw = await redis.get(profile_key)
    if not raw:
        return None
    if isinstance(raw, bytes):
        raw = raw.decode("utf-8")
    try:
        return json.loads(raw)
    except json.JSONDecodeError:
        return None


def _compress_profile_for_prompt(profile: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
    if not profile:
        return None
    total_rows = profile.get("total_rows") or 0
    data_rows = max(total_rows - 1, 0)
    columns = profile.get("columns") or []
    compressed_cols = []
    for col in columns[:20]:
        non_empty = col.get("non_empty_count") or 0
        fill_rate = (non_empty / data_rows) if data_rows else 0.0
        compressed_cols.append(
            {
                "header": col.get("header", ""),
                "type": col.get("inferred_type", "text"),
                "fill_rate": round(fill_rate, 3),
                "unique_count": col.get("unique_count", 0),
                "likely_key": bool(col.get("likely_key")),
                "patterns": (col.get("top_patterns") or [])[:2],
            }
        )
    return {
        "total_rows": total_rows,
        "total_cols": profile.get("total_cols") or 0,
        "profiled_at": profile.get("profiled_at") or "",
        "profile_truncated": bool(profile.get("profile_truncated")),
        "duplicate_signals": profile.get("duplicate_signals") or {},
        "columns": compressed_cols,
    }


def _format_profile_section(profile: Optional[Dict[str, Any]]) -> str:
    if not profile:
        return ""
    lines = ["## Full Sheet Profile"]
    total_rows = profile.get("total_rows") or 0
    total_cols = profile.get("total_cols") or 0
    profiled_at = profile.get("profiled_at") or ""
    truncated = bool(profile.get("profile_truncated"))
    trunc_note = " (sampled)" if truncated else ""
    lines.append(f"- Total size: {total_rows} rows x {total_cols} columns{trunc_note}")
    if profiled_at:
        lines.append(f"- Profiled at: {profiled_at}")

    columns = profile.get("columns") or []
    if columns:
        lines.append("### Column Summary (top 15)")
        for col in columns[:15]:
            header = col.get("header", "")
            col_type = col.get("type", "text")
            uniq = col.get("unique_count", 0)
            fill_rate = col.get("fill_rate", 0.0)
            likely_key = "yes" if col.get("likely_key") else "no"
            patterns = col.get("patterns") or []
            pattern_str = ", ".join(patterns[:2]) if patterns else "n/a"
            fill_pct = int(round(fill_rate * 100))
            lines.append(
                f"- {header}: type={col_type}, fill={fill_pct}%, unique={uniq}, key={likely_key}, patterns={pattern_str}"
            )

    dup = profile.get("duplicate_signals") or {}
    if dup:
        est = dup.get("estimated_duplicate_rows") or 0
        cols = dup.get("likely_duplicate_columns") or []
        conf = dup.get("duplicate_confidence") or 0
        if est or cols:
            lines.append("### Duplicate Signals")
            if est:
                lines.append(f"- Estimated duplicates: ~{est} rows")
            if cols:
                lines.append(f"- Likely dedupe columns: {', '.join(cols[:5])}")
            lines.append(f"- Confidence: {conf}")

    return "\n".join(lines)


def _build_context_section(
    context: SheetContext,
    context_level: ContextLevel,
    preview_rows: List[List[Any]],
    resolution_target: Optional[ResolutionTarget] = None,
    resolution_target_set: Optional[bool] = None,
    key_columns: Optional[List[str]] = None,
    key_column_types: Optional[Dict[str, str]] = None,
    key_columns_with_values: Optional[List[str]] = None,
    key_missing: Optional[bool] = None,
    include_key_status: bool = True,
) -> str:
    headers_str = ", ".join(context.headers[:20]) if context.headers else "(no headers)"
    other_sheets_str = ", ".join(context.other_sheets[:10]) if context.other_sheets else "(none)"
    sample_preview = _format_sample_preview(preview_rows)
    column_types_str = _format_column_types(context.column_types)
    total_rows = context.total_rows or context.total_row_count or context.row_count
    total_cols = context.total_cols or context.column_count
    preview_rows_count = (
        context.chat_preview_rows
        if context.chat_preview_rows is not None
        else len(preview_rows)
    )
    preview_cols_count = context.chat_preview_cols or context.column_count
    preview_suffix = (
        f" (of {total_rows} rows)"
        if context.chat_preview_truncated and total_rows
        else ""
    )
    preview_line = (
        f"- Preview shown: {preview_rows_count} rows x {preview_cols_count} cols{preview_suffix}\n"
    )
    profile_ts = None
    if context.profile_timestamp:
        if isinstance(context.profile_timestamp, datetime):
            profile_ts = context.profile_timestamp.isoformat()
        else:
            profile_ts = str(context.profile_timestamp)
    if context.profile_available:
        profile_label = "available"
        if context.profile_truncated:
            profile_label = "available (sampled)"
        profile_line = (
            f"- Full sheet profile: {profile_label}"
            + (f" (profiled at {profile_ts})\n" if profile_ts else "\n")
        )
    else:
        profile_line = "- Full sheet profile: not available (recommend profiling for precision)\n"
    resolution_line = ""
    resolution_set_line = ""
    sfdc_line = ""
    if include_key_status:
        resolution_value = getattr(resolution_target, "value", resolution_target)
        resolution_line = (
            f"- Resolution target: {resolution_value}\n" if resolution_value else ""
        )
        if resolution_target_set is not None:
            resolution_set_line = (
                "- Resolution target set: yes\n"
                if resolution_target_set
                else "- Resolution target set: no\n"
            )
        sfdc_line = "- Salesforce connected: yes\n" if context.salesforce_connected else ""
    key_line = ""
    if include_key_status:
        key_columns = key_columns or []
        key_column_types = key_column_types or {}
        key_columns_with_values = key_columns_with_values or []
        if key_columns:
            labeled = [
                f"{col} ({key_column_types.get(col, 'key')})" for col in key_columns
            ]
            key_line = f"- Key columns detected: {', '.join(labeled)}\n"
            if key_columns_with_values:
                key_line += f"- Key columns with values: {', '.join(key_columns_with_values)}\n"
            else:
                key_line += "- Key columns with values: none\n" if context.sample_rows else "- Key columns with values: unknown\n"
        else:
            key_line = "- Key columns detected: none\n"
        if key_missing:
            key_line += "- Key status: missing canonical key (domain/uid)\n"

    return (
        f'## Current Sheet Context\n'
        f'- Sheet: "{context.sheet_name}" ({context.row_count} rows x {context.column_count} columns)\n'
        f"- Context level: {context_level.value}\n"
        f"- Total size: {total_rows} rows x {total_cols} columns\n"
        f"{preview_line}"
        f"{profile_line}"
        f"- Headers: {headers_str}\n"
        f"- Column types: {column_types_str}\n"
        f"- Other tabs: {other_sheets_str}\n"
        f"- Active selection: {context.active_range or '(none)'}\n"
        f"{resolution_line}"
        f"{resolution_set_line}"
        f"{sfdc_line}"
        f"{key_line}"
        f"\n## Sample Data ({len(preview_rows)} rows shown):\n{sample_preview}\n"
    )


def _should_surface_resolution_signals(
    message: str,
    intent: Optional[IntentResult],
) -> bool:
    if intent:
        if intent.signals.resolution_target_explicit:
            return True
        if intent.mode == GremlinMode.RESEARCH:
            return True
    message_lower = (message or "").lower()
    return any(keyword in message_lower for keyword in RESOLUTION_HINT_KEYWORDS)


def _build_chat_prompt(
    context: SheetContext,
    message: str,
    intent: Optional[IntentResult] = None,
    tool_set: Optional[ToolSet] = None,
    context_level: Optional[ContextLevel] = None,
    freestyle_mode: bool = False,
    history_str: str = "",
    profile: Optional[Dict[str, Any]] = None,
) -> str:
    """Build the LLM prompt from context, user message, and mode."""
    mode = intent.mode if intent else GremlinMode.DATA_QUALITY
    resolution_target = intent.signals.resolution_target if intent else None
    resolution_target_set = intent.signals.resolution_target_set if intent else None
    resolution_target_value = getattr(resolution_target, "value", resolution_target)
    include_key_status = _should_surface_resolution_signals(message, intent)
    key_columns = intent.signals.key_columns if intent and include_key_status else None
    key_column_types = intent.signals.key_column_types if intent and include_key_status else None
    key_columns_with_values = (
        intent.signals.key_columns_with_values if intent and include_key_status else None
    )
    key_missing = intent.signals.key_missing if intent and include_key_status else None
    tool_set = tool_set or get_tools_for_mode(mode)
    suggested_level = intent.signals.suggested_context_level if intent else ContextLevel.HEAVY
    try:
        context_level = ContextLevel(context_level or suggested_level)
    except Exception:
        context_level = ContextLevel.HEAVY

    preview_rows = _preview_rows_for_level(context, context_level)
    mode_prompt_template = _load_prompt_template(MODE_PROMPTS.get(mode))
    max_cells = tool_set.constraints.max_cells_per_action
    max_cells_value = "n/a" if max_cells is None or max_cells == 0 else max_cells
    template_vars = {
        "headers": ", ".join(context.headers[:20]) if context.headers else "(no headers)",
        "row_count": context.row_count,
        "total_rows": context.total_rows or context.total_row_count or context.row_count,
        "sample_size": len(preview_rows),
        "column_types": _format_column_types(context.column_types),
        "batch_size": tool_set.constraints.max_rows_per_batch,
        "max_cells": max_cells_value,
        "resolution_target": resolution_target_value or "",
        "resolution_target_set": "yes" if resolution_target_set else "no",
    }
    mode_prompt = (
        _render_prompt_template(mode_prompt_template, template_vars)
        if mode_prompt_template
        else f"You are Gremlin in {mode.value} mode."
    )

    tool_section = _format_toolset_for_prompt(tool_set)

    registry_allowlist: List[str] = []
    if TOOL_REGISTRY:
        registry_allowlist = [name for name in tool_set.get_tool_names() if name in TOOL_REGISTRY]
    legacy_tool_descriptions = ""
    if TOOLS_AVAILABLE:
        if registry_allowlist:
            legacy_tool_descriptions = get_tool_prompt_descriptions(allowlist=set(registry_allowlist))
        else:
            legacy_tool_descriptions = get_tool_prompt_descriptions()

    freestyle_section = ""
    if freestyle_mode:
        freestyle_section = (
            "## Freestyle Mode (ENABLED)\n"
            "You may execute multiple tools in sequence, infer ranges when unspecified, "
            "and batch non-destructive operations. Destructive steps still require confirmation."
        )

    multi_intent_note = ""
    if intent and intent.signals.multi_intent:
        secondary = intent.signals.secondary_mode.value if intent.signals.secondary_mode else "secondary"
        multi_intent_note = (
            f"## Multi-Intent Plan\n"
            f"Primary mode: {mode.value}. Secondary: {secondary}. Execute as two-phase plan."
        )

    context_section = _build_context_section(
        context,
        context_level,
        preview_rows,
        resolution_target=resolution_target,
        resolution_target_set=resolution_target_set,
        key_columns=key_columns,
        key_column_types=key_column_types,
        key_columns_with_values=key_columns_with_values,
        key_missing=key_missing,
        include_key_status=include_key_status,
    )
    additional_sheets_section = _format_additional_sheets(context)
    compressed_profile = _compress_profile_for_prompt(profile)
    profile_section = _format_profile_section(compressed_profile)

    response_contract = f"""
## Response Format
Return a JSON object with:
- "thinking": (optional) Brief 1-2 sentence analysis of what you're doing (shown to user as "glass box" transparency)
- "message": Your response to the user (conversational, helpful)
- "actions": Array of RUN_TOOL or sheet actions to execute

### Action Format
All operations go through RUN_TOOL when calling registered tools:
{{
  "type": "RUN_TOOL",
  "label": "Short button label",
  "description": "What this will do",
  "payload": {{"tool": "tool_name", "args": {{...}}}},
  "affected_range": "A2:A{context.row_count}",
  "destructive": false
}}

OPEN_WIZARD routes users to template wizards (do NOT build models inline):
{{
  "type": "OPEN_WIZARD",
  "label": "Open Capacity Builder",
  "description": "Open the capacity planning wizard",
  "payload": {{"template_type": "capacity", "suggested_inputs": {{...}}}}
}}

Direct sheet actions are also allowed:
{{
  "type": "UPDATE_CELLS",
  "label": "Update A2:A5",
  "description": "Write formulas or values",
  "payload": {{"range": "A2:A5", "values": [["=SUM(B2:C2)"], ...]}}
}}

READ_SHEET remains the only action for fetching other tabs:
{{
  "type": "READ_SHEET",
  "label": "Reading 'Accounts'...",
  "description": "Need to examine the Accounts tab",
  "payload": {{"sheet_name": "Accounts"}}
}}

### Parameter Inference Rules
1. Range inference: If user doesn't specify range, infer from context (e.g., "column A" -> "A2:A{context.row_count}")
2. Column inference: Match user language to actual headers (case-insensitive)
3. Threshold defaults: Use 80 for matching, 100 for exact dedupe
4. Validate before proposing: Don't propose tools that require missing prerequisites

### Safety Rules
1. NEVER propose dedupe without explicit user intent to remove duplicates
2. ALWAYS set destructive=true for data-removing operations
3. ALWAYS include affected_range so user knows what will change
4. For Salesforce ops, verify salesforce_connected before proposing
5. Use sfdc_query_template (not raw SOQL) for Salesforce queries

Respond with JSON only, no markdown code blocks.
"""

    prompt_parts = [
        mode_prompt,
        context_section,
        profile_section,
        additional_sheets_section,
        tool_section,
        legacy_tool_descriptions,
        freestyle_section,
        multi_intent_note,
        history_str,
        "## User Request\n" + message,
        response_contract,
    ]

    return "\n\n".join(part for part in prompt_parts if part)

def _free_tier_response(context: SheetContext, message: str) -> str:
    """Generate a helpful response for free tier users."""
    return json.dumps({
        "message": f"I see you have a sheet with {context.row_count} rows. To get AI-powered assistance with your request, upgrade to Pro tier. In the meantime, try the 'What should I do?' button for suggested actions!",
        "actions": [
            {
                "type": "RUN_TOOL",
                "label": "What should I do?",
                "description": "Get suggested actions based on your sheet",
                "payload": {"tool": "triage", "args": {}},
                "confirm_required": False
            }
        ]
    })


def _parse_llm_response(raw_text: str, user_message: str) -> Dict[str, Any]:
    """
    Parse LLM response with graceful fallback.
    If JSON is invalid, return a plain-language response instead of error.
    """
    import json

    if not raw_text:
        return {
            "message": "I couldn't process that request. Could you try rephrasing?",
            "actions": [],
        }

    # Try direct parse
    try:
        return json.loads(raw_text.strip())
    except json.JSONDecodeError:
        pass

    # Try extracting JSON from markdown code blocks
    cleaned = raw_text.strip()
    if cleaned.startswith("```"):
        lines = cleaned.split("\n")
        if lines[0].startswith("```"):
            lines = lines[1:]
        if lines and lines[-1].strip() == "```":
            lines = lines[:-1]
        cleaned = "\n".join(lines)
        try:
            return json.loads(cleaned)
        except json.JSONDecodeError:
            pass

    # Try finding JSON object in text
    start = cleaned.find("{")
    end = cleaned.rfind("}")
    if start != -1 and end > start:
        try:
            return json.loads(cleaned[start:end + 1])
        except json.JSONDecodeError:
            pass

    # Fallback: return the raw text as a message (chat doesn't die)
    logger.warning("LLM returned unparseable response, using fallback")
    return {
        "message": raw_text[:500] if len(raw_text) > 500 else raw_text,
        "actions": [],
        "_parse_fallback": True,  # Flag for debugging
    }


# ---------------------------------------------------------------------------
# API Endpoints
# ---------------------------------------------------------------------------

@router.post("/chat", response_model=ChatStartResponse)
async def start_chat(
    request: ChatStartRequest,
    background_tasks: BackgroundTasks,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
):
    """
    Start a new chat turn.

    Returns a job_id that the client polls for results.
    The LLM generation runs in the background, pushing tokens to Redis.
    """
    # Validate context is present (can be null when switching sheets)
    if request.context is None:
        raise HTTPException(
            status_code=400,
            detail="Sheet context is required. Please refresh the sidebar or switch back to a sheet tab.",
    )

    # Enforce Gremlin usage limits per workspace (daily, tier-based).
    gateway = UsageGateway(db)
    job_id = str(uuid4())
    conversation_id = request.conversation_id or str(uuid4())
    consume = await gateway.check_and_consume(
        account_id=account_id,
        resource="gremlin",
        amount=1,
        source=f"gremlin_chat:{job_id}",
    )
    if not consume.allowed:
        now = datetime.now(timezone.utc)
        reset_at = consume.reset_at or datetime(
            now.year, now.month, now.day, tzinfo=timezone.utc
        ) + timedelta(days=1)
        retry_after_seconds = max(
            int((reset_at - now).total_seconds()), 0
        )
        status_code = 429 if consume.limit and consume.limit > 0 else 403
        raise HTTPException(
            status_code=status_code,
            detail={
                "error": "GREMLIN_LIMIT_EXCEEDED"
                if status_code == 429
                else "GREMLIN_NOT_ENABLED",
                "message": consume.message
                or "Gremlin chat limit reached.",
                "tier": consume.tier,
                "used": consume.used,
                "limit": consume.limit,
                "remaining": consume.remaining,
                "reset_at": reset_at.isoformat(),
                "retry_after_seconds": retry_after_seconds,
            },
        )

    tier = consume.tier

    # Determine mode/intent (user override wins)
    try:
        if request.mode_locked and request.mode:
            intent = IntentResult(
                mode=request.mode,
                confidence=1.0,
                signals=IntentSignals(
                    keywords_hit=[],
                    rationale="User override",
                    resolution_target=request.resolution_target or ResolutionTarget.SMART,
                    resolution_target_set=bool(
                        request.resolution_target_set or request.resolution_target
                    ),
                    resolution_target_explicit=bool(request.resolution_target),
                    salesforce_connected=bool(getattr(request.context, "salesforce_connected", False)),
                    **sniff_key_columns(request.context),
                ),
            )
        else:
            intent = await classify_intent(request.message, request.context)
    except Exception as exc:  # Defensive default
        logger.exception("Gremlin intent router failed, defaulting: %s", exc)
        intent = IntentResult(
            mode=GremlinMode.DATA_QUALITY,
            confidence=0.5,
            signals=IntentSignals(
                rationale="Router error; defaulted to DATA_QUALITY",
                resolution_target=request.resolution_target or ResolutionTarget.SMART,
                resolution_target_set=bool(
                    request.resolution_target_set or request.resolution_target
                ),
                resolution_target_explicit=bool(request.resolution_target),
                salesforce_connected=bool(getattr(request.context, "salesforce_connected", False)),
                **sniff_key_columns(request.context),
            ),
        )

    # Apply user override for resolution target if provided
    if request.resolution_target:
        intent.signals = intent.signals.copy(
            update={
                "resolution_target": request.resolution_target,
                "resolution_target_set": True,
                "resolution_target_explicit": True,
            }
        )
    else:
        target_set = bool(
            request.resolution_target_set
            or intent.signals.resolution_target_explicit
        )
        intent.signals = intent.signals.copy(
            update={"resolution_target_set": target_set}
        )

    tool_set = get_tools_for_mode(intent.mode)
    suggested_context_level = request.context_level or intent.signals.suggested_context_level

    # Freestyle mode requires Unleashed or Scale tier
    freestyle_mode = request.freestyle_mode and tier in ("unleashed", "scale")

    # Optionally prepare Salesforce gateway for resolution if requested/available
    sfdc_gateway = None
    if (
        request.context
        and request.context.salesforce_connected
        and intent.signals.resolution_target in (ResolutionTarget.SALESFORCE, ResolutionTarget.SMART)
    ):
        try:
            sfdc_gateway = await SalesforceGateway.for_tenant(account_id, db)
        except Exception as exc:
            logger.warning("Salesforce gateway unavailable for account %s: %s", account_id, exc)
            # Degrade gracefully to FoundryGraph
            intent.signals = intent.signals.copy(
                update={
                    "resolution_target": ResolutionTarget.FOUNDRYGRAPH,
                    "rationale": "Salesforce unavailable, falling back to FoundryGraph",
                }
            )

    # Initialize Redis state
    redis = await get_redis()
    if redis:
        state_key = CHAT_STATE_KEY.format(job_id=job_id)
        await redis.set(state_key, "pending", ex=CHAT_TTL_SECONDS)

    # Queue background generation
    background_tasks.add_task(
        _generate_chat_response,
        job_id=job_id,
        context=request.context,
        message=request.message,
        tier=tier,
        account_id=account_id,
        freestyle_mode=freestyle_mode,
        db=db,
        conversation_id=conversation_id,
        sfdc_gateway=sfdc_gateway,
        intent=intent,
        tool_set=tool_set,
        context_level=suggested_context_level,
    )

    # Compute context hash for integrity checks
    ctx_hash = compute_context_hash(request.context)

    logger.info("Started chat job %s for account %s (tier=%s)", job_id, account_id, tier)

    return ChatStartResponse(
        job_id=job_id,
        conversation_id=conversation_id,
        context_hash=ctx_hash,
        freestyle_enabled=freestyle_mode,
        detected_mode=intent.mode,
        mode_confidence=intent.confidence,
        mode_signals=intent.signals.dict(),
        suggested_context_level=suggested_context_level,
        has_secondary_intent=bool(intent.signals.multi_intent),
        secondary_mode=intent.signals.secondary_mode,
        resolution_target=intent.signals.resolution_target,
    )


@router.get("/chat/{job_id}", response_model=ChatPollResponse)
async def poll_chat(
    job_id: str,
    account_id: str = Depends(require_account),
):
    """
    Poll for chat results.

    Returns accumulated tokens since last poll.
    Client should poll every 300-500ms until status is "complete" or "error".
    """
    redis = await get_redis()
    if not redis:
        raise HTTPException(
            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
            detail="Chat service temporarily unavailable"
        )

    stream_key = CHAT_STREAM_KEY.format(job_id=job_id)
    state_key = CHAT_STATE_KEY.format(job_id=job_id)

    # Get current state
    state = await redis.get(state_key)
    if not state:
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND,
            detail="Chat job not found or expired"
        )
    # Redis returns bytes - decode to string
    if isinstance(state, bytes):
        state = state.decode("utf-8")

    # Pop all available tokens
    tokens_list = []
    actions = []
    thinking = None

    while True:
        token = await redis.lpop(stream_key)
        if not token:
            break

        # Redis returns bytes - decode to string
        if isinstance(token, bytes):
            token = token.decode("utf-8")

        # Parse token (could be plain text or JSON)
        try:
            parsed = json.loads(token)
            if parsed.get("type") == "text":
                tokens_list.append(parsed.get("content", ""))
            elif parsed.get("type") == "action":
                actions.append(GremlinAction(**parsed.get("content", {})))
            elif parsed.get("type") == "thinking":
                thinking = parsed.get("content")
        except json.JSONDecodeError:
            # Plain text token
            tokens_list.append(token)

    return ChatPollResponse(
        status=state,
        tokens="".join(tokens_list),
        actions=actions,
        thinking=thinking,
    )
