import hashlib
import json
import threading
from datetime import datetime, UTC
from pathlib import Path

from flask import g, session
from loguru import logger

from ...config.llm_config import get_llm

# Output directory for research results
from ...config.paths import get_research_outputs_directory
from ...config.search_config import get_search
from ...constants import ResearchStatus
from ...database.models import ResearchHistory, ResearchStrategy
from ...database.session_context import get_user_db_session
from ...error_handling.report_generator import ErrorReportGenerator
from ...utilities.thread_context import set_search_context
from ...report_generator import IntegratedReportGenerator
from ...search_system import AdvancedSearchSystem
from ...text_optimization import CitationFormatter, CitationMode
from ...utilities.log_utils import log_for_research
from ...utilities.search_utilities import extract_links_from_search_results
from ...utilities.threading_utils import thread_context, thread_with_app_context
from ..models.database import calculate_duration
from .socket_service import SocketIOService

OUTPUT_DIR = get_research_outputs_directory()


def _parse_research_metadata(research_meta) -> dict:
    """Parse research_meta into a dict, handling both dict and JSON string types."""
    if isinstance(research_meta, dict):
        return dict(research_meta)
    if isinstance(research_meta, str):
        try:
            return json.loads(research_meta)
        except json.JSONDecodeError:
            logger.exception("Failed to parse research_meta as JSON")
            return {}
    return {}


def get_citation_formatter():
    """Get citation formatter with settings from thread context."""
    # Import here to avoid circular imports
    from ...config.search_config import get_setting_from_snapshot

    citation_format = get_setting_from_snapshot(
        "report.citation_format", "number_hyperlinks"
    )
    mode_map = {
        "number_hyperlinks": CitationMode.NUMBER_HYPERLINKS,
        "domain_hyperlinks": CitationMode.DOMAIN_HYPERLINKS,
        "domain_id_hyperlinks": CitationMode.DOMAIN_ID_HYPERLINKS,
        "domain_id_always_hyperlinks": CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS,
        "no_hyperlinks": CitationMode.NO_HYPERLINKS,
    }
    mode = mode_map.get(citation_format, CitationMode.NUMBER_HYPERLINKS)
    return CitationFormatter(mode=mode)


def export_report_to_memory(
    markdown_content: str, format: str, title: str = None
):
    """
    Export a markdown report to different formats in memory.

    Uses the modular exporter registry to support multiple formats.
    Available formats can be queried with ExporterRegistry.get_available_formats().

    Args:
        markdown_content: The markdown content to export
        format: Export format (e.g., 'pdf', 'odt', 'latex', 'quarto', 'ris')
        title: Optional title for the document

    Returns:
        Tuple of (content_bytes, filename, mimetype)
    """
    from ...exporters import ExporterRegistry, ExportOptions

    # Normalize format
    format_lower = format.lower()

    # Get exporter from registry
    exporter = ExporterRegistry.get_exporter(format_lower)

    if exporter is None:
        available = ExporterRegistry.get_available_formats()
        raise ValueError(
            f"Unsupported export format: {format}. "
            f"Available formats: {', '.join(available)}"
        )

    # Title prepending is now handled by each exporter via _prepend_title_if_needed()
    # PDF and ODT exporters prepend titles; RIS and other formats ignore them

    # Create options
    options = ExportOptions(title=title)

    # Export
    result = exporter.export(markdown_content, options)

    logger.info(
        f"Generated {format_lower} in memory, size: {len(result.content)} bytes"
    )

    return result.content, result.filename, result.mimetype


def save_research_strategy(research_id, strategy_name, username=None):
    """
    Save the strategy used for a research to the database.

    Args:
        research_id: The ID of the research
        strategy_name: The name of the strategy used
        username: The username for database access (required for thread context)
    """
    try:
        logger.debug(
            f"save_research_strategy called with research_id={research_id}, strategy_name={strategy_name}"
        )
        with get_user_db_session(username) as session:
            try:
                # Check if a strategy already exists for this research
                existing_strategy = (
                    session.query(ResearchStrategy)
                    .filter_by(research_id=research_id)
                    .first()
                )

                if existing_strategy:
                    # Update existing strategy
                    existing_strategy.strategy_name = strategy_name
                    logger.debug(
                        f"Updating existing strategy for research {research_id}"
                    )
                else:
                    # Create new strategy record
                    new_strategy = ResearchStrategy(
                        research_id=research_id, strategy_name=strategy_name
                    )
                    session.add(new_strategy)
                    logger.debug(
                        f"Creating new strategy record for research {research_id}"
                    )

                session.commit()
                logger.info(
                    f"Saved strategy '{strategy_name}' for research {research_id}"
                )
            finally:
                session.close()
    except Exception:
        logger.exception("Error saving research strategy")


def get_research_strategy(research_id, username=None):
    """
    Get the strategy used for a research.

    Args:
        research_id: The ID of the research
        username: The username for database access (required for thread context)

    Returns:
        str: The strategy name or None if not found
    """
    try:
        with get_user_db_session(username) as session:
            try:
                strategy = (
                    session.query(ResearchStrategy)
                    .filter_by(research_id=research_id)
                    .first()
                )

                return strategy.strategy_name if strategy else None
            finally:
                session.close()
    except Exception:
        logger.exception("Error getting research strategy")
        return None


def start_research_process(
    research_id,
    query,
    mode,
    active_research,
    termination_flags,
    run_research_callback,
    **kwargs,
):
    """
    Start a research process in a background thread.

    Args:
        research_id: The ID of the research
        query: The research query
        mode: The research mode (quick/detailed)
        active_research: Dictionary of active research processes
        termination_flags: Dictionary of termination flags
        run_research_callback: The callback function to run the research
        **kwargs: Additional parameters to pass to the research process (model, search_engine, etc.)

    Returns:
        threading.Thread: The thread running the research
    """
    # Pass the app context to the thread.
    run_research_callback = thread_with_app_context(run_research_callback)

    # Start research process in a background thread
    thread = threading.Thread(
        target=run_research_callback,
        args=(
            thread_context(),
            research_id,
            query,
            mode,
            active_research,
            termination_flags,
        ),
        kwargs=kwargs,
    )
    thread.daemon = True
    thread.start()

    active_research[research_id] = {
        "thread": thread,
        "progress": 0,
        "status": ResearchStatus.IN_PROGRESS,
        "log": [],
        "settings": kwargs,  # Store settings for reference
    }

    return thread


def _generate_report_path(query: str) -> Path:
    """
    Generates a path for a new report file based on the query.

    Args:
        query: The query used for the report.

    Returns:
        The path that it generated.

    """
    # Generate a unique filename that does not contain
    # non-alphanumeric characters.
    query_hash = hashlib.md5(  # DevSkim: ignore DS126858
        query.encode("utf-8"), usedforsecurity=False
    ).hexdigest()[:10]
    return OUTPUT_DIR / (
        f"research_report_{query_hash}_{int(datetime.now(UTC).timestamp())}.md"
    )


@log_for_research
def run_research_process(
    research_id, query, mode, active_research, termination_flags, **kwargs
):
    """
    Run the research process in the background for a given research ID.

    Args:
        research_id: The ID of the research
        query: The research query
        mode: The research mode (quick/detailed)
        active_research: Dictionary of active research processes
        termination_flags: Dictionary of termination flags
        **kwargs: Additional parameters for the research (model_provider, model, search_engine, etc.)
                 MUST include 'username' for database access
    """

    # Extract username - required for database access
    username = kwargs.get("username")
    logger.info(f"Research thread started with username: {username}")
    if not username:
        logger.error("No username provided to research thread")
        raise ValueError("Username is required for research process")
    try:
        # Check if this research has been terminated before we even start
        if termination_flags.get(research_id):
            logger.info(
                f"Research {research_id} was terminated before starting"
            )
            cleanup_research_resources(
                research_id, active_research, termination_flags, username
            )
            return

        logger.info(
            f"Starting research process for ID {research_id}, query: {query}"
        )

        # Extract key parameters
        model_provider = kwargs.get("model_provider")
        model = kwargs.get("model")
        custom_endpoint = kwargs.get("custom_endpoint")
        search_engine = kwargs.get("search_engine")
        max_results = kwargs.get("max_results")
        time_period = kwargs.get("time_period")
        iterations = kwargs.get("iterations")
        questions_per_iteration = kwargs.get("questions_per_iteration")
        strategy = kwargs.get(
            "strategy", "source-based"
        )  # Default to source-based
        settings_snapshot = kwargs.get(
            "settings_snapshot", {}
        )  # Complete settings snapshot

        # Log settings snapshot to debug
        from ...settings.logger import log_settings

        log_settings(settings_snapshot, "Settings snapshot received in thread")

        # Strategy should already be saved in the database before thread starts
        logger.info(f"Research strategy: {strategy}")

        # Log all parameters for debugging
        logger.info(
            f"Research parameters: provider={model_provider}, model={model}, "
            f"search_engine={search_engine}, max_results={max_results}, "
            f"time_period={time_period}, iterations={iterations}, "
            f"questions_per_iteration={questions_per_iteration}, "
            f"custom_endpoint={custom_endpoint}, strategy={strategy}"
        )

        # Set up the AI Context Manager
        output_dir = OUTPUT_DIR / f"research_{research_id}"
        output_dir.mkdir(parents=True, exist_ok=True)

        # Create a settings context that uses snapshot if available, otherwise falls back to database
        # This allows the research to be independent of database during execution
        class SettingsContext:
            def __init__(self, snapshot, username):
                self.snapshot = snapshot or {}
                self.username = username
                # Extract values from setting objects if needed
                self.values = {}
                for key, setting in self.snapshot.items():
                    if isinstance(setting, dict) and "value" in setting:
                        # It's a full setting object, extract the value
                        self.values[key] = setting["value"]
                    else:
                        # It's already just a value
                        self.values[key] = setting

            def get_setting(self, key, default=None):
                """Get setting from snapshot only - no database access in threads"""
                if key in self.values:
                    return self.values[key]
                # No fallback to database - threads must use snapshot only
                logger.debug(
                    f"Setting '{key}' not found in snapshot, using default"
                )
                return default

        settings_context = SettingsContext(settings_snapshot, username)

        # Only log settings if explicitly enabled via LDR_LOG_SETTINGS env var
        from ...settings.logger import log_settings

        log_settings(
            settings_context.values, "SettingsContext values extracted"
        )

        # Set the settings context for this thread
        from ...config.thread_settings import (
            clear_settings_context,
            set_settings_context,
        )

        set_settings_context(settings_context)

        # Get user password if provided
        user_password = kwargs.get("user_password")

        # Create shared research context that can be updated during research
        shared_research_context = {
            "research_id": research_id,
            "research_query": query,
            "research_mode": mode,
            "research_phase": "init",
            "search_iteration": 0,
            "search_engines_planned": None,
            "search_engine_selected": search_engine,
            "username": username,  # Add username for queue operations
            "user_password": user_password,  # Add password for metrics access
        }

        # If this is a follow-up research, include the parent context
        if "research_context" in kwargs and kwargs["research_context"]:
            logger.info(
                f"Adding parent research context with {len(kwargs['research_context'].get('past_findings', ''))} chars of findings"
            )
            shared_research_context.update(kwargs["research_context"])

        # Do not log context keys as they may contain sensitive information
        logger.info(f"Created shared_research_context for user: {username}")

        # Set search context for search tracking
        set_search_context(shared_research_context)

        # Set up progress callback
        def progress_callback(message, progress_percent, metadata):
            # Frequent termination check
            if termination_flags.get(research_id):
                handle_termination(
                    research_id, active_research, termination_flags, username
                )
                raise Exception("Research was terminated by user")

            # Bind research_id to logger for this specific log
            bound_logger = logger.bind(research_id=research_id)
            bound_logger.log("MILESTONE", message)

            if "SEARCH_PLAN:" in message:
                engines = message.split("SEARCH_PLAN:")[1].strip()
                metadata["planned_engines"] = engines
                metadata["phase"] = "search_planning"  # Use existing phase
                # Update shared context for token tracking
                shared_research_context["search_engines_planned"] = engines
                shared_research_context["research_phase"] = "search_planning"

            if "ENGINE_SELECTED:" in message:
                engine = message.split("ENGINE_SELECTED:")[1].strip()
                metadata["selected_engine"] = engine
                metadata["phase"] = "search"  # Use existing 'search' phase
                # Update shared context for token tracking
                shared_research_context["search_engine_selected"] = engine
                shared_research_context["research_phase"] = "search"

            # Capture other research phases for better context tracking
            if metadata.get("phase"):
                shared_research_context["research_phase"] = metadata["phase"]

            # Update search iteration if available
            if "iteration" in metadata:
                shared_research_context["search_iteration"] = metadata[
                    "iteration"
                ]

            # Adjust progress based on research mode
            adjusted_progress = progress_percent
            if (
                mode == "detailed"
                and metadata.get("phase") == "output_generation"
            ):
                # For detailed mode, adjust the progress range for output generation
                adjusted_progress = min(80, progress_percent)
            elif (
                mode == "detailed"
                and metadata.get("phase") == "report_generation"
            ):
                # Scale the progress from 80% to 95% for the report generation phase
                if progress_percent is not None:
                    normalized = progress_percent / 100
                    adjusted_progress = 80 + (normalized * 15)
            elif (
                mode == "quick" and metadata.get("phase") == "output_generation"
            ):
                # For quick mode, ensure we're at least at 85% during output generation
                adjusted_progress = max(85, progress_percent)
                # Map any further progress within output_generation to 85-95% range
                if progress_percent is not None and progress_percent > 0:
                    normalized = progress_percent / 100
                    adjusted_progress = 85 + (normalized * 10)

            # Don't let progress go backwards
            if research_id in active_research and adjusted_progress is not None:
                current_progress = active_research[research_id].get(
                    "progress", 0
                )
                adjusted_progress = max(current_progress, adjusted_progress)

            # Update active research record
            if research_id in active_research:
                if adjusted_progress is not None:
                    active_research[research_id]["progress"] = adjusted_progress

                # Queue the progress update to be processed in main thread
                if adjusted_progress is not None:
                    from ..queue.processor_v2 import queue_processor

                    if username:
                        queue_processor.queue_progress_update(
                            username, research_id, adjusted_progress
                        )
                    else:
                        logger.warning(
                            f"Cannot queue progress update for research {research_id} - no username available"
                        )

                # Emit a socket event
                try:
                    # Basic event data
                    event_data = {"progress": adjusted_progress}

                    SocketIOService().emit_to_subscribers(
                        "progress", research_id, event_data
                    )
                except Exception:
                    logger.exception("Socket emit error (non-critical)")

        # Function to check termination during long-running operations
        def check_termination():
            if termination_flags.get(research_id):
                handle_termination(
                    research_id, active_research, termination_flags, username
                )
                raise Exception(
                    "Research was terminated by user during long-running operation"
                )
            return False  # Not terminated

        # Configure the system with the specified parameters
        use_llm = None
        if model or search_engine or model_provider:
            # Log that we're overriding system settings
            logger.info(
                f"Overriding system settings with: provider={model_provider}, model={model}, search_engine={search_engine}"
            )

        # Override LLM if model or model_provider specified
        if model or model_provider:
            try:
                # Get LLM with the overridden settings
                # Use the shared_research_context which includes username
                use_llm = get_llm(
                    model_name=model,
                    provider=model_provider,
                    openai_endpoint_url=custom_endpoint,
                    research_id=research_id,
                    research_context=shared_research_context,
                )

                logger.info(
                    f"Successfully set LLM to: provider={model_provider}, model={model}"
                )
            except Exception as e:
                logger.exception(
                    f"Error setting LLM provider={model_provider}, model={model}"
                )
                error_msg = str(e)
                # Surface configuration errors to user instead of silently continuing
                config_error_keywords = [
                    "model path",
                    "llamacpp",
                    "cannot connect",
                    "server",
                    "not configured",
                    "not responding",
                    "directory",
                    ".gguf",
                ]
                if any(
                    keyword in error_msg.lower()
                    for keyword in config_error_keywords
                ):
                    # This is a configuration error the user can fix
                    raise ValueError(
                        f"LLM Configuration Error: {error_msg}"
                    ) from e
                # For other errors, re-raise to avoid silent failures
                raise

        # Create search engine first if specified, to avoid default creation without username
        use_search = None
        if search_engine:
            try:
                # Create a new search object with these settings
                use_search = get_search(
                    search_tool=search_engine,
                    llm_instance=use_llm,
                    username=username,
                    settings_snapshot=settings_snapshot,
                )
                logger.info(
                    f"Successfully created search engine: {search_engine}"
                )
            except Exception as e:
                logger.exception(
                    f"Error creating search engine {search_engine}"
                )
                error_msg = str(e)
                # Surface configuration errors to user instead of silently continuing
                config_error_keywords = [
                    "searxng",
                    "instance_url",
                    "api_key",
                    "cannot connect",
                    "connection",
                    "timeout",
                    "not configured",
                ]
                if any(
                    keyword in error_msg.lower()
                    for keyword in config_error_keywords
                ):
                    # This is a configuration error the user can fix
                    raise ValueError(
                        f"Search Engine Configuration Error ({search_engine}): {error_msg}"
                    ) from e
                # For other errors, re-raise to avoid silent failures
                raise

        # Set the progress callback in the system
        system = AdvancedSearchSystem(
            llm=use_llm,
            search=use_search,
            strategy_name=strategy,
            max_iterations=iterations,
            questions_per_iteration=questions_per_iteration,
            username=username,
            settings_snapshot=settings_snapshot,
            research_id=research_id,
            research_context=shared_research_context,
        )
        system.set_progress_callback(progress_callback)

        # Run the search
        progress_callback("Starting research process", 5, {"phase": "init"})

        try:
            results = system.analyze_topic(query)
            if mode == "quick":
                progress_callback(
                    "Search complete, preparing to generate summary...",
                    85,
                    {"phase": "output_generation"},
                )
            else:
                progress_callback(
                    "Search complete, generating output",
                    80,
                    {"phase": "output_generation"},
                )
        except Exception as search_error:
            # Better handling of specific search errors
            error_message = str(search_error)
            error_type = "unknown"

            # Extract error details for common issues
            if "status code: 503" in error_message:
                error_message = "Ollama AI service is unavailable (HTTP 503). Please check that Ollama is running properly on your system."
                error_type = "ollama_unavailable"
            elif "status code: 404" in error_message:
                error_message = "Ollama model not found (HTTP 404). Please check that you have pulled the required model."
                error_type = "model_not_found"
            elif "status code:" in error_message:
                # Extract the status code for other HTTP errors
                status_code = error_message.split("status code:")[1].strip()
                error_message = f"API request failed with status code {status_code}. Please check your configuration."
                error_type = "api_error"
            elif "connection" in error_message.lower():
                error_message = "Connection error. Please check that your LLM service (Ollama/API) is running and accessible."
                error_type = "connection_error"

            # Raise with improved error message
            raise Exception(f"{error_message} (Error type: {error_type})")

        # Generate output based on mode
        if mode == "quick":
            # Quick Summary
            if results.get("findings") or results.get("formatted_findings"):
                raw_formatted_findings = results["formatted_findings"]

                # Check if formatted_findings contains an error message
                if isinstance(
                    raw_formatted_findings, str
                ) and raw_formatted_findings.startswith("Error:"):
                    logger.exception(
                        f"Detected error in formatted findings: {raw_formatted_findings[:100]}..."
                    )

                    # Determine error type for better user feedback
                    error_type = "unknown"
                    error_message = raw_formatted_findings.lower()

                    if (
                        "token limit" in error_message
                        or "context length" in error_message
                    ):
                        error_type = "token_limit"
                        # Log specific error type
                        logger.warning(
                            "Detected token limit error in synthesis"
                        )

                        # Update progress with specific error type
                        progress_callback(
                            "Synthesis hit token limits. Attempting fallback...",
                            87,
                            {
                                "phase": "synthesis_error",
                                "error_type": error_type,
                            },
                        )
                    elif (
                        "timeout" in error_message
                        or "timed out" in error_message
                    ):
                        error_type = "timeout"
                        logger.warning("Detected timeout error in synthesis")
                        progress_callback(
                            "Synthesis timed out. Attempting fallback...",
                            87,
                            {
                                "phase": "synthesis_error",
                                "error_type": error_type,
                            },
                        )
                    elif "rate limit" in error_message:
                        error_type = "rate_limit"
                        logger.warning("Detected rate limit error in synthesis")
                        progress_callback(
                            "LLM rate limit reached. Attempting fallback...",
                            87,
                            {
                                "phase": "synthesis_error",
                                "error_type": error_type,
                            },
                        )
                    elif (
                        "connection" in error_message
                        or "network" in error_message
                    ):
                        error_type = "connection"
                        logger.warning("Detected connection error in synthesis")
                        progress_callback(
                            "Connection issue with LLM. Attempting fallback...",
                            87,
                            {
                                "phase": "synthesis_error",
                                "error_type": error_type,
                            },
                        )
                    elif (
                        "llm error" in error_message
                        or "final answer synthesis fail" in error_message
                    ):
                        error_type = "llm_error"
                        logger.warning(
                            "Detected general LLM error in synthesis"
                        )
                        progress_callback(
                            "LLM error during synthesis. Attempting fallback...",
                            87,
                            {
                                "phase": "synthesis_error",
                                "error_type": error_type,
                            },
                        )
                    else:
                        # Generic error
                        logger.warning("Detected unknown error in synthesis")
                        progress_callback(
                            "Error during synthesis. Attempting fallback...",
                            87,
                            {
                                "phase": "synthesis_error",
                                "error_type": "unknown",
                            },
                        )

                    # Extract synthesized content from findings if available
                    synthesized_content = ""
                    for finding in results.get("findings", []):
                        if finding.get("phase") == "Final synthesis":
                            synthesized_content = finding.get("content", "")
                            break

                    # Use synthesized content as fallback
                    if (
                        synthesized_content
                        and not synthesized_content.startswith("Error:")
                    ):
                        logger.info(
                            "Using existing synthesized content as fallback"
                        )
                        raw_formatted_findings = synthesized_content

                    # Or use current_knowledge as another fallback
                    elif results.get("current_knowledge"):
                        logger.info("Using current_knowledge as fallback")
                        raw_formatted_findings = results["current_knowledge"]

                    # Or combine all finding contents as last resort
                    elif results.get("findings"):
                        logger.info("Combining all findings as fallback")
                        # First try to use any findings that are not errors
                        valid_findings = [
                            f"## {finding.get('phase', 'Finding')}\n\n{finding.get('content', '')}"
                            for finding in results.get("findings", [])
                            if finding.get("content")
                            and not finding.get("content", "").startswith(
                                "Error:"
                            )
                        ]

                        if valid_findings:
                            raw_formatted_findings = (
                                "# Research Results (Fallback Mode)\n\n"
                            )
                            raw_formatted_findings += "\n\n".join(
                                valid_findings
                            )
                            raw_formatted_findings += f"\n\n## Error Information\n{raw_formatted_findings}"
                        else:
                            # Last resort: use everything including errors
                            raw_formatted_findings = (
                                "# Research Results (Emergency Fallback)\n\n"
                            )
                            raw_formatted_findings += "The system encountered errors during final synthesis.\n\n"
                            raw_formatted_findings += "\n\n".join(
                                f"## {finding.get('phase', 'Finding')}\n\n{finding.get('content', '')}"
                                for finding in results.get("findings", [])
                                if finding.get("content")
                            )

                    progress_callback(
                        f"Using fallback synthesis due to {error_type} error",
                        88,
                        {
                            "phase": "synthesis_fallback",
                            "error_type": error_type,
                        },
                    )

                logger.info(
                    "Found formatted_findings of length: %s",
                    len(str(raw_formatted_findings)),
                )

                try:
                    # Check if we have an error in the findings and use enhanced error handling
                    if isinstance(
                        raw_formatted_findings, str
                    ) and raw_formatted_findings.startswith("Error:"):
                        logger.info(
                            "Generating enhanced error report using ErrorReportGenerator"
                        )

                        # Get LLM for error explanation if available
                        try:
                            llm = get_llm(
                                research_id=research_id,
                                research_context=shared_research_context,
                            )
                        except Exception:
                            llm = None
                            logger.warning(
                                "Could not get LLM for error explanation"
                            )

                        # Generate comprehensive error report
                        error_generator = ErrorReportGenerator(llm)
                        clean_markdown = error_generator.generate_error_report(
                            error_message=raw_formatted_findings,
                            query=query,
                            partial_results=results,
                            search_iterations=results.get("iterations", 0),
                            research_id=research_id,
                        )

                        logger.info(
                            "Generated enhanced error report with %d characters",
                            len(clean_markdown),
                        )
                    else:
                        # Get the synthesized content from the LLM directly
                        clean_markdown = raw_formatted_findings

                    # Extract all sources from findings to add them to the summary
                    all_links = []
                    for finding in results.get("findings", []):
                        search_results = finding.get("search_results", [])
                        if search_results:
                            try:
                                links = extract_links_from_search_results(
                                    search_results
                                )
                                all_links.extend(links)
                            except Exception:
                                logger.exception(
                                    "Error processing search results/links"
                                )

                    logger.info(
                        "Successfully converted to clean markdown of length: %s",
                        len(clean_markdown),
                    )

                    # First send a progress update for generating the summary
                    progress_callback(
                        "Generating clean summary from research data...",
                        90,
                        {"phase": "output_generation"},
                    )

                    # Send progress update for saving report
                    progress_callback(
                        "Saving research report to database...",
                        95,
                        {"phase": "report_complete"},
                    )

                    # Format citations in the markdown content
                    formatter = get_citation_formatter()
                    formatted_content = formatter.format_document(
                        clean_markdown
                    )

                    # Prepare complete report content
                    full_report_content = f"""{formatted_content}

## Research Metrics
- Search Iterations: {results["iterations"]}
- Generated at: {datetime.now(UTC).isoformat()}
"""

                    # Save sources to database
                    from .research_sources_service import ResearchSourcesService

                    sources_service = ResearchSourcesService()
                    if all_links:
                        logger.info(
                            f"Quick summary: Saving {len(all_links)} sources to database"
                        )
                        sources_saved = sources_service.save_research_sources(
                            research_id=research_id,
                            sources=all_links,
                            username=username,
                        )
                        logger.info(
                            f"Quick summary: Saved {sources_saved} sources for research {research_id}"
                        )

                    # Save report using storage abstraction
                    from ...storage import get_report_storage

                    with get_user_db_session(username) as db_session:
                        storage = get_report_storage(session=db_session)

                        # Prepare metadata
                        metadata = {
                            "iterations": results["iterations"],
                            "generated_at": datetime.now(UTC).isoformat(),
                        }

                        # Save report using storage abstraction
                        success = storage.save_report(
                            research_id=research_id,
                            content=full_report_content,
                            metadata=metadata,
                            username=username,
                        )

                        if not success:
                            raise Exception("Failed to save research report")

                        logger.info(
                            f"Report saved for research_id: {research_id}"
                        )

                    # Skip export to additional formats - we're storing in database only

                    # Update research status in database
                    completed_at = datetime.now(UTC).isoformat()

                    with get_user_db_session(username) as db_session:
                        research = (
                            db_session.query(ResearchHistory)
                            .filter_by(id=research_id)
                            .first()
                        )

                        # Preserve existing metadata and update with new values
                        metadata = _parse_research_metadata(
                            research.research_meta
                        )

                        metadata.update(
                            {
                                "iterations": results["iterations"],
                                "generated_at": datetime.now(UTC).isoformat(),
                            }
                        )

                        # Use the helper function for consistent duration calculation
                        duration_seconds = calculate_duration(
                            research.created_at, completed_at
                        )

                        research.status = ResearchStatus.COMPLETED
                        research.completed_at = completed_at
                        research.duration_seconds = duration_seconds
                        # Note: report_content is saved by CachedResearchService
                        # report_path is not used in encrypted database version

                        # Generate headline and topics only for news searches
                        if (
                            metadata.get("is_news_search")
                            or metadata.get("search_type") == "news_analysis"
                        ):
                            try:
                                from ...news.utils.headline_generator import (
                                    generate_headline,
                                )
                                from ...news.utils.topic_generator import (
                                    generate_topics,
                                )

                                # Get the report content from database for better headline/topic generation
                                report_content = ""
                                try:
                                    research = (
                                        db_session.query(ResearchHistory)
                                        .filter_by(id=research_id)
                                        .first()
                                    )
                                    if research and research.report_content:
                                        report_content = research.report_content
                                        logger.info(
                                            f"Retrieved {len(report_content)} chars from database for headline generation"
                                        )
                                    else:
                                        logger.warning(
                                            f"No report content found in database for research_id: {research_id}"
                                        )
                                except Exception as e:
                                    logger.warning(
                                        f"Could not retrieve report content from database: {e}"
                                    )

                                # Generate headline
                                logger.info(
                                    f"Generating headline for query: {query[:100]}"
                                )
                                headline = generate_headline(
                                    query, report_content
                                )
                                metadata["generated_headline"] = headline

                                # Generate topics
                                logger.info(
                                    f"Generating topics with category: {metadata.get('category', 'News')}"
                                )
                                topics = generate_topics(
                                    query=query,
                                    findings=report_content,
                                    category=metadata.get("category", "News"),
                                    max_topics=6,
                                )
                                metadata["generated_topics"] = topics

                                logger.info(f"Generated headline: {headline}")
                                logger.info(f"Generated topics: {topics}")

                            except Exception as e:
                                logger.warning(
                                    f"Could not generate headline/topics: {e}"
                                )

                        research.research_meta = metadata

                        db_session.commit()
                        logger.info(
                            f"Database commit completed for research_id: {research_id}"
                        )

                        # Update subscription if this was triggered by a subscription
                        if metadata.get("subscription_id"):
                            try:
                                from ...news.subscription_manager.storage import (
                                    SQLSubscriptionStorage,
                                )
                                from datetime import (
                                    datetime as dt,
                                    timezone,
                                    timedelta,
                                )

                                sub_storage = SQLSubscriptionStorage()
                                subscription_id = metadata["subscription_id"]

                                # Get subscription to find refresh interval
                                subscription = sub_storage.get(subscription_id)
                                if subscription:
                                    refresh_minutes = subscription.get(
                                        "refresh_minutes", 240
                                    )
                                    now = dt.now(timezone.utc)
                                    next_refresh = now + timedelta(
                                        minutes=refresh_minutes
                                    )

                                    # Update refresh times
                                    sub_storage.update_refresh_time(
                                        subscription_id=subscription_id,
                                        last_refresh=now,
                                        next_refresh=next_refresh,
                                    )

                                    # Increment stats
                                    sub_storage.increment_stats(
                                        subscription_id, 1
                                    )

                                    logger.info(
                                        f"Updated subscription {subscription_id} refresh times"
                                    )
                            except Exception as e:
                                logger.warning(
                                    f"Could not update subscription refresh time: {e}"
                                )

                    logger.info(
                        f"Database updated successfully for research_id: {research_id}"
                    )

                    # Send the final completion message
                    progress_callback(
                        "Research completed successfully",
                        100,
                        {"phase": "complete"},
                    )

                    # Clean up resources
                    logger.info(
                        "Cleaning up resources for research_id: %s", research_id
                    )
                    cleanup_research_resources(
                        research_id,
                        active_research,
                        termination_flags,
                        username,
                    )
                    logger.info(
                        "Resources cleaned up for research_id: %s", research_id
                    )

                except Exception as inner_e:
                    logger.exception("Error during quick summary generation")
                    raise Exception(
                        f"Error generating quick summary: {inner_e!s}"
                    )
            else:
                raise Exception(
                    "No research findings were generated. Please try again."
                )
        else:
            # Full Report
            progress_callback(
                "Generating detailed report...",
                85,
                {"phase": "report_generation"},
            )

            # Extract the search system from the results if available
            search_system = results.get("search_system", None)

            # Pass the existing search system to maintain citation indices
            report_generator = IntegratedReportGenerator(
                search_system=search_system,
                settings_snapshot=settings_snapshot,
            )
            final_report = report_generator.generate_report(results, query)

            progress_callback(
                "Report generation complete", 95, {"phase": "report_complete"}
            )

            # Format citations in the report content
            formatter = get_citation_formatter()
            formatted_content = formatter.format_document(
                final_report["content"]
            )

            # Save sources to database
            from .research_sources_service import ResearchSourcesService

            sources_service = ResearchSourcesService()
            if (
                hasattr(search_system, "all_links_of_system")
                and search_system.all_links_of_system
            ):
                logger.info(
                    f"Saving {len(search_system.all_links_of_system)} sources to database"
                )
                sources_saved = sources_service.save_research_sources(
                    research_id=research_id,
                    sources=search_system.all_links_of_system,
                    username=username,
                )
                logger.info(
                    f"Saved {sources_saved} sources for research {research_id}"
                )

            # Save report to database
            with get_user_db_session(username) as db_session:
                # Update metadata
                metadata = final_report["metadata"]
                metadata["iterations"] = results["iterations"]

                # Save report to database
                try:
                    research = (
                        db_session.query(ResearchHistory)
                        .filter_by(id=research_id)
                        .first()
                    )

                    if not research:
                        logger.error(f"Research {research_id} not found")
                        success = False
                    else:
                        research.report_content = formatted_content
                        if research.research_meta:
                            research.research_meta.update(metadata)
                        else:
                            research.research_meta = metadata
                        db_session.commit()
                        success = True
                        logger.info(
                            f"Saved report for research {research_id} to database"
                        )
                except Exception:
                    logger.exception("Error saving report to database")
                    db_session.rollback()
                    success = False

                if not success:
                    raise Exception("Failed to save research report")

                logger.info(
                    f"Report saved to database for research_id: {research_id}"
                )

            # Update research status in database
            completed_at = datetime.now(UTC).isoformat()

            with get_user_db_session(username) as db_session:
                research = (
                    db_session.query(ResearchHistory)
                    .filter_by(id=research_id)
                    .first()
                )

                # Preserve existing metadata and merge with report metadata
                metadata = _parse_research_metadata(research.research_meta)

                metadata.update(final_report["metadata"])
                metadata["iterations"] = results["iterations"]

                # Use the helper function for consistent duration calculation
                duration_seconds = calculate_duration(
                    research.created_at, completed_at
                )

                research.status = ResearchStatus.COMPLETED
                research.completed_at = completed_at
                research.duration_seconds = duration_seconds
                # Note: report_content is saved by CachedResearchService
                # report_path is not used in encrypted database version

                # Generate headline and topics only for news searches
                if (
                    metadata.get("is_news_search")
                    or metadata.get("search_type") == "news_analysis"
                ):
                    try:
                        from ..news.utils.headline_generator import (
                            generate_headline,
                        )
                        from ..news.utils.topic_generator import (
                            generate_topics,
                        )

                        # Get the report content from database for better headline/topic generation
                        report_content = ""
                        try:
                            research = (
                                db_session.query(ResearchHistory)
                                .filter_by(id=research_id)
                                .first()
                            )
                            if research and research.report_content:
                                report_content = research.report_content
                            else:
                                logger.warning(
                                    f"No report content found in database for research_id: {research_id}"
                                )
                        except Exception as e:
                            logger.warning(
                                f"Could not retrieve report content from database: {e}"
                            )

                        # Generate headline
                        headline = generate_headline(query, report_content)
                        metadata["generated_headline"] = headline

                        # Generate topics
                        topics = generate_topics(
                            query=query,
                            findings=report_content,
                            category=metadata.get("category", "News"),
                            max_topics=6,
                        )
                        metadata["generated_topics"] = topics

                        logger.info(f"Generated headline: {headline}")
                        logger.info(f"Generated topics: {topics}")

                    except Exception as e:
                        logger.warning(
                            f"Could not generate headline/topics: {e}"
                        )

                research.research_meta = metadata

                db_session.commit()

                # Update subscription if this was triggered by a subscription
                if metadata.get("subscription_id"):
                    try:
                        from ...news.subscription_manager.storage import (
                            SQLSubscriptionStorage,
                        )
                        from datetime import datetime as dt, timezone, timedelta

                        sub_storage = SQLSubscriptionStorage()
                        subscription_id = metadata["subscription_id"]

                        # Get subscription to find refresh interval
                        subscription = sub_storage.get(subscription_id)
                        if subscription:
                            refresh_minutes = subscription.get(
                                "refresh_minutes", 240
                            )
                            now = dt.now(timezone.utc)
                            next_refresh = now + timedelta(
                                minutes=refresh_minutes
                            )

                            # Update refresh times
                            sub_storage.update_refresh_time(
                                subscription_id=subscription_id,
                                last_refresh=now,
                                next_refresh=next_refresh,
                            )

                            # Increment stats
                            sub_storage.increment_stats(subscription_id, 1)

                            logger.info(
                                f"Updated subscription {subscription_id} refresh times"
                            )
                    except Exception as e:
                        logger.warning(
                            f"Could not update subscription refresh time: {e}"
                        )

            progress_callback(
                "Research completed successfully",
                100,
                {"phase": "complete"},
            )

            # Clean up resources
            cleanup_research_resources(
                research_id, active_research, termination_flags, username
            )

    except Exception as e:
        # Handle error
        error_message = f"Research failed: {e!s}"
        logger.exception(error_message)

        try:
            # Check for common Ollama error patterns in the exception and provide more user-friendly errors
            user_friendly_error = str(e)
            error_context = {}

            if "Error type: ollama_unavailable" in user_friendly_error:
                user_friendly_error = "Ollama AI service is unavailable. Please check that Ollama is running properly on your system."
                error_context = {
                    "solution": "Start Ollama with 'ollama serve' or check if it's installed correctly."
                }
            elif "Error type: model_not_found" in user_friendly_error:
                user_friendly_error = "Required Ollama model not found. Please pull the model first."
                error_context = {
                    "solution": "Run 'ollama pull mistral' to download the required model."
                }
            elif "Error type: connection_error" in user_friendly_error:
                user_friendly_error = "Connection error with LLM service. Please check that your AI service is running."
                error_context = {
                    "solution": "Ensure Ollama or your API service is running and accessible."
                }
            elif "Error type: api_error" in user_friendly_error:
                # Keep the original error message as it's already improved
                error_context = {
                    "solution": "Check API configuration and credentials."
                }

            # Generate enhanced error report for failed research
            enhanced_report_content = None
            try:
                # Get LLM for error explanation if available
                try:
                    llm = get_llm(
                        research_id=research_id,
                        research_context=shared_research_context,
                    )
                except Exception:
                    llm = None
                    logger.warning(
                        "Could not get LLM for error explanation in failure handler"
                    )

                # Get partial results if they exist
                partial_results = results if "results" in locals() else None
                search_iterations = (
                    results.get("iterations", 0) if partial_results else 0
                )

                # Generate comprehensive error report
                error_generator = ErrorReportGenerator(llm)
                enhanced_report_content = error_generator.generate_error_report(
                    error_message=f"Research failed: {e!s}",
                    query=query,
                    partial_results=partial_results,
                    search_iterations=search_iterations,
                    research_id=research_id,
                )

                logger.info(
                    "Generated enhanced error report for failed research (length: %d)",
                    len(enhanced_report_content),
                )

                # Save enhanced error report to encrypted database
                try:
                    # Get username from the research context
                    username = getattr(g, "username", None) or session.get(
                        "username"
                    )
                    if username:
                        from ...storage import get_report_storage

                        with get_user_db_session(username) as db_session:
                            storage = get_report_storage(session=db_session)
                            success = storage.save_report(
                                research_id=research_id,
                                content=enhanced_report_content,
                                metadata={"error_report": True},
                                username=username,
                            )
                            if success:
                                logger.info(
                                    "Saved enhanced error report to encrypted database for research %s",
                                    research_id,
                                )
                            else:
                                logger.warning(
                                    "Failed to save enhanced error report to database for research %s",
                                    research_id,
                                )
                    else:
                        logger.warning(
                            "Cannot save error report: username not available"
                        )

                except Exception as report_error:
                    logger.exception(
                        "Failed to save enhanced error report: %s", report_error
                    )

            except Exception as error_gen_error:
                logger.exception(
                    "Failed to generate enhanced error report: %s",
                    error_gen_error,
                )
                enhanced_report_content = None

            # Get existing metadata from database first
            existing_metadata = {}
            try:
                # Get username from the research context
                username = getattr(g, "username", None) or session.get(
                    "username"
                )
                if username:
                    with get_user_db_session(username) as db_session:
                        research = (
                            db_session.query(ResearchHistory)
                            .filter_by(id=research_id)
                            .first()
                        )
                        if research and research.research_meta:
                            existing_metadata = dict(research.research_meta)
            except Exception:
                logger.exception("Failed to get existing metadata")

            # Update metadata with more context about the error while preserving existing values
            metadata = existing_metadata
            metadata.update({"phase": "error", "error": user_friendly_error})
            if error_context:
                metadata.update(error_context)
            if enhanced_report_content:
                metadata["has_enhanced_report"] = True

            # If we still have an active research record, update its log
            if research_id in active_research:
                progress_callback(user_friendly_error, None, metadata)

            # If termination was requested, mark as suspended instead of failed
            status = (
                ResearchStatus.SUSPENDED
                if (termination_flags.get(research_id))
                else ResearchStatus.FAILED
            )
            message = (
                "Research was terminated by user"
                if status == ResearchStatus.SUSPENDED
                else user_friendly_error
            )

            # Calculate duration up to termination point - using UTC consistently
            now = datetime.now(UTC)
            completed_at = now.isoformat()

            # NOTE: Database updates from threads are handled by queue processor
            # The queue_processor.queue_error_update() method is already being used below
            # to safely update the database from the main thread

            # Queue the error update to be processed in main thread
            # Using the queue processor v2 system
            from ..queue.processor_v2 import queue_processor

            if username:
                queue_processor.queue_error_update(
                    username=username,
                    research_id=research_id,
                    status=status,
                    error_message=message,
                    metadata=metadata,
                    completed_at=completed_at,
                    report_path=None,
                )
                logger.info(
                    f"Queued error update for research {research_id} with status '{status}'"
                )
            else:
                logger.error(
                    f"Cannot queue error update for research {research_id} - no username provided. "
                    f"Status: '{status}', Message: {message}"
                )

            try:
                SocketIOService().emit_to_subscribers(
                    "research_progress",
                    research_id,
                    {"status": status, "error": message},
                )
            except Exception:
                logger.exception("Failed to emit error via socket")

        except Exception:
            logger.exception("Error in error handler")

        # Clean up resources
        cleanup_research_resources(
            research_id, active_research, termination_flags, username
        )

    finally:
        # Clear thread-local contexts to prevent leaks when threads are reused
        from ...utilities.thread_context import clear_search_context
        from ...config.thread_settings import clear_settings_context

        clear_search_context()
        clear_settings_context()


def cleanup_research_resources(
    research_id, active_research, termination_flags, username=None
):
    """
    Clean up resources for a completed research.

    Args:
        research_id: The ID of the research
        active_research: Dictionary of active research processes
        termination_flags: Dictionary of termination flags
        username: The username for database access (required for thread context)
    """
    logger.info("Cleaning up resources for research %s", research_id)

    # For testing: Add a small delay to simulate research taking time
    # This helps test concurrent research limits
    from ...settings.env_registry import is_test_mode

    if is_test_mode():
        import time

        logger.info(
            f"Test mode: Adding 5 second delay before cleanup for {research_id}"
        )
        time.sleep(5)

    # Get the current status from the database to determine the final status message
    current_status = ResearchStatus.COMPLETED  # Default

    # NOTE: Queue processor already handles database updates from the main thread
    # The notify_research_completed() method is called at the end of this function
    # which safely updates the database status

    # Notify queue processor that research completed
    # This uses processor_v2 which handles database updates in the main thread
    # avoiding the Flask request context issues that occur in background threads
    from ..queue.processor_v2 import queue_processor

    if username:
        queue_processor.notify_research_completed(username, research_id)
        logger.info(
            f"Notified queue processor of completion for research {research_id} (user: {username})"
        )
    else:
        logger.warning(
            f"Cannot notify completion for research {research_id} - no username provided"
        )

    # Remove from active research
    if research_id in active_research:
        del active_research[research_id]

    # Remove from termination flags
    if research_id in termination_flags:
        del termination_flags[research_id]

    # Send a final message to subscribers
    try:
        # Import here to avoid circular imports
        from ..routes.globals import get_globals

        globals_dict = get_globals()
        socket_subscriptions = globals_dict.get("socket_subscriptions", {})

        # Send a final message to any remaining subscribers with explicit status
        if socket_subscriptions.get(research_id):
            # Use the proper status message based on database status
            if current_status in (
                ResearchStatus.SUSPENDED,
                ResearchStatus.FAILED,
            ):
                final_message = {
                    "status": current_status,
                    "message": f"Research was {current_status}",
                    "progress": 0,  # For suspended research, show 0% not 100%
                }
            else:
                final_message = {
                    "status": ResearchStatus.COMPLETED,
                    "message": "Research process has ended and resources have been cleaned up",
                    "progress": 100,
                }

            logger.info(
                "Sending final %s socket message for research %s",
                current_status,
                research_id,
            )

            SocketIOService().emit_to_subscribers(
                "research_progress", research_id, final_message
            )

    except Exception:
        logger.exception("Error sending final cleanup message")


def handle_termination(
    research_id, active_research, termination_flags, username=None
):
    """
    Handle the termination of a research process.

    Args:
        research_id: The ID of the research
        active_research: Dictionary of active research processes
        termination_flags: Dictionary of termination flags
        username: The username for database access (required for thread context)
    """
    logger.info(f"Handling termination for research {research_id}")

    # Queue the status update to be processed in the main thread
    # This avoids Flask request context errors in background threads
    try:
        from ..queue.processor_v2 import queue_processor

        now = datetime.now(UTC)
        completed_at = now.isoformat()

        # Queue the suspension update
        queue_processor.queue_error_update(
            username=username,
            research_id=research_id,
            status=ResearchStatus.SUSPENDED,
            error_message="Research was terminated by user",
            metadata={"terminated_at": completed_at},
            completed_at=completed_at,
            report_path=None,
        )

        logger.info(f"Queued suspension update for research {research_id}")
    except Exception:
        logger.exception(
            f"Error queueing termination update for research {research_id}"
        )

    # Clean up resources (this already handles things properly)
    cleanup_research_resources(
        research_id, active_research, termination_flags, username
    )


def cancel_research(research_id, username=None):
    """
    Cancel/terminate a research process using ORM.

    Args:
        research_id: The ID of the research to cancel
        username: The username of the user cancelling the research (optional, will try to get from session if not provided)

    Returns:
        bool: True if the research was found and cancelled, False otherwise
    """
    try:
        # Import globals from research routes
        from ..routes.globals import get_globals

        globals_dict = get_globals()
        active_research = globals_dict["active_research"]
        termination_flags = globals_dict["termination_flags"]

        # Set termination flag
        termination_flags[research_id] = True

        # Check if the research is active
        if research_id in active_research:
            # Call handle_termination to update database
            handle_termination(
                research_id, active_research, termination_flags, username
            )
            return True
        else:
            # Update database directly if not found in active_research
            # Get username from parameter or session
            if not username:
                from flask import session

                username = session.get("username")

            if not username:
                logger.warning(
                    f"No username available for cancelling research {research_id}"
                )
                return False

            try:
                with get_user_db_session(username) as db_session:
                    research = (
                        db_session.query(ResearchHistory)
                        .filter_by(id=research_id)
                        .first()
                    )
                    if not research:
                        logger.info(
                            f"Research {research_id} not found in database"
                        )
                        return False

                    # Check if already in a terminal state
                    if research.status in (
                        ResearchStatus.COMPLETED,
                        ResearchStatus.SUSPENDED,
                        ResearchStatus.FAILED,
                        ResearchStatus.ERROR,
                    ):
                        logger.info(
                            f"Research {research_id} already in terminal state: {research.status}"
                        )
                        return True  # Consider this a success since it's already stopped

                    # If it exists but isn't in active_research, still update status
                    research.status = ResearchStatus.SUSPENDED
                    db_session.commit()
                    logger.info(
                        f"Successfully suspended research {research_id}"
                    )
            except Exception:
                logger.exception(
                    f"Error accessing database for research {research_id}"
                )
                return False

        return True
    except Exception:
        logger.exception(
            f"Unexpected error in cancel_research for {research_id}"
        )
        return False
