"""
Settings Routes Module

This module handles all settings-related HTTP endpoints for the application.

CHECKBOX HANDLING PATTERN:
--------------------------
This module supports TWO submission modes to handle checkboxes correctly:

**MODE 1: AJAX/JSON Submission (Primary - /save_all_settings)**
- JavaScript intercepts form submission with e.preventDefault()
- Checkbox values read directly from DOM via checkbox.checked
- Data sent as JSON: {"setting.key": true/false}
- Hidden fallback inputs are managed but NOT used in this mode
- Provides better UX with instant feedback and validation

**MODE 2: Traditional POST Submission (Fallback - /save_settings)**
- Used when JavaScript is disabled (accessibility/no-JS environments)
- Browser submits form data naturally via request.form
- Hidden fallback pattern CRITICAL here:
  * Checked checkbox: Submits checkbox value, hidden input disabled
  * Unchecked checkbox: Submits hidden input value "false"
- Ensures unchecked checkboxes are captured (HTML limitation workaround)

**Implementation Details:**
1. Each checkbox has `data-hidden-fallback` attribute → hidden input ID
2. checkbox_handler.js manages hidden input disabled state
3. AJAX mode: settings.js reads checkbox.checked directly (lines 2233-2240)
4. POST mode: Flask reads request.form including enabled hidden inputs
5. Both modes use convert_setting_value() for consistent boolean conversion

**Why Both Patterns?**
- AJAX: Better UX, immediate validation, no page reload
- Traditional POST: Accessibility, progressive enhancement, JavaScript-free operation
- Hidden inputs: Only meaningful for traditional POST, ignored in AJAX mode

This dual-mode approach ensures the app works for all users while providing
optimal experience when JavaScript is available.
"""

import json
import platform
import subprocess
from typing import Any, Optional, Tuple
from datetime import datetime, UTC, timedelta

import requests
from flask import (
    Blueprint,
    flash,
    jsonify,
    redirect,
    request,
    session,
    url_for,
)
from flask_wtf.csrf import generate_csrf
from loguru import logger

from ...config.paths import get_data_directory, get_encrypted_database_path
from ...database.models import Setting, SettingType
from ...database.session_context import get_user_db_session
from ...database.encrypted_db import db_manager
from ...utilities.db_utils import get_settings_manager
from ...utilities.url_utils import normalize_url
from ..auth.decorators import login_required
from ...settings import SettingsManager
from ...settings.manager import get_typed_setting_value, parse_boolean
from ..services.settings_service import (
    create_or_update_setting,
    set_setting,
)
from ..utils.templates import render_template_with_defaults
from ..server_config import sync_from_settings
from ...llm.providers.implementations.custom_openai_endpoint import (
    CustomOpenAIEndpointProvider,
)

# Create a Blueprint for settings
settings_bp = Blueprint("settings", __name__, url_prefix="/settings")

# Settings with dynamically populated options (excluded from validation)
DYNAMIC_SETTINGS = ["llm.provider", "llm.model", "search.tool"]


def _get_setting_from_session(key: str, default=None):
    """Helper to get a setting using the current session context."""
    username = session.get("username")
    with get_user_db_session(username) as db_session:
        if db_session:
            settings_manager = get_settings_manager(db_session, username)
            return settings_manager.get_setting(key, default)
    return default


def calculate_warnings():
    """Calculate current warning conditions based on settings"""
    warnings = []

    try:
        # Get current settings using proper session context
        username = session.get("username")
        with get_user_db_session(username) as db_session:
            if db_session:
                settings_manager = get_settings_manager(db_session, username)
                provider = settings_manager.get_setting(
                    "llm.provider", "ollama"
                ).lower()
                local_context = settings_manager.get_setting(
                    "llm.local_context_window_size", 4096
                )

                # Get dismissal settings
                dismiss_high_context = settings_manager.get_setting(
                    "app.warnings.dismiss_high_context", False
                )

                logger.debug(
                    f"Starting warning calculation - provider={provider}"
                )

        # Check warning conditions
        is_local_provider = provider in [
            "ollama",
            "llamacpp",
            "lmstudio",
            "vllm",
        ]

        # High context warning for local providers
        if (
            is_local_provider
            and local_context > 8192
            and not dismiss_high_context
        ):
            warnings.append(
                {
                    "type": "high_context",
                    "icon": "⚠️",
                    "title": "High Context Warning",
                    "message": f"Context size ({local_context:,} tokens) may cause memory issues with {provider}. Increase VRAM or reduce context size if you experience slowdowns.",
                    "dismissKey": "app.warnings.dismiss_high_context",
                }
            )

        # Get additional warning settings
        with get_user_db_session(username) as db_session:
            if db_session:
                settings_manager = get_settings_manager(db_session, username)
                dismiss_model_mismatch = settings_manager.get_setting(
                    "app.warnings.dismiss_model_mismatch", False
                )

                # Get current strategy and model (these need to be passed from the frontend or retrieved differently)
                # For now, we'll implement basic warnings that don't require form state

                # Model mismatch warning (simplified - checking setting instead of form value)
                current_model = settings_manager.get_setting("llm.model", "")
        if (
            current_model
            and "70b" in current_model.lower()
            and is_local_provider
            and local_context > 8192
            and not dismiss_model_mismatch
        ):
            warnings.append(
                {
                    "type": "model_mismatch",
                    "icon": "🧠",
                    "title": "Model & Context Warning",
                    "message": f"Large model ({current_model}) with high context ({local_context:,}) may exceed VRAM. Consider reducing context size or upgrading GPU memory.",
                    "dismissKey": "app.warnings.dismiss_model_mismatch",
                }
            )

    except Exception as e:
        logger.warning(f"Error calculating warnings: {e}")

    return warnings


def validate_setting(
    setting: Setting, value: Any
) -> Tuple[bool, Optional[str]]:
    """
    Validate a setting value based on its type and constraints.

    Args:
        setting: The Setting object to validate against
        value: The value to validate

    Returns:
        Tuple of (is_valid, error_message)
    """
    # Convert value to appropriate type first using SettingsManager's logic
    value = get_typed_setting_value(
        key=setting.key,
        value=value,
        ui_element=setting.ui_element,
        default=None,
        check_env=False,
    )

    # Validate based on UI element type
    if setting.ui_element == "checkbox":
        # After conversion, should be boolean
        if not isinstance(value, bool):
            return False, "Value must be a boolean"

    elif setting.ui_element in ("number", "slider", "range"):
        # After conversion, should be numeric
        if not isinstance(value, (int, float)):
            return False, "Value must be a number"

        # Check min/max constraints if defined
        if setting.min_value is not None and value < setting.min_value:
            return False, f"Value must be at least {setting.min_value}"
        if setting.max_value is not None and value > setting.max_value:
            return False, f"Value must be at most {setting.max_value}"

    elif setting.ui_element == "select":
        # Check if value is in the allowed options
        if setting.options:
            # Skip options validation for dynamically populated dropdowns
            if setting.key not in DYNAMIC_SETTINGS:
                allowed_values = [opt.get("value") for opt in setting.options]
                if value not in allowed_values:
                    return (
                        False,
                        f"Value must be one of: {', '.join(str(v) for v in allowed_values)}",
                    )

    # All checks passed
    return True, None


@settings_bp.route("/", methods=["GET"])
@login_required
def settings_page():
    """Main settings dashboard with links to specialized config pages"""
    return render_template_with_defaults("settings_dashboard.html")


@settings_bp.route("/save_all_settings", methods=["POST"])
@login_required
def save_all_settings():
    """Handle saving all settings at once from the unified settings page"""
    username = session.get("username")

    with get_user_db_session(username) as db_session:
        # Get the settings manager but we don't need to assign it to a variable right now
        # get_db_settings_manager(db_session)

        try:
            # Process JSON data
            form_data = request.get_json()
            if not form_data:
                return (
                    jsonify(
                        {
                            "status": "error",
                            "message": "No settings data provided",
                        }
                    ),
                    400,
                )

            # Track validation errors
            validation_errors = []
            settings_by_type = {}

            # Track changes for logging
            updated_settings = []
            created_settings = []

            # Store original values for better messaging
            original_values = {}

            # Fetch all settings once to avoid N+1 query problem
            all_db_settings = {
                setting.key: setting
                for setting in db_session.query(Setting).all()
            }

            # Update each setting
            for key, value in form_data.items():
                # Skip corrupted keys or empty strings as keys
                if not key or not isinstance(key, str) or key.strip() == "":
                    continue

                # Get the setting metadata from pre-fetched dict
                current_setting = all_db_settings.get(key)

                # EARLY VALIDATION: Convert checkbox values BEFORE any other processing
                # This prevents incorrect triggering of corrupted value detection
                if current_setting and current_setting.ui_element == "checkbox":
                    if not isinstance(value, bool):
                        logger.debug(
                            f"Converting checkbox {key} from {type(value).__name__} to bool: {value}"
                        )
                        value = parse_boolean(value)
                        form_data[key] = (
                            value  # Update the form_data with converted value
                        )

                # Store original value for messaging
                if current_setting:
                    original_values[key] = current_setting.value

                # Determine setting type and category
                if key.startswith("llm."):
                    setting_type = SettingType.LLM
                    category = "llm_general"
                    if (
                        "temperature" in key
                        or "max_tokens" in key
                        or "batch" in key
                        or "layers" in key
                    ):
                        category = "llm_parameters"
                elif key.startswith("search."):
                    setting_type = SettingType.SEARCH
                    category = "search_general"
                    if (
                        "iterations" in key
                        or "results" in key
                        or "region" in key
                        or "questions" in key
                        or "section" in key
                    ):
                        category = "search_parameters"
                elif key.startswith("report."):
                    setting_type = SettingType.REPORT
                    category = "report_parameters"
                elif key.startswith("database."):
                    setting_type = SettingType.DATABASE
                    category = "database_parameters"
                elif key.startswith("app."):
                    setting_type = SettingType.APP
                    category = "app_interface"
                else:
                    setting_type = None
                    category = None

                # Special handling for corrupted or empty values
                if value == "[object Object]" or (
                    isinstance(value, str)
                    and value.strip() in ["{}", "[]", "{", "["]
                ):
                    if key.startswith("report."):
                        value = {}
                    else:
                        # Use default or null for other types
                        if key == "llm.model":
                            value = "gpt-3.5-turbo"
                        elif key == "llm.provider":
                            value = "openai"
                        elif key == "search.tool":
                            value = "auto"
                        elif key in ["app.theme", "app.default_theme"]:
                            value = "dark"
                        else:
                            value = None

                    logger.warning(
                        f"Corrected corrupted value for {key}: {value}"
                    )

                    # Handle JSON string values (already parsed by JavaScript)
                    if isinstance(value, (dict, list)):
                        # Keep as is, already parsed
                        pass
                    # Handle string values that might be JSON
                    elif isinstance(value, str) and (
                        value.startswith("{") or value.startswith("[")
                    ):
                        try:
                            # Try to parse the string as JSON
                            value = json.loads(value)
                        except json.JSONDecodeError:
                            # If it fails to parse, keep as string
                            pass

                if current_setting:
                    # Convert value to appropriate type using SettingsManager's logic
                    converted_value = get_typed_setting_value(
                        key=current_setting.key,
                        value=value,
                        ui_element=current_setting.ui_element,
                        default=None,
                        check_env=False,
                    )

                    # Validate the setting
                    is_valid, error_message = validate_setting(
                        current_setting, converted_value
                    )

                    if is_valid:
                        # Save the converted setting using the same session
                        success = set_setting(
                            key, converted_value, db_session=db_session
                        )
                        if success:
                            updated_settings.append(key)

                        # Track settings by type for exporting
                        if current_setting.type not in settings_by_type:
                            settings_by_type[current_setting.type] = []
                        settings_by_type[current_setting.type].append(
                            current_setting
                        )
                    else:
                        # Add to validation errors
                        validation_errors.append(
                            {
                                "key": key,
                                "name": current_setting.name,
                                "error": error_message,
                            }
                        )
                else:
                    # Create a new setting
                    new_setting = {
                        "key": key,
                        "value": value,
                        "type": setting_type.value.lower(),
                        "name": key.split(".")[-1].replace("_", " ").title(),
                        "description": f"Setting for {key}",
                        "category": category,
                        "ui_element": "text",  # Default UI element
                    }

                    # Determine better UI element based on value type
                    if isinstance(value, bool):
                        new_setting["ui_element"] = "checkbox"
                    elif isinstance(value, (int, float)) and not isinstance(
                        value, bool
                    ):
                        new_setting["ui_element"] = "number"
                    elif isinstance(value, (dict, list)):
                        new_setting["ui_element"] = "textarea"

                    # Create the setting
                    db_setting = create_or_update_setting(
                        new_setting, db_session=db_session
                    )

                    if db_setting:
                        created_settings.append(key)
                        # Track settings by type for exporting
                        if db_setting.type not in settings_by_type:
                            settings_by_type[db_setting.type] = []
                        settings_by_type[db_setting.type].append(db_setting)
                    else:
                        validation_errors.append(
                            {
                                "key": key,
                                "name": new_setting["name"],
                                "error": "Failed to create setting",
                            }
                        )

            # Report validation errors if any
            if validation_errors:
                return (
                    jsonify(
                        {
                            "status": "error",
                            "message": "Validation errors",
                            "errors": validation_errors,
                        }
                    ),
                    400,
                )

            # Get all settings to return to the client for proper state update
            all_settings = []
            for setting in db_session.query(Setting).all():
                # Convert enum to string if present
                setting_type = setting.type
                if hasattr(setting_type, "value"):
                    setting_type = setting_type.value

                all_settings.append(
                    {
                        "key": setting.key,
                        "value": setting.value,
                        "name": setting.name,
                        "description": setting.description,
                        "type": setting_type,
                        "category": setting.category,
                        "ui_element": setting.ui_element,
                        "editable": setting.editable,
                        "options": setting.options,
                    }
                )

            # Customize the success message based on what changed
            success_message = ""
            if len(updated_settings) == 1:
                # For a single update, provide more specific info about what changed
                key = updated_settings[0]
                # Reuse the already-fetched setting from our pre-fetched dict
                updated_setting = all_db_settings.get(key)
                name = (
                    updated_setting.name
                    if updated_setting
                    else key.split(".")[-1].replace("_", " ").title()
                )

                # Format the message
                if key in original_values:
                    # Get original value but comment out if not used
                    # old_value = original_values[key]
                    new_value = (
                        updated_setting.value if updated_setting else None
                    )

                    # If it's a boolean, use "enabled/disabled" language
                    if isinstance(new_value, bool):
                        state = "enabled" if new_value else "disabled"
                        success_message = f"{name} {state}"
                    else:
                        # For non-boolean values
                        if isinstance(new_value, (dict, list)):
                            success_message = f"{name} updated"
                        else:
                            success_message = f"{name} updated"
                else:
                    success_message = f"{name} updated"
            else:
                # Multiple settings or generic message
                success_message = f"Settings saved successfully ({len(updated_settings)} updated, {len(created_settings)} created)"

            # Check if any warning-affecting settings were changed and include warnings
            response_data = {
                "status": "success",
                "message": success_message,
                "updated": updated_settings,
                "created": created_settings,
                "settings": all_settings,
            }

            warning_affecting_keys = [
                "llm.provider",
                "search.tool",
                "search.iterations",
                "search.questions_per_iteration",
                "llm.local_context_window_size",
                "llm.context_window_unrestricted",
                "llm.context_window_size",
            ]

            # Check if any warning-affecting settings were changed
            if any(
                key in warning_affecting_keys
                for key in updated_settings + created_settings
            ):
                warnings = calculate_warnings()
                response_data["warnings"] = warnings
                logger.info(
                    f"Bulk settings update affected warning keys, calculated {len(warnings)} warnings"
                )

            return jsonify(response_data)

        except Exception:
            logger.exception("Error saving settings")
            return (
                jsonify(
                    {
                        "status": "error",
                        "message": "An internal error occurred while saving settings.",
                    }
                ),
                500,
            )


@settings_bp.route("/reset_to_defaults", methods=["POST"])
@login_required
def reset_to_defaults():
    """Reset all settings to their default values"""
    username = session.get("username")

    with get_user_db_session(username) as db_session:
        # Import default settings from files
        try:
            # Create settings manager with proper session context
            username = session.get("username")
            with get_user_db_session(username) as db_session:
                settings_mgr = SettingsManager(db_session)
                # Import settings from default files
                settings_mgr.load_from_defaults_file()

            logger.info("Successfully imported settings from default files")

        except Exception:
            logger.exception("Error importing default settings")

        # Return success
        return jsonify(
            {
                "status": "success",
                "message": "All settings have been reset to default values",
            }
        )


@settings_bp.route("/save_settings", methods=["POST"])
@login_required
def save_settings():
    """Save all settings from the form using POST method - fallback when JavaScript is disabled"""
    try:
        username = session.get("username")

        # Get form data
        form_data = request.form.to_dict()

        # Remove CSRF token from the data
        form_data.pop("csrf_token", None)

        with get_user_db_session(username) as db_session:
            settings_manager = SettingsManager(db_session)

            updated_count = 0
            failed_count = 0

            # Fetch all settings once to avoid N+1 query problem
            all_db_settings = {
                setting.key: setting
                for setting in db_session.query(Setting).all()
            }

            # Process each setting
            for key, value in form_data.items():
                try:
                    # Get the setting from pre-fetched dict
                    db_setting = all_db_settings.get(key)

                    # Convert value to appropriate type using SettingsManager's logic
                    if db_setting:
                        value = get_typed_setting_value(
                            key=db_setting.key,
                            value=value,
                            ui_element=db_setting.ui_element,
                            default=None,
                            check_env=False,
                        )

                    # Save the setting
                    if settings_manager.set_setting(key, value, commit=False):
                        updated_count += 1
                    else:
                        failed_count += 1
                        logger.warning(f"Failed to save setting {key}")

                except Exception:
                    logger.exception(f"Error saving setting {key}")
                    failed_count += 1

            # Commit all changes at once
            try:
                db_session.commit()

                flash(
                    f"Settings saved successfully! Updated {updated_count} settings.",
                    "success",
                )
                if failed_count > 0:
                    flash(
                        f"Warning: {failed_count} settings failed to save.",
                        "warning",
                    )

                # Sync server config
                settings_snapshot = settings_manager.get_settings_snapshot()
                sync_from_settings(settings_snapshot)

            except Exception:
                db_session.rollback()
                logger.exception("Failed to commit settings")
                flash("Error saving settings. Please try again.", "error")

        return redirect(url_for("settings.settings_page"))

    except Exception:
        logger.exception("Error in save_settings")
        flash("An internal error occurred while saving settings.", "error")
        return redirect(url_for("settings.settings_page"))


# API Routes
@settings_bp.route("/api", methods=["GET"])
@login_required
def api_get_all_settings():
    """Get all settings"""
    try:
        # Get query parameters
        category = request.args.get("category")
        username = session.get("username")

        with get_user_db_session(username) as db_session:
            # Create settings manager with the session from context
            # This ensures thread safety
            settings_manager = SettingsManager(db_session)

            # Get settings
            settings = settings_manager.get_all_settings()

            # Filter by category if requested
            if category:
                filtered_settings = {}
                # Need to get all setting details to check category
                db_settings = db_session.query(Setting).all()
                category_keys = [
                    s.key for s in db_settings if s.category == category
                ]

                # Filter settings by keys
                for key, value in settings.items():
                    if key in category_keys:
                        filtered_settings[key] = value

                settings = filtered_settings

            return jsonify({"status": "success", "settings": settings})
    except Exception:
        logger.exception("Error getting settings")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/<path:key>", methods=["GET"])
@login_required
def api_get_db_setting(key):
    """Get a specific setting by key"""
    try:
        username = session.get("username")

        with get_user_db_session(username) as db_session:
            # Get setting from database using the same session
            db_setting = (
                db_session.query(Setting).filter(Setting.key == key).first()
            )

            if db_setting:
                # Return full setting details
                setting_data = {
                    "key": db_setting.key,
                    "value": db_setting.value,
                    "type": db_setting.type
                    if isinstance(db_setting.type, str)
                    else db_setting.type.value,
                    "name": db_setting.name,
                    "description": db_setting.description,
                    "category": db_setting.category,
                    "ui_element": db_setting.ui_element,
                    "options": db_setting.options,
                    "min_value": db_setting.min_value,
                    "max_value": db_setting.max_value,
                    "step": db_setting.step,
                    "visible": db_setting.visible,
                    "editable": db_setting.editable,
                }
                return jsonify(setting_data)
            else:
                # Setting not found
                return jsonify({"error": f"Setting not found: {key}"}), 404
    except Exception:
        logger.exception(f"Error getting setting {key}")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/<path:key>", methods=["PUT"])
@login_required
def api_update_setting(key):
    """Update a setting"""
    try:
        # Get request data
        data = request.get_json()
        if not data:
            return jsonify({"error": "No data provided"}), 400

        value = data.get("value")
        if value is None:
            return jsonify({"error": "No value provided"}), 400

        username = session.get("username")

        with get_user_db_session(username) as db_session:
            # Only use settings_manager if needed - we don't need to assign if not used
            # get_db_settings_manager(db_session)

            # Check if setting exists
            db_setting = (
                db_session.query(Setting).filter(Setting.key == key).first()
            )

            if db_setting:
                # Check if setting is editable
                if not db_setting.editable:
                    return jsonify(
                        {"error": f"Setting {key} is not editable"}
                    ), 403

                # Update setting
                # Pass the db_session to avoid session lookup issues
                success = set_setting(key, value, db_session=db_session)
                if success:
                    # Sync server config
                    settings_manager = SettingsManager(db_session)
                    settings_snapshot = settings_manager.get_settings_snapshot()
                    sync_from_settings(settings_snapshot)

                    response_data = {
                        "message": f"Setting {key} updated successfully"
                    }

                    # If this is a key that affects warnings, include warning calculations
                    warning_affecting_keys = [
                        "llm.provider",
                        "search.tool",
                        "search.iterations",
                        "search.questions_per_iteration",
                        "llm.local_context_window_size",
                        "llm.context_window_unrestricted",
                        "llm.context_window_size",
                    ]

                    if key in warning_affecting_keys:
                        warnings = calculate_warnings()
                        response_data["warnings"] = warnings
                        logger.debug(
                            f"Setting {key} changed to {value}, calculated {len(warnings)} warnings"
                        )

                    return jsonify(response_data)
                else:
                    return jsonify(
                        {"error": f"Failed to update setting {key}"}
                    ), 500
            else:
                # Create new setting with default metadata
                setting_dict = {
                    "key": key,
                    "value": value,
                    "name": key.split(".")[-1].replace("_", " ").title(),
                    "description": f"Setting for {key}",
                }

                # Add additional metadata if provided
                for field in [
                    "type",
                    "name",
                    "description",
                    "category",
                    "ui_element",
                    "options",
                    "min_value",
                    "max_value",
                    "step",
                    "visible",
                    "editable",
                ]:
                    if field in data:
                        setting_dict[field] = data[field]

                # Create setting
                db_setting = create_or_update_setting(
                    setting_dict, db_session=db_session
                )

                if db_setting:
                    # Sync server config
                    settings_manager = SettingsManager(db_session)
                    settings_snapshot = settings_manager.get_settings_snapshot()
                    sync_from_settings(settings_snapshot)

                    return (
                        jsonify(
                            {
                                "message": f"Setting {key} created successfully",
                                "setting": {
                                    "key": db_setting.key,
                                    "value": db_setting.value,
                                    "type": db_setting.type.value,
                                    "name": db_setting.name,
                                },
                            }
                        ),
                        201,
                    )
                else:
                    return jsonify(
                        {"error": f"Failed to create setting {key}"}
                    ), 500
    except Exception:
        logger.exception(f"Error updating setting {key}")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/<path:key>", methods=["DELETE"])
@login_required
def api_delete_setting(key):
    """Delete a setting"""
    try:
        username = session.get("username")

        with get_user_db_session(username) as db_session:
            # Create settings manager with the session from context
            settings_manager = SettingsManager(db_session)

            # Check if setting exists
            db_setting = (
                db_session.query(Setting).filter(Setting.key == key).first()
            )
            if not db_setting:
                return jsonify({"error": f"Setting not found: {key}"}), 404

            # Delete setting
            success = settings_manager.delete_setting(key)
            if success:
                return jsonify(
                    {"message": f"Setting {key} deleted successfully"}
                )
            else:
                return jsonify(
                    {"error": f"Failed to delete setting {key}"}
                ), 500
    except Exception:
        logger.exception(f"Error deleting setting {key}")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/import", methods=["POST"])
@login_required
def api_import_settings():
    """Import settings from defaults file"""
    try:
        username = session.get("username")
        with get_user_db_session(username) as db_session:
            settings_manager = SettingsManager(db_session)
            success = settings_manager.load_from_defaults_file()

        if success:
            return jsonify({"message": "Settings imported successfully"})
        else:
            return jsonify({"error": "Failed to import settings"}), 500
    except Exception:
        logger.exception("Error importing settings")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/categories", methods=["GET"])
@login_required
def api_get_categories():
    """Get all setting categories"""
    try:
        username = session.get("username")

        with get_user_db_session(username) as db_session:
            # Get all distinct categories
            categories = db_session.query(Setting.category).distinct().all()
            category_list = [c[0] for c in categories if c[0] is not None]

            return jsonify({"categories": category_list})
    except Exception:
        logger.exception("Error getting categories")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/types", methods=["GET"])
@login_required
def api_get_types():
    """Get all setting types"""
    try:
        # Get all setting types
        types = [t.value for t in SettingType]
        return jsonify({"types": types})
    except Exception:
        logger.exception("Error getting types")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/ui_elements", methods=["GET"])
@login_required
def api_get_ui_elements():
    """Get all UI element types"""
    try:
        # Define supported UI element types
        ui_elements = [
            "text",
            "select",
            "checkbox",
            "slider",
            "number",
            "textarea",
            "color",
            "date",
            "file",
            "password",
        ]

        return jsonify({"ui_elements": ui_elements})
    except Exception:
        logger.exception("Error getting UI elements")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/available-models", methods=["GET"])
@login_required
def api_get_available_models():
    """Get available LLM models from various providers"""
    try:
        from flask import request

        from ...database.models import ProviderModel

        # Check if force_refresh is requested
        force_refresh = (
            request.args.get("force_refresh", "false").lower() == "true"
        )

        # Get all auto-discovered providers
        from ...llm.providers import get_discovered_provider_options

        provider_options = get_discovered_provider_options()

        # Add xAI provider
        provider_options.append({"value": "XAI", "label": "xAI Grok (Cloud)"})

        # Add remaining hardcoded providers (complex local providers not yet migrated)
        provider_options.extend(
            [
                {"value": "VLLM", "label": "vLLM (Local)"},
                {"value": "LLAMACPP", "label": "Llama.cpp (Local)"},
            ]
        )

        # Available models by provider
        providers = {}

        # Check database cache first (unless force_refresh is True)
        if not force_refresh:
            try:
                # Define cache expiration (24 hours)
                cache_expiry = datetime.now(UTC) - timedelta(hours=24)

                # Get cached models from database
                username = session.get("username")
                with get_user_db_session(username) as db_session:
                    cached_models = (
                        db_session.query(ProviderModel)
                        .filter(ProviderModel.last_updated > cache_expiry)
                        .all()
                    )

                if cached_models:
                    logger.info(
                        f"Found {len(cached_models)} cached models in database"
                    )

                    # Group models by provider
                    for model in cached_models:
                        provider_key = f"{model.provider.lower()}_models"
                        if provider_key not in providers:
                            providers[provider_key] = []

                        providers[provider_key].append(
                            {
                                "value": model.model_key,
                                "label": model.model_label,
                                "provider": model.provider.upper(),
                            }
                        )

                    # If we have cached data for all providers, return it
                    if providers:
                        logger.info("Returning cached models from database")
                        return jsonify(
                            {
                                "provider_options": provider_options,
                                "providers": providers,
                            }
                        )

            except Exception as e:
                logger.warning(
                    f"Error reading cached models from database: {e}"
                )
                # Continue to fetch fresh data

        # Try to get Ollama models
        ollama_models = []
        try:
            import json
            import re

            import requests

            # Try to query the Ollama API directly
            try:
                logger.info("Attempting to connect to Ollama API")

                raw_base_url = _get_setting_from_session(
                    "llm.ollama.url", "http://localhost:11434"
                )
                base_url = (
                    normalize_url(raw_base_url)
                    if raw_base_url
                    else "http://localhost:11434"
                )

                ollama_response = requests.get(
                    f"{base_url}/api/tags", timeout=5
                )

                logger.debug(
                    f"Ollama API response: Status {ollama_response.status_code}"
                )

                # Try to parse the response even if status code is not 200 to help with debugging
                response_text = ollama_response.text
                logger.debug(
                    f"Ollama API raw response: {response_text[:500]}..."
                )

                if ollama_response.status_code == 200:
                    try:
                        ollama_data = ollama_response.json()
                        logger.debug(
                            f"Ollama API JSON data: {json.dumps(ollama_data)[:500]}..."
                        )

                        if "models" in ollama_data:
                            # Format for newer Ollama API
                            logger.info(
                                f"Found {len(ollama_data.get('models', []))} models in newer Ollama API format"
                            )
                            for model in ollama_data.get("models", []):
                                # Extract name correctly from the model object
                                name = model.get("name", "")
                                if name:
                                    # Improved display name formatting
                                    display_name = re.sub(
                                        r"[:/]", " ", name
                                    ).strip()
                                    display_name = " ".join(
                                        word.capitalize()
                                        for word in display_name.split()
                                    )
                                    # Create the model entry with value and label
                                    ollama_models.append(
                                        {
                                            "value": name,  # Original model name as value (for API calls)
                                            "label": f"{display_name} (Ollama)",  # Pretty name as label
                                            "provider": "OLLAMA",  # Add provider field for consistency
                                        }
                                    )
                                    logger.debug(
                                        f"Added Ollama model: {name} -> {display_name}"
                                    )
                        else:
                            # Format for older Ollama API
                            logger.info(
                                f"Found {len(ollama_data)} models in older Ollama API format"
                            )
                            for model in ollama_data:
                                name = model.get("name", "")
                                if name:
                                    # Improved display name formatting
                                    display_name = re.sub(
                                        r"[:/]", " ", name
                                    ).strip()
                                    display_name = " ".join(
                                        word.capitalize()
                                        for word in display_name.split()
                                    )
                                    ollama_models.append(
                                        {
                                            "value": name,
                                            "label": f"{display_name} (Ollama)",
                                            "provider": "OLLAMA",  # Add provider field for consistency
                                        }
                                    )
                                    logger.debug(
                                        f"Added Ollama model: {name} -> {display_name}"
                                    )

                    except json.JSONDecodeError as json_err:
                        logger.exception(
                            f"Failed to parse Ollama API response as JSON: {json_err}"
                        )
                        raise Exception(
                            f"Ollama API returned invalid JSON: {json_err}"
                        )
                else:
                    logger.warning(
                        f"Ollama API returned non-200 status code: {ollama_response.status_code}"
                    )
                    raise Exception(
                        f"Ollama API returned status code {ollama_response.status_code}"
                    )

            except requests.exceptions.RequestException as e:
                logger.warning(f"Could not connect to Ollama API: {e!s}")
                # No fallback models - just return empty list
                logger.info("Ollama not available - no models to display")
                ollama_models = []

            # Always set the ollama_models in providers, whether we got real or fallback models
            providers["ollama_models"] = ollama_models
            logger.info(f"Final Ollama models count: {len(ollama_models)}")

            # Log some model names for debugging
            if ollama_models:
                model_names = [m["value"] for m in ollama_models[:5]]
                logger.info(f"Sample Ollama models: {', '.join(model_names)}")

        except Exception:
            logger.exception("Error getting Ollama models")
            # No fallback models - just return empty list
            logger.info("Error getting Ollama models - no models to display")
            providers["ollama_models"] = []

        # Try to get Custom OpenAI Endpoint models using the provider class
        openai_endpoint_models = []
        try:
            logger.info("Fetching Custom OpenAI Endpoint models")

            # Get the endpoint URL and API key from settings
            endpoint_url = _get_setting_from_session(
                "llm.openai_endpoint.url", ""
            )
            api_key = _get_setting_from_session(
                "llm.openai_endpoint.api_key", ""
            )

            if endpoint_url and api_key:
                # Need to temporarily set the URL for the provider class
                original_url = CustomOpenAIEndpointProvider.default_base_url
                CustomOpenAIEndpointProvider.default_base_url = endpoint_url

                try:
                    # Use the provider's list_models_for_api method
                    models = CustomOpenAIEndpointProvider.list_models_for_api(
                        api_key
                    )

                    # Format models for the API response
                    for model in models:
                        # Create a clean display name
                        display_name = model["label"].replace("-", " ").strip()
                        display_name = " ".join(
                            word.capitalize() for word in display_name.split()
                        )

                        openai_endpoint_models.append(
                            {
                                "value": model["value"],
                                "label": f"{display_name} (Custom)",
                                "provider": "OPENAI_ENDPOINT",
                            }
                        )

                    logger.info(
                        f"Successfully fetched {len(openai_endpoint_models)} models from Custom OpenAI Endpoint"
                    )

                finally:
                    # Restore original URL
                    CustomOpenAIEndpointProvider.default_base_url = original_url

            else:
                logger.info(
                    "Custom OpenAI Endpoint URL or API key not configured"
                )

        except Exception as e:
            logger.exception(
                f"Error getting Custom OpenAI Endpoint models: {e!s}"
            )

        # Always set the openai_endpoint_models in providers
        providers["openai_endpoint_models"] = openai_endpoint_models
        logger.info(
            f"Final OpenAI Endpoint models count: {len(openai_endpoint_models)}"
        )

        # Get OpenAI models using the OpenAI package
        openai_models = []
        try:
            logger.info(
                "Attempting to connect to OpenAI API using OpenAI package"
            )

            # Get the API key from settings
            api_key = _get_setting_from_session("llm.openai.api_key", "")

            if api_key:
                import openai
                from openai import OpenAI

                # Create OpenAI client
                client = OpenAI(api_key=api_key)

                try:
                    # Fetch models using the client
                    logger.debug("Fetching models from OpenAI API")
                    models_response = client.models.list()

                    # Process models from the response
                    for model in models_response.data:
                        model_id = model.id
                        if model_id:
                            # Create a clean display name
                            display_name = model_id.replace("-", " ").strip()
                            display_name = " ".join(
                                word.capitalize()
                                for word in display_name.split()
                            )

                            openai_models.append(
                                {
                                    "value": model_id,
                                    "label": f"{display_name} (OpenAI)",
                                    "provider": "OPENAI",
                                }
                            )
                            logger.debug(
                                f"Added OpenAI model: {model_id} -> {display_name}"
                            )

                    # Keep original order from OpenAI - their models are returned in a
                    # meaningful order (newer/more capable models first)

                except openai.APIError as api_err:
                    logger.exception(f"OpenAI API error: {api_err!s}")
                    logger.info("No OpenAI models found due to API error")

            else:
                logger.info(
                    "OpenAI API key not configured, no models available"
                )

        except Exception as e:
            logger.exception(f"Error getting OpenAI models: {e!s}")
            logger.info("No OpenAI models available due to error")

        # Always set the openai_models in providers (will be empty array if no models found)
        providers["openai_models"] = openai_models
        logger.info(f"Final OpenAI models count: {len(openai_models)}")

        # Try to get Anthropic models using the Anthropic package
        anthropic_models = []
        try:
            logger.info(
                "Attempting to connect to Anthropic API using Anthropic package"
            )

            # Get the API key from settings
            api_key = _get_setting_from_session("llm.anthropic.api_key", "")

            if api_key:
                # Import Anthropic package here to avoid dependency issues if not installed
                from anthropic import Anthropic

                # Create Anthropic client
                client = Anthropic(api_key=api_key)

                try:
                    # Fetch models using the client
                    logger.debug("Fetching models from Anthropic API")
                    models_response = client.models.list()

                    # Process models from the response
                    for model in models_response.data:
                        model_id = model.id
                        if model_id:
                            # Create a clean display name
                            display_name = model_id.replace("-", " ").strip()
                            display_name = " ".join(
                                word.capitalize()
                                for word in display_name.split()
                            )

                            anthropic_models.append(
                                {
                                    "value": model_id,
                                    "label": f"{display_name} (Anthropic)",
                                    "provider": "ANTHROPIC",
                                }
                            )
                            logger.debug(
                                f"Added Anthropic model: {model_id} -> {display_name}"
                            )

                except Exception as api_err:
                    logger.exception(f"Anthropic API error: {api_err!s}")
            else:
                logger.info("Anthropic API key not configured")

        except ImportError:
            logger.warning(
                "Anthropic package not installed. No models will be available."
            )
        except Exception as e:
            logger.exception(f"Error getting Anthropic models: {e!s}")

        # Set anthropic_models in providers (could be empty if API call failed)
        providers["anthropic_models"] = anthropic_models
        logger.info(f"Final Anthropic models count: {len(anthropic_models)}")

        # Fetch models from auto-discovered providers
        from ...llm.providers import discover_providers

        discovered_providers = discover_providers()

        for provider_key, provider_info in discovered_providers.items():
            provider_models = []
            try:
                logger.info(
                    f"Fetching models from {provider_info.provider_name}"
                )

                # Get the provider class
                provider_class = provider_info.provider_class

                # Get API key if configured
                api_key = _get_setting_from_session(
                    provider_class.api_key_setting, ""
                )

                # Use the provider's list_models_for_api method
                models = provider_class.list_models_for_api(api_key)

                # Format models for the API response
                for model in models:
                    provider_models.append(
                        {
                            "value": model["value"],
                            "label": f"{model['label']} ({provider_key})",
                            "provider": provider_key,
                        }
                    )

                logger.info(
                    f"Successfully fetched {len(provider_models)} models from {provider_info.provider_name}"
                )

            except Exception as e:
                logger.exception(
                    f"Error getting {provider_info.provider_name} models: {e!s}"
                )

            # Set models in providers dict using lowercase key
            providers[f"{provider_key.lower()}_models"] = provider_models
            logger.info(
                f"Final {provider_key} models count: {len(provider_models)}"
            )

        # Try to get xAI models using the provider class
        xai_models = []
        try:
            logger.info("Fetching xAI Grok models")

            from ...llm.providers.xai import XAIProvider

            # Get API key if configured (xAI requires it for listing models)
            api_key = _get_setting_from_session("llm.xai.api_key", "")

            # Use the provider's list_models_for_api method
            models = XAIProvider.list_models_for_api(api_key)

            # Format models for the API response
            for model in models:
                xai_models.append(
                    {
                        "value": model["value"],
                        "label": f"{model['label']} (xAI)",
                        "provider": "XAI",
                    }
                )

            logger.info(
                f"Successfully fetched {len(xai_models)} models from xAI"
            )

        except Exception as e:
            logger.exception(f"Error getting xAI models: {e!s}")

        # Set xai_models in providers
        providers["xai_models"] = xai_models
        logger.info(f"Final xAI models count: {len(xai_models)}")
        # Save fetched models to database cache
        if force_refresh or providers:
            # We fetched fresh data, save it to database
            username = session.get("username")
            with get_user_db_session(username) as db_session:
                try:
                    # Clear old cache entries for providers we're updating
                    for provider_key in providers:
                        provider_name = provider_key.replace(
                            "_models", ""
                        ).upper()
                        db_session.query(ProviderModel).filter(
                            ProviderModel.provider == provider_name
                        ).delete()

                    # Insert new models
                    for provider_key, models in providers.items():
                        provider_name = provider_key.replace(
                            "_models", ""
                        ).upper()
                        for model in models:
                            if (
                                isinstance(model, dict)
                                and "value" in model
                                and "label" in model
                            ):
                                new_model = ProviderModel(
                                    provider=provider_name,
                                    model_key=model["value"],
                                    model_label=model["label"],
                                    last_updated=datetime.now(UTC),
                                )
                                db_session.add(new_model)

                    db_session.commit()
                    logger.info("Successfully cached models to database")

                except Exception:
                    logger.exception("Error saving models to database cache")
                    db_session.rollback()

        # Return all options
        return jsonify(
            {"provider_options": provider_options, "providers": providers}
        )

    except Exception:
        logger.exception("Error getting available models")
        return jsonify(
            {"status": "error", "message": "Failed to save settings"}
        ), 500


def _get_engine_icon_and_category(
    engine_data: dict, engine_class=None
) -> tuple:
    """
    Get icon emoji and category label for a search engine based on its attributes.

    Args:
        engine_data: Engine configuration dictionary
        engine_class: Optional loaded engine class to check attributes

    Returns:
        Tuple of (icon, category) strings
    """
    # Check attributes from either the class or the engine data
    if engine_class:
        is_scientific = getattr(engine_class, "is_scientific", False)
        is_generic = getattr(engine_class, "is_generic", False)
        is_local = getattr(engine_class, "is_local", False)
        is_news = getattr(engine_class, "is_news", False)
        is_code = getattr(engine_class, "is_code", False)
    else:
        is_scientific = engine_data.get("is_scientific", False)
        is_generic = engine_data.get("is_generic", False)
        is_local = engine_data.get("is_local", False)
        is_news = engine_data.get("is_news", False)
        is_code = engine_data.get("is_code", False)

    # Return icon and category based on engine type
    # Priority: local > scientific > news > code > generic > default
    if is_local:
        return "📁", "Local RAG"
    elif is_scientific:
        return "🔬", "Scientific"
    elif is_news:
        return "📰", "News"
    elif is_code:
        return "💻", "Code"
    elif is_generic:
        return "🌐", "Web Search"
    else:
        return "🔍", "Search"


@settings_bp.route("/api/available-search-engines", methods=["GET"])
@login_required
def api_get_available_search_engines():
    """Get available search engines"""
    try:
        # Get search engines using the same approach as search_engines_config.py
        from ...web_search_engines.search_engines_config import search_config
        from ...database.session_context import get_user_db_session

        username = session.get("username")
        with get_user_db_session(username) as db_session:
            search_engines = search_config(
                username=username, db_session=db_session
            )

            # Extract search engines from config
            engines_dict = {}
            engine_options = []

            if search_engines:
                # Format engines for API response with metadata
                from importlib import import_module

                for engine_id, engine_data in search_engines.items():
                    # Try to load the engine class to get metadata
                    engine_class = None
                    try:
                        module_path = engine_data.get("module_path")
                        class_name = engine_data.get("class_name")
                        if module_path and class_name:
                            # Handle relative imports
                            package = None
                            if module_path.startswith("."):
                                package = (
                                    "local_deep_research.web_search_engines"
                                )
                            module = import_module(module_path, package=package)
                            engine_class = getattr(module, class_name, None)
                    except Exception as e:
                        logger.debug(
                            f"Could not load engine class for {engine_id}: {e}"
                        )

                    # Get icon and category from engine attributes
                    icon, category = _get_engine_icon_and_category(
                        engine_data, engine_class
                    )

                    # Build display name with icon and category
                    base_name = engine_data.get("display_name", engine_id)
                    label = f"{icon} {base_name} ({category})"

                    engines_dict[engine_id] = {
                        "display_name": base_name,
                        "description": engine_data.get("description", ""),
                        "strengths": engine_data.get("strengths", []),
                        "icon": icon,
                        "category": category,
                    }

                    engine_options.append(
                        {
                            "value": engine_id,
                            "label": label,
                            "icon": icon,
                            "category": category,
                        }
                    )

            # If no engines found, log the issue but return empty list
            if not engine_options:
                logger.warning("No search engines found in configuration")

            return jsonify(
                {"engines": engines_dict, "engine_options": engine_options}
            )

    except Exception:
        logger.exception("Error getting available search engines")
        return jsonify({"error": "Failed to retrieve settings"}), 500


# Legacy routes for backward compatibility - these will redirect to the new routes
@settings_bp.route("/main", methods=["GET"])
@login_required
def main_config_page():
    """Redirect to app settings page"""
    return redirect(url_for("settings.settings_page"))


@settings_bp.route("/collections", methods=["GET"])
@login_required
def collections_config_page():
    """Redirect to app settings page"""
    return redirect(url_for("settings.settings_page"))


@settings_bp.route("/api_keys", methods=["GET"])
@login_required
def api_keys_config_page():
    """Redirect to LLM settings page"""
    return redirect(url_for("settings.settings_page"))


@settings_bp.route("/search_engines", methods=["GET"])
@login_required
def search_engines_config_page():
    """Redirect to search settings page"""
    return redirect(url_for("settings.settings_page"))


@settings_bp.route("/open_file_location", methods=["POST"])
@login_required
def open_file_location():
    """Open the location of a configuration file"""
    file_path = request.form.get("file_path")

    if not file_path:
        flash("No file path provided", "error")
        return redirect(url_for("settings.settings_page"))

    try:
        # Use centralized path validator for security
        from ...security.path_validator import PathValidator
        from ...config.paths import get_data_directory

        try:
            # PathValidator.validate_config_path already checks existence
            resolved_path = PathValidator.validate_config_path(
                file_path, get_data_directory()
            )
        except ValueError as e:
            # The validator will raise ValueError if file doesn't exist
            flash(f"Invalid file path: {str(e)}", "error")
            return redirect(url_for("settings.settings_page"))

        # Get the directory containing the file
        dir_path = resolved_path.parent
        file_path = resolved_path  # Use resolved path going forward

        # Open the directory in the file explorer
        if platform.system() == "Windows":
            subprocess.Popen(["explorer", str(dir_path)])
        elif platform.system() == "Darwin":  # macOS
            subprocess.Popen(["open", str(dir_path)])
        else:  # Linux
            subprocess.Popen(["xdg-open", str(dir_path)])

        flash(f"Opening folder: {dir_path}", "success")
    except Exception as e:
        logger.exception("Error opening folder")
        flash(f"Error opening folder: {e!s}", "error")

    # Redirect back to the settings page
    return redirect(url_for("settings.settings_page"))


@settings_bp.context_processor
def inject_csrf_token():
    """Inject CSRF token into the template context for all settings routes."""
    return dict(csrf_token=generate_csrf)


@settings_bp.route("/fix_corrupted_settings", methods=["POST"])
@login_required
def fix_corrupted_settings():
    """Fix corrupted settings in the database"""
    username = session.get("username")

    with get_user_db_session(username) as db_session:
        try:
            # Track fixed and removed settings
            fixed_settings = []
            removed_duplicate_settings = []
            fixed_scoping_issues = []

            # First, find and remove duplicate settings with the same key
            # This happens because of errors in settings import/export
            from sqlalchemy import func as sql_func

            # Find keys with duplicates
            duplicate_keys = (
                db_session.query(Setting.key)
                .group_by(Setting.key)
                .having(sql_func.count(Setting.key) > 1)
                .all()
            )
            duplicate_keys = [key[0] for key in duplicate_keys]

            # For each duplicate key, keep the latest updated one and remove others
            for key in duplicate_keys:
                dupe_settings = (
                    db_session.query(Setting)
                    .filter(Setting.key == key)
                    .order_by(Setting.updated_at.desc())
                    .all()
                )

                # Keep the first one (most recently updated) and delete the rest
                for i, setting in enumerate(dupe_settings):
                    if i > 0:  # Skip the first one (keep it)
                        db_session.delete(setting)
                        removed_duplicate_settings.append(key)

            # Fix scoping issues - remove app.* settings that should be in other categories
            # Report settings
            for key in [
                "app.enable_fact_checking",
                "app.knowledge_accumulation",
                "app.knowledge_accumulation_context_limit",
                "app.output_dir",
            ]:
                setting = (
                    db_session.query(Setting).filter(Setting.key == key).first()
                )
                if setting:
                    # Move to proper category if not already there
                    proper_key = key.replace("app.", "report.")
                    existing_proper = (
                        db_session.query(Setting)
                        .filter(Setting.key == proper_key)
                        .first()
                    )

                    if not existing_proper:
                        # Create proper setting
                        new_setting = Setting(
                            key=proper_key,
                            value=setting.value,
                            type=SettingType.REPORT,
                            name=setting.name,
                            description=setting.description,
                            category=(
                                setting.category.replace("app", "report")
                                if setting.category
                                else "report_parameters"
                            ),
                            ui_element=setting.ui_element,
                            options=setting.options,
                            min_value=setting.min_value,
                            max_value=setting.max_value,
                            step=setting.step,
                            visible=setting.visible,
                            editable=setting.editable,
                        )
                        db_session.add(new_setting)

                    # Delete the app one
                    db_session.delete(setting)
                    fixed_scoping_issues.append(key)

            # Search settings
            for key in [
                "app.questions_per_iteration",
                "app.search_engine",
                "app.iterations",
                "app.max_results",
                "app.region",
                "app.safe_search",
                "app.search_language",
                "app.snippets_only",
            ]:
                setting = (
                    db_session.query(Setting).filter(Setting.key == key).first()
                )
                if setting:
                    # Move to proper category if not already there
                    proper_key = key.replace("app.", "search.")
                    existing_proper = (
                        db_session.query(Setting)
                        .filter(Setting.key == proper_key)
                        .first()
                    )

                    if not existing_proper:
                        # Create proper setting
                        new_setting = Setting(
                            key=proper_key,
                            value=setting.value,
                            type=SettingType.SEARCH,
                            name=setting.name,
                            description=setting.description,
                            category=(
                                setting.category.replace("app", "search")
                                if setting.category
                                else "search_parameters"
                            ),
                            ui_element=setting.ui_element,
                            options=setting.options,
                            min_value=setting.min_value,
                            max_value=setting.max_value,
                            step=setting.step,
                            visible=setting.visible,
                            editable=setting.editable,
                        )
                        db_session.add(new_setting)

                    # Delete the app one
                    db_session.delete(setting)
                    fixed_scoping_issues.append(key)

            # LLM settings
            for key in [
                "app.model",
                "app.provider",
                "app.temperature",
                "app.max_tokens",
                "app.openai_endpoint_url",
                "app.lmstudio_url",
                "app.llamacpp_model_path",
            ]:
                setting = (
                    db_session.query(Setting).filter(Setting.key == key).first()
                )
                if setting:
                    # Move to proper category if not already there
                    proper_key = key.replace("app.", "llm.")
                    existing_proper = (
                        db_session.query(Setting)
                        .filter(Setting.key == proper_key)
                        .first()
                    )

                    if not existing_proper:
                        # Create proper setting
                        new_setting = Setting(
                            key=proper_key,
                            value=setting.value,
                            type=SettingType.LLM,
                            name=setting.name,
                            description=setting.description,
                            category=(
                                setting.category.replace("app", "llm")
                                if setting.category
                                else "llm_parameters"
                            ),
                            ui_element=setting.ui_element,
                            options=setting.options,
                            min_value=setting.min_value,
                            max_value=setting.max_value,
                            step=setting.step,
                            visible=setting.visible,
                            editable=setting.editable,
                        )
                        db_session.add(new_setting)

                    # Delete the app one
                    db_session.delete(setting)
                    fixed_scoping_issues.append(key)

            # Check for settings with corrupted values
            all_settings = db_session.query(Setting).all()
            for setting in all_settings:
                # Check different types of corruption
                is_corrupted = False

                if (
                    setting.value is None
                    or (
                        isinstance(setting.value, str)
                        and setting.value
                        in [
                            "{",
                            "[",
                            "{}",
                            "[]",
                            "[object Object]",
                            "null",
                            "undefined",
                        ]
                    )
                    or (
                        isinstance(setting.value, dict)
                        and len(setting.value) == 0
                    )
                ):
                    is_corrupted = True

                # Skip if not corrupted
                if not is_corrupted:
                    continue

                # Get default value from migrations
                # Import commented out as it's not directly used
                # from ..database.migrations import setup_predefined_settings

                default_value = None

                # Try to find a matching default setting based on key
                if setting.key.startswith("llm."):
                    if setting.key == "llm.model":
                        default_value = "gpt-3.5-turbo"
                    elif setting.key == "llm.provider":
                        default_value = "openai"
                    elif setting.key == "llm.temperature":
                        default_value = 0.7
                    elif setting.key == "llm.max_tokens":
                        default_value = 1024
                elif setting.key.startswith("search."):
                    if setting.key == "search.tool":
                        default_value = "auto"
                    elif setting.key == "search.max_results":
                        default_value = 10
                    elif setting.key == "search.region":
                        default_value = "us"
                    elif setting.key == "search.questions_per_iteration":
                        default_value = 3
                    elif setting.key == "search.searches_per_section":
                        default_value = 2
                    elif setting.key == "search.skip_relevance_filter":
                        default_value = False
                    elif setting.key == "search.safe_search":
                        default_value = True
                    elif setting.key == "search.search_language":
                        default_value = "English"
                elif setting.key.startswith("report."):
                    if setting.key == "report.searches_per_section":
                        default_value = 2
                    elif (
                        setting.key == "report.enable_fact_checking"
                        or setting.key == "report.detailed_citations"
                    ):
                        default_value = True
                elif setting.key.startswith("app."):
                    if (
                        setting.key == "app.theme"
                        or setting.key == "app.default_theme"
                    ):
                        default_value = "dark"
                    elif setting.key == "app.enable_notifications" or (
                        setting.key == "app.enable_web"
                        or setting.key == "app.web_interface"
                    ):
                        default_value = True
                    elif setting.key == "app.host":
                        default_value = "0.0.0.0"
                    elif setting.key == "app.port":
                        default_value = 5000
                    elif setting.key == "app.debug":
                        default_value = True

                # Update the setting with the default value if found
                if default_value is not None:
                    setting.value = default_value
                    fixed_settings.append(setting.key)
                else:
                    # If no default found but it's a corrupted JSON, set to empty object
                    if setting.key.startswith("report."):
                        setting.value = {}
                        fixed_settings.append(setting.key)

            # Commit changes
            if (
                fixed_settings
                or removed_duplicate_settings
                or fixed_scoping_issues
            ):
                db_session.commit()
                logger.info(
                    f"Fixed {len(fixed_settings)} corrupted settings: {', '.join(fixed_settings)}"
                )
                if removed_duplicate_settings:
                    logger.info(
                        f"Removed {len(removed_duplicate_settings)} duplicate settings"
                    )
                if fixed_scoping_issues:
                    logger.info(
                        f"Fixed {len(fixed_scoping_issues)} scoping issues"
                    )

            # Return success
            return jsonify(
                {
                    "status": "success",
                    "message": f"Fixed {len(fixed_settings)} corrupted settings, removed {len(removed_duplicate_settings)} duplicates, and fixed {len(fixed_scoping_issues)} scoping issues",
                    "fixed_settings": fixed_settings,
                    "removed_duplicates": removed_duplicate_settings,
                    "fixed_scoping": fixed_scoping_issues,
                }
            )

        except Exception:
            logger.exception("Error fixing corrupted settings")
            db_session.rollback()
            return (
                jsonify(
                    {
                        "status": "error",
                        "message": "An internal error occurred while fixing corrupted settings. Please try again later.",
                    }
                ),
                500,
            )


@settings_bp.route("/api/warnings", methods=["GET"])
@login_required
def api_get_warnings():
    """Get current warnings based on settings"""
    try:
        warnings = calculate_warnings()
        return jsonify({"warnings": warnings})
    except Exception:
        logger.exception("Error getting warnings")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/ollama-status", methods=["GET"])
@login_required
def check_ollama_status():
    """Check if Ollama is running and available"""
    try:
        # Get Ollama URL from settings
        raw_base_url = _get_setting_from_session(
            "llm.ollama.url", "http://localhost:11434"
        )
        base_url = (
            normalize_url(raw_base_url)
            if raw_base_url
            else "http://localhost:11434"
        )

        response = requests.get(f"{base_url}/api/version", timeout=2.0)

        if response.status_code == 200:
            return jsonify(
                {
                    "running": True,
                    "version": response.json().get("version", "unknown"),
                }
            )
        else:
            return jsonify(
                {
                    "running": False,
                    "error": f"Ollama returned status code {response.status_code}",
                }
            )
    except requests.exceptions.RequestException:
        logger.exception("Ollama check failed")
        return jsonify(
            {"running": False, "error": "Failed to check search engine status"}
        )


@settings_bp.route("/api/rate-limiting/status", methods=["GET"])
@login_required
def api_get_rate_limiting_status():
    """Get current rate limiting status and statistics"""
    try:
        from ...web_search_engines.rate_limiting import get_tracker

        tracker = get_tracker()

        # Get basic status
        status = {
            "enabled": tracker.enabled,
            "exploration_rate": tracker.exploration_rate,
            "learning_rate": tracker.learning_rate,
            "memory_window": tracker.memory_window,
        }

        # Get engine statistics
        engine_stats = tracker.get_stats()
        engines = []

        for stat in engine_stats:
            (
                engine_type,
                base_wait,
                min_wait,
                max_wait,
                last_updated,
                total_attempts,
                success_rate,
            ) = stat
            engines.append(
                {
                    "engine_type": engine_type,
                    "base_wait_seconds": round(base_wait, 2),
                    "min_wait_seconds": round(min_wait, 2),
                    "max_wait_seconds": round(max_wait, 2),
                    "last_updated": last_updated,
                    "total_attempts": total_attempts,
                    "success_rate": (
                        round(success_rate * 100, 1) if success_rate else 0.0
                    ),
                }
            )

        return jsonify({"status": status, "engines": engines})

    except Exception:
        logger.exception("Error getting rate limiting status")
        return jsonify({"error": "An internal error occurred"}), 500


@settings_bp.route(
    "/api/rate-limiting/engines/<engine_type>/reset", methods=["POST"]
)
@login_required
def api_reset_engine_rate_limiting(engine_type):
    """Reset rate limiting data for a specific engine"""
    try:
        from ...web_search_engines.rate_limiting import get_tracker

        tracker = get_tracker()
        tracker.reset_engine(engine_type)

        return jsonify(
            {"message": f"Rate limiting data reset for {engine_type}"}
        )

    except Exception:
        logger.exception(f"Error resetting rate limiting for {engine_type}")
        return jsonify({"error": "An internal error occurred"}), 500


@settings_bp.route("/api/rate-limiting/cleanup", methods=["POST"])
@login_required
def api_cleanup_rate_limiting():
    """Clean up old rate limiting data"""
    try:
        from ...web_search_engines.rate_limiting import get_tracker

        days = request.json.get("days", 30) if request.is_json else 30

        tracker = get_tracker()
        tracker.cleanup_old_data(days)

        return jsonify(
            {"message": f"Cleaned up rate limiting data older than {days} days"}
        )

    except Exception:
        logger.exception("Error cleaning up rate limiting data")
        return jsonify({"error": "An internal error occurred"}), 500


@settings_bp.route("/api/bulk", methods=["GET"])
@login_required
def get_bulk_settings():
    """Get multiple settings at once for performance."""
    try:
        # Get requested settings from query parameters
        requested = request.args.getlist("keys[]")
        if not requested:
            # Default to common settings if none specified
            requested = [
                "llm.provider",
                "llm.model",
                "search.tool",
                "search.iterations",
                "search.questions_per_iteration",
                "search.search_strategy",
                "benchmark.evaluation.provider",
                "benchmark.evaluation.model",
                "benchmark.evaluation.temperature",
                "benchmark.evaluation.endpoint_url",
            ]

        # Fetch all settings at once
        result = {}
        for key in requested:
            try:
                value = _get_setting_from_session(key)
                result[key] = {"value": value, "exists": value is not None}
            except Exception as e:
                logger.warning(f"Error getting setting {key}: {e}")
                result[key] = {
                    "value": None,
                    "exists": False,
                    "error": "Failed to retrieve setting",
                }

        return jsonify({"success": True, "settings": result})

    except Exception:
        logger.exception("Error getting bulk settings")
        return jsonify(
            {"success": False, "error": "An internal error occurred"}
        ), 500


@settings_bp.route("/api/data-location", methods=["GET"])
@login_required
def api_get_data_location():
    """Get information about data storage location and security"""
    try:
        # Get the data directory path
        data_dir = get_data_directory()
        # Get the encrypted databases path
        encrypted_db_path = get_encrypted_database_path()

        # Check if LDR_DATA_DIR environment variable is set
        from local_deep_research.settings.manager import SettingsManager

        settings_manager = SettingsManager()
        custom_data_dir = settings_manager.get_setting("bootstrap.data_dir")

        # Get platform-specific default location info
        platform_info = {
            "Windows": "C:\\Users\\Username\\AppData\\Local\\local-deep-research",
            "macOS": "~/Library/Application Support/local-deep-research",
            "Linux": "~/.local/share/local-deep-research",
        }

        # Current platform
        current_platform = platform.system()
        if current_platform == "Darwin":
            current_platform = "macOS"

        # Get SQLCipher settings from environment
        from ...database.sqlcipher_utils import get_sqlcipher_settings

        # Debug logging
        logger.info(f"db_manager type: {type(db_manager)}")
        logger.info(
            f"db_manager.has_encryption: {getattr(db_manager, 'has_encryption', 'ATTRIBUTE NOT FOUND')}"
        )

        cipher_settings = (
            get_sqlcipher_settings() if db_manager.has_encryption else {}
        )

        return jsonify(
            {
                "data_directory": str(data_dir),
                "database_path": str(encrypted_db_path),
                "encrypted_database_path": str(encrypted_db_path),
                "is_custom": custom_data_dir is not None,
                "custom_env_var": "LDR_DATA_DIR",
                "custom_env_value": custom_data_dir,
                "platform": current_platform,
                "platform_default": platform_info.get(
                    current_platform, str(data_dir)
                ),
                "platform_info": platform_info,
                "security_notice": {
                    "encrypted": db_manager.has_encryption,
                    "warning": "All data including API keys stored in the database are securely encrypted."
                    if db_manager.has_encryption
                    else "All data including API keys stored in the database are currently unencrypted. Please ensure appropriate file system permissions are set.",
                    "recommendation": "Your data is protected with database encryption."
                    if db_manager.has_encryption
                    else "Consider using environment variables for sensitive API keys instead of storing them in the database.",
                },
                "encryption_settings": cipher_settings,
            }
        )

    except Exception:
        logger.exception("Error getting data location information")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/notifications/test-url", methods=["POST"])
@login_required
def api_test_notification_url():
    """
    Test a notification service URL.

    This endpoint creates a temporary NotificationService instance to test
    the provided URL. No database session or password is required because:
    - The service URL is provided directly in the request body
    - Test notifications use a temporary Apprise instance
    - No user settings or database queries are performed

    Security note: Rate limiting is not applied here because users need to
    test URLs when configuring notifications. Abuse is mitigated by the
    @login_required decorator and the fact that users can only spam their
    own notification services.
    """
    try:
        from ...notifications.service import NotificationService

        data = request.get_json()
        if not data or "service_url" not in data:
            return jsonify(
                {"success": False, "error": "service_url is required"}
            ), 400

        service_url = data["service_url"]

        # Create notification service instance and test the URL
        # No password/session needed - URL provided directly, no DB access
        notification_service = NotificationService()
        result = notification_service.test_service(service_url)

        # Only return expected fields to prevent information leakage
        safe_response = {
            "success": result.get("success", False),
            "message": result.get("message", ""),
            "error": result.get("error", ""),
        }
        return jsonify(safe_response)

    except Exception:
        logger.exception("Error testing notification URL")
        return jsonify(
            {
                "success": False,
                "error": "Failed to test notification service. Check logs for details.",
            }
        ), 500
