"""
Settings Routes Module

This module handles all settings-related HTTP endpoints for the application.

CHECKBOX HANDLING PATTERN:
--------------------------
This module supports TWO submission modes to handle checkboxes correctly:

**MODE 1: AJAX/JSON Submission (Primary - /save_all_settings)**
- JavaScript intercepts form submission with e.preventDefault()
- Checkbox values read directly from DOM via checkbox.checked
- Data sent as JSON: {"setting.key": true/false}
- Hidden fallback inputs are managed but NOT used in this mode
- Provides better UX with instant feedback and validation

**MODE 2: Traditional POST Submission (Fallback - /save_settings)**
- Used when JavaScript is disabled (accessibility/no-JS environments)
- Browser submits form data naturally via request.form
- Hidden fallback pattern CRITICAL here:
  * Checked checkbox: Submits checkbox value, hidden input disabled
  * Unchecked checkbox: Submits hidden input value "false"
- Ensures unchecked checkboxes are captured (HTML limitation workaround)

**Implementation Details:**
1. Each checkbox has `data-hidden-fallback` attribute → hidden input ID
2. checkbox_handler.js manages hidden input disabled state
3. AJAX mode: settings.js reads checkbox.checked directly (lines 2233-2240)
4. POST mode: Flask reads request.form including enabled hidden inputs
5. Both modes use convert_setting_value() for consistent boolean conversion

**Why Both Patterns?**
- AJAX: Better UX, immediate validation, no page reload
- Traditional POST: Accessibility, progressive enhancement, JavaScript-free operation
- Hidden inputs: Only meaningful for traditional POST, ignored in AJAX mode

This dual-mode approach ensures the app works for all users while providing
optimal experience when JavaScript is available.
"""

import platform
from typing import Any, Optional, Tuple
from datetime import UTC, datetime, timedelta, timezone

import requests
from flask import (
    Blueprint,
    flash,
    jsonify,
    redirect,
    request,
    session,
    url_for,
)
from flask_wtf.csrf import generate_csrf
from loguru import logger

from ...config.constants import DEFAULT_OLLAMA_URL
from ...config.paths import get_data_directory, get_encrypted_database_path
from ...database.models import Setting, SettingType
from ...database.session_context import get_user_db_session
from ...database.encrypted_db import db_manager
from ...utilities.db_utils import get_settings_manager
from ...utilities.url_utils import normalize_url
from ...security.decorators import require_json_body
from ..auth.decorators import login_required
from ...security.rate_limiter import settings_limit
from ...settings.manager import get_typed_setting_value, parse_boolean
from ..services.settings_service import (
    create_or_update_setting,
    invalidate_settings_caches,
    set_setting,
)
from ..utils.route_decorators import with_user_session
from ..utils.templates import render_template_with_defaults


from ...security import safe_get
from ..warning_checks import calculate_warnings

# Create a Blueprint for settings
settings_bp = Blueprint("settings", __name__, url_prefix="/settings")

# NOTE: Routes use session["username"] (not .get()) intentionally.
# @login_required guarantees the key exists; direct access fails fast
# if the decorator is ever removed.

# Settings with dynamically populated options (excluded from validation)
DYNAMIC_SETTINGS = ["llm.provider", "llm.model", "search.tool"]


def _get_setting_from_session(key: str, default=None):
    """Helper to get a setting using the current session context."""
    username = session.get("username")
    with get_user_db_session(username) as db_session:
        if db_session:
            settings_manager = get_settings_manager(db_session, username)
            return settings_manager.get_setting(key, default)
    return default


def validate_setting(
    setting: Setting, value: Any
) -> Tuple[bool, Optional[str]]:
    """
    Validate a setting value based on its type and constraints.

    Args:
        setting: The Setting object to validate against
        value: The value to validate

    Returns:
        Tuple of (is_valid, error_message)
    """
    # Convert value to appropriate type first using SettingsManager's logic
    value = get_typed_setting_value(
        key=str(setting.key),
        value=value,
        ui_element=str(setting.ui_element),
        default=None,
        check_env=False,
    )

    # Validate based on UI element type
    if setting.ui_element == "checkbox":
        # After conversion, should be boolean
        if not isinstance(value, bool):
            return False, "Value must be a boolean"

    elif setting.ui_element in ("number", "slider", "range"):
        # After conversion, should be numeric
        if not isinstance(value, (int, float)):
            return False, "Value must be a number"

        # Check min/max constraints if defined
        if setting.min_value is not None and value < setting.min_value:
            return False, f"Value must be at least {setting.min_value}"
        if setting.max_value is not None and value > setting.max_value:
            return False, f"Value must be at most {setting.max_value}"

    elif setting.ui_element == "select":
        # Check if value is in the allowed options
        if setting.options:
            # Skip options validation for dynamically populated dropdowns
            if setting.key not in DYNAMIC_SETTINGS:
                allowed_values = [
                    opt.get("value") if isinstance(opt, dict) else opt
                    for opt in list(setting.options)  # type: ignore[arg-type]
                ]
                if value not in allowed_values:
                    return (
                        False,
                        f"Value must be one of: {', '.join(str(v) for v in allowed_values)}",
                    )

    # All checks passed
    return True, None


def coerce_setting_for_write(key: str, value: Any, ui_element: str) -> Any:
    """Coerce an incoming value to the correct type before writing to the DB.

    All web routes that save settings should use this function to ensure
    consistent type conversion.

    No JSON pre-parsing (``json.loads``) is needed here because:
    - ``get_typed_setting_value`` already parses JSON strings internally
      via ``_parse_json_value`` (for ``ui_element="json"``) and
      ``_parse_multiselect`` (for ``ui_element="multiselect"``).
    - For JSON API endpoints, ``request.get_json()`` already delivers
      dicts/lists as native Python objects.
    - For ``ui_element="text"``, pre-parsing would corrupt data: a JSON
      string like ``'{"k": "v"}'`` would become a dict, then ``str()``
      would produce ``"{'k': 'v'}"`` (Python repr, not valid JSON).
    """
    # check_env=False: we are persisting a user-supplied value, not reading
    # from an environment variable override.  check_env=True (the default)
    # would silently replace the user's value with an env var, which is
    # incorrect on the write path.
    return get_typed_setting_value(
        key=key,
        value=value,
        ui_element=ui_element,
        default=None,
        check_env=False,
    )


@settings_bp.route("/", methods=["GET"])
@login_required
def settings_page():
    """Main settings dashboard with links to specialized config pages"""
    return render_template_with_defaults("settings_dashboard.html")


@settings_bp.route("/save_all_settings", methods=["POST"])
@login_required
@settings_limit
@require_json_body(
    error_format="status", error_message="No settings data provided"
)
@with_user_session()
def save_all_settings(db_session=None, settings_manager=None):
    """Handle saving all settings at once from the unified settings page"""
    try:
        # Process JSON data
        form_data = request.get_json()
        if not form_data:
            return (
                jsonify(
                    {
                        "status": "error",
                        "message": "No settings data provided",
                    }
                ),
                400,
            )

        # Track validation errors
        validation_errors = []
        settings_by_type: dict[str, Any] = {}

        # Track changes for logging
        updated_settings = []
        created_settings = []

        # Store original values for better messaging
        original_values = {}

        # Fetch all settings once to avoid N+1 query problem
        all_db_settings = {
            setting.key: setting for setting in db_session.query(Setting).all()
        }

        # Filter out non-editable settings
        non_editable_keys = [
            key
            for key in form_data.keys()
            if key in all_db_settings and not all_db_settings[key].editable
        ]
        if non_editable_keys:
            logger.warning(
                f"Skipping non-editable settings: {non_editable_keys}"
            )
            for key in non_editable_keys:
                del form_data[key]

        # Update each setting
        for key, value in form_data.items():
            # Skip corrupted keys or empty strings as keys
            if not key or not isinstance(key, str) or key.strip() == "":
                continue

            # Get the setting metadata from pre-fetched dict
            current_setting = all_db_settings.get(key)

            # EARLY VALIDATION: Convert checkbox values BEFORE any other processing
            # This prevents incorrect triggering of corrupted value detection
            if current_setting and current_setting.ui_element == "checkbox":
                if not isinstance(value, bool):
                    logger.debug(
                        f"Converting checkbox {key} from {type(value).__name__} to bool: {value}"
                    )
                    value = parse_boolean(value)
                    form_data[key] = (
                        value  # Update the form_data with converted value
                    )

            # Store original value for messaging
            if current_setting:
                original_values[key] = current_setting.value

            # Determine setting type and category
            if key.startswith("llm."):
                setting_type = SettingType.LLM
                category = "llm_general"
                if (
                    "temperature" in key
                    or "max_tokens" in key
                    or "batch" in key
                    or "layers" in key
                ):
                    category = "llm_parameters"
            elif key.startswith("search."):
                setting_type = SettingType.SEARCH
                category = "search_general"
                if (
                    "iterations" in key
                    or "results" in key
                    or "region" in key
                    or "questions" in key
                    or "section" in key
                ):
                    category = "search_parameters"
            elif key.startswith("report."):
                setting_type = SettingType.REPORT
                category = "report_parameters"
            elif key.startswith("database."):
                setting_type = SettingType.DATABASE
                category = "database_parameters"
            elif key.startswith("app."):
                setting_type = SettingType.APP
                category = "app_interface"
            else:
                setting_type = None
                category = None

            # Special handling for corrupted or empty values
            if value == "[object Object]" or (
                isinstance(value, str)
                and value.strip() in ["{}", "[]", "{", "["]
            ):
                if key.startswith("report."):
                    value = {}
                else:
                    # Use default or null for other types
                    if key == "llm.model":
                        value = "gpt-3.5-turbo"
                    elif key == "llm.provider":
                        value = "openai"
                    elif key == "search.tool":
                        value = "auto"
                    elif key in ["app.theme", "app.default_theme"]:
                        value = "dark"
                    else:
                        value = None

                logger.warning(f"Corrected corrupted value for {key}: {value}")
                # NOTE: No JSON pre-parsing is done here.  After the
                # corruption replacement above, values are Python dicts
                # (e.g. {}), hardcoded strings, or None — none are JSON
                # strings that need parsing.  Type conversion below via
                # coerce_setting_for_write() handles everything; that
                # function delegates to get_typed_setting_value() which
                # already parses JSON internally for "json" and
                # "multiselect" ui_elements.

            if current_setting:
                # Coerce to correct Python type (e.g. str "5" → int 5
                # for number settings, str "true" → bool for checkboxes).
                converted_value = coerce_setting_for_write(
                    key=current_setting.key,
                    value=value,
                    ui_element=current_setting.ui_element,
                )

                # Validate the setting
                is_valid, error_message = validate_setting(
                    current_setting, converted_value
                )

                if is_valid:
                    # Save the converted setting using the same session
                    success = set_setting(
                        key, converted_value, db_session=db_session
                    )
                    if success:
                        updated_settings.append(key)

                    # Track settings by type for exporting
                    if current_setting.type not in settings_by_type:
                        settings_by_type[current_setting.type] = []
                    settings_by_type[current_setting.type].append(
                        current_setting
                    )
                else:
                    # Add to validation errors
                    validation_errors.append(
                        {
                            "key": key,
                            "name": current_setting.name,
                            "error": error_message,
                        }
                    )
            else:
                # Create a new setting
                new_setting = {
                    "key": key,
                    "value": value,
                    "type": setting_type.value.lower()
                    if setting_type is not None
                    else "app",
                    "name": key.split(".")[-1].replace("_", " ").title(),
                    "description": f"Setting for {key}",
                    "category": category,
                    "ui_element": "text",  # Default UI element
                }

                # Determine better UI element based on value type
                if isinstance(value, bool):
                    new_setting["ui_element"] = "checkbox"
                elif isinstance(value, (int, float)) and not isinstance(
                    value, bool
                ):
                    new_setting["ui_element"] = "number"
                elif isinstance(value, (dict, list)):
                    new_setting["ui_element"] = "textarea"

                # Create the setting
                db_setting = create_or_update_setting(
                    new_setting, db_session=db_session
                )

                if db_setting:
                    created_settings.append(key)
                    # Track settings by type for exporting
                    if db_setting.type not in settings_by_type:
                        settings_by_type[db_setting.type] = []
                    settings_by_type[db_setting.type].append(db_setting)
                else:
                    validation_errors.append(
                        {
                            "key": key,
                            "name": new_setting["name"],
                            "error": "Failed to create setting",
                        }
                    )

        # Report validation errors if any
        if validation_errors:
            return (
                jsonify(
                    {
                        "status": "error",
                        "message": "Validation errors",
                        "errors": validation_errors,
                    }
                ),
                400,
            )

        # Get all settings to return to the client for proper state update
        all_settings = {}
        for setting in db_session.query(Setting).all():
            # Convert enum to string if present
            setting_type = setting.type
            if hasattr(setting_type, "value"):
                setting_type = setting_type.value

            all_settings[setting.key] = {
                "value": setting.value,
                "name": setting.name,
                "description": setting.description,
                "type": setting_type,
                "category": setting.category,
                "ui_element": setting.ui_element,
                "editable": setting.editable,
                "options": setting.options,
                "visible": setting.visible,
                "min_value": setting.min_value,
                "max_value": setting.max_value,
                "step": setting.step,
            }

        # Customize the success message based on what changed
        success_message = ""
        if len(updated_settings) == 1:
            # For a single update, provide more specific info about what changed
            key = updated_settings[0]
            # Reuse the already-fetched setting from our pre-fetched dict
            updated_setting = all_db_settings.get(key)
            name = (
                updated_setting.name
                if updated_setting
                else key.split(".")[-1].replace("_", " ").title()
            )

            # Format the message
            if key in original_values:
                new_value = updated_setting.value if updated_setting else None

                # If it's a boolean, use "enabled/disabled" language
                if isinstance(new_value, bool):
                    state = "enabled" if new_value else "disabled"
                    success_message = f"{name} {state}"
                else:
                    # For non-boolean values
                    if isinstance(new_value, (dict, list)):
                        success_message = f"{name} updated"
                    else:
                        success_message = f"{name} updated"
            else:
                success_message = f"{name} updated"
        else:
            # Multiple settings or generic message
            success_message = f"Settings saved successfully ({len(updated_settings)} updated, {len(created_settings)} created)"

        # Check if any warning-affecting settings were changed and include warnings
        response_data = {
            "status": "success",
            "message": success_message,
            "updated": updated_settings,
            "created": created_settings,
            "settings": all_settings,
        }

        warning_affecting_keys = [
            "llm.provider",
            "search.tool",
            "search.iterations",
            "search.questions_per_iteration",
            "llm.local_context_window_size",
            "llm.context_window_unrestricted",
            "llm.context_window_size",
        ]

        # Check if any warning-affecting settings were changed
        if any(
            key in warning_affecting_keys
            for key in updated_settings + created_settings
        ):
            warnings = calculate_warnings()
            response_data["warnings"] = warnings
            logger.info(
                f"Bulk settings update affected warning keys, calculated {len(warnings)} warnings"
            )

        invalidate_settings_caches(session["username"])
        return jsonify(response_data)

    except Exception:
        logger.exception("Error saving settings")
        return (
            jsonify(
                {
                    "status": "error",
                    "message": "An internal error occurred while saving settings.",
                }
            ),
            500,
        )


@settings_bp.route("/reset_to_defaults", methods=["POST"])
@login_required
@settings_limit
@with_user_session()
def reset_to_defaults(db_session=None, settings_manager=None):
    """Reset all settings to their default values"""
    try:
        settings_manager.load_from_defaults_file()

        logger.info("Successfully imported settings from default files")

    except Exception:
        logger.exception("Error importing default settings")
        return jsonify(
            {
                "status": "error",
                "message": "Failed to reset settings to defaults",
            }
        ), 500

    invalidate_settings_caches(session["username"])
    return jsonify(
        {
            "status": "success",
            "message": "All settings have been reset to default values",
        }
    )


@settings_bp.route("/save_settings", methods=["POST"])
@login_required
@settings_limit
@with_user_session()
def save_settings(db_session=None, settings_manager=None):
    """Save all settings from the form using POST method - fallback when JavaScript is disabled"""
    try:
        # Get form data
        form_data = request.form.to_dict()

        # Remove CSRF token from the data
        form_data.pop("csrf_token", None)

        updated_count = 0
        failed_count = 0

        # Fetch all settings once to avoid N+1 query problem
        all_db_settings = {
            setting.key: setting for setting in db_session.query(Setting).all()
        }

        # Filter out non-editable settings
        non_editable_keys = [
            key
            for key in form_data.keys()
            if key in all_db_settings and not all_db_settings[key].editable
        ]
        if non_editable_keys:
            logger.warning(
                f"Skipping non-editable settings: {non_editable_keys}"
            )
            for key in non_editable_keys:
                del form_data[key]

        # Process each setting
        for key, value in form_data.items():
            try:
                # Get the setting from pre-fetched dict
                db_setting = all_db_settings.get(key)

                # Coerce form POST string to correct Python type.
                if db_setting:
                    value = coerce_setting_for_write(
                        key=db_setting.key,
                        value=value,
                        ui_element=db_setting.ui_element,
                    )

                # Save the setting
                if settings_manager.set_setting(key, value, commit=False):
                    updated_count += 1
                else:
                    failed_count += 1
                    logger.warning(f"Failed to save setting {key}")

            except Exception:
                logger.exception(f"Error saving setting {key}")
                failed_count += 1

        # Commit all changes at once
        try:
            db_session.commit()

            flash(
                f"Settings saved successfully! Updated {updated_count} settings.",
                "success",
            )
            if failed_count > 0:
                flash(
                    f"Warning: {failed_count} settings failed to save.",
                    "warning",
                )
            invalidate_settings_caches(session["username"])

        except Exception:
            db_session.rollback()
            logger.exception("Failed to commit settings")
            flash("Error saving settings. Please try again.", "error")

        return redirect(url_for("settings.settings_page"))

    except Exception:
        logger.exception("Error in save_settings")
        flash("An internal error occurred while saving settings.", "error")
        return redirect(url_for("settings.settings_page"))


# API Routes
@settings_bp.route("/api", methods=["GET"])
@login_required
@with_user_session()
def api_get_all_settings(db_session=None, settings_manager=None):
    """Get all settings"""
    try:
        # Get query parameters
        category = request.args.get("category")

        # Get settings
        settings = settings_manager.get_all_settings()

        # Filter by category if requested
        if category:
            # Need to get all setting details to check category
            db_settings = db_session.query(Setting).all()
            category_keys = [
                s.key for s in db_settings if s.category == category
            ]

            # Filter settings by keys
            settings = {
                key: value
                for key, value in settings.items()
                if key in category_keys
            }

        return jsonify({"status": "success", "settings": settings})
    except Exception:
        logger.exception("Error getting settings")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/<path:key>", methods=["GET"])
@login_required
@with_user_session()
def api_get_db_setting(key, db_session=None, settings_manager=None):
    """Get a specific setting by key from DB, falling back to defaults."""
    try:
        # Get setting from database using the same session
        db_setting = (
            db_session.query(Setting).filter(Setting.key == key).first()
        )

        if db_setting:
            # Return full setting details from DB
            setting_data = {
                "key": db_setting.key,
                "value": db_setting.value,
                "type": db_setting.type
                if isinstance(db_setting.type, str)
                else db_setting.type.value,
                "name": db_setting.name,
                "description": db_setting.description,
                "category": db_setting.category,
                "ui_element": db_setting.ui_element,
                "options": db_setting.options,
                "min_value": db_setting.min_value,
                "max_value": db_setting.max_value,
                "step": db_setting.step,
                "visible": db_setting.visible,
                "editable": db_setting.editable,
            }
            return jsonify(setting_data)

        # Not in DB — check defaults so this endpoint is consistent
        # with GET /settings/api which includes default settings
        default_meta = settings_manager.default_settings.get(key)
        if default_meta:
            setting_data = {
                "key": key,
                "value": default_meta.get("value"),
                "type": default_meta.get("type", "APP"),
                "name": default_meta.get("name", key),
                "description": default_meta.get("description"),
                "category": default_meta.get("category"),
                "ui_element": default_meta.get("ui_element", "text"),
                "options": default_meta.get("options"),
                "min_value": default_meta.get("min_value"),
                "max_value": default_meta.get("max_value"),
                "step": default_meta.get("step"),
                "visible": default_meta.get("visible", True),
                "editable": default_meta.get("editable", True),
            }
            return jsonify(setting_data)

        return jsonify({"error": f"Setting not found: {key}"}), 404
    except Exception:
        logger.exception(f"Error getting setting {key}")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/<path:key>", methods=["PUT"])
@login_required
@require_json_body(error_message="No data provided")
@with_user_session(include_settings_manager=False)
def api_update_setting(key, db_session=None):
    """Update a setting"""
    try:
        # Get request data
        data = request.get_json()
        value = data.get("value")
        if value is None:
            return jsonify({"error": "No value provided"}), 400

        # Check if setting exists
        db_setting = (
            db_session.query(Setting).filter(Setting.key == key).first()
        )

        if db_setting:
            # Check if setting is editable
            if not db_setting.editable:
                return jsonify({"error": f"Setting {key} is not editable"}), 403

            # Coerce to correct Python type before saving.
            # Without this, values from JSON API requests are stored
            # as-is (e.g. string "5" instead of int 5 for number
            # settings, string "true" instead of bool for checkboxes).
            value = coerce_setting_for_write(
                key=db_setting.key,
                value=value,
                ui_element=db_setting.ui_element,
            )

            # Validate the setting (matches save_all_settings pattern)
            is_valid, error_message = validate_setting(db_setting, value)
            if not is_valid:
                logger.warning(
                    f"Validation failed for setting {key}: {error_message}"
                )
                return jsonify(
                    {"error": f"Invalid value for setting {key}"}
                ), 400

            # Update setting
            # Pass the db_session to avoid session lookup issues
            success = set_setting(key, value, db_session=db_session)
            if success:
                response_data: dict[str, Any] = {
                    "message": f"Setting {key} updated successfully"
                }

                # If this is a key that affects warnings, include warning calculations
                warning_affecting_keys = [
                    "llm.provider",
                    "search.tool",
                    "search.iterations",
                    "search.questions_per_iteration",
                    "llm.local_context_window_size",
                    "llm.context_window_unrestricted",
                    "llm.context_window_size",
                ]

                if key in warning_affecting_keys:
                    warnings = calculate_warnings()
                    response_data["warnings"] = warnings
                    logger.debug(
                        f"Setting {key} changed to {value}, calculated {len(warnings)} warnings"
                    )

                invalidate_settings_caches(session["username"])
                return jsonify(response_data)
            return jsonify({"error": f"Failed to update setting {key}"}), 500
        # Create new setting with default metadata
        setting_dict = {
            "key": key,
            "value": value,
            "name": key.split(".")[-1].replace("_", " ").title(),
            "description": f"Setting for {key}",
        }

        # Add additional metadata if provided
        for field in [
            "type",
            "name",
            "description",
            "category",
            "ui_element",
            "options",
            "min_value",
            "max_value",
            "step",
            "visible",
            "editable",
        ]:
            if field in data:
                setting_dict[field] = data[field]

        # Create setting
        db_setting = create_or_update_setting(
            setting_dict, db_session=db_session
        )

        if db_setting:
            invalidate_settings_caches(session["username"])
            return (
                jsonify(
                    {
                        "message": f"Setting {key} created successfully",
                        "setting": {
                            "key": db_setting.key,
                            "value": db_setting.value,
                            "type": db_setting.type.value,
                            "name": db_setting.name,
                        },
                    }
                ),
                201,
            )
        return jsonify({"error": f"Failed to create setting {key}"}), 500
    except Exception:
        logger.exception(f"Error updating setting {key}")
        return jsonify({"error": "Failed to update setting"}), 500


@settings_bp.route("/api/<path:key>", methods=["DELETE"])
@login_required
@with_user_session()
def api_delete_setting(key, db_session=None, settings_manager=None):
    """Delete a setting"""
    try:
        # Check if setting exists
        db_setting = (
            db_session.query(Setting).filter(Setting.key == key).first()
        )
        if not db_setting:
            return jsonify({"error": f"Setting not found: {key}"}), 404

        # Check if setting is editable
        if not db_setting.editable:
            return jsonify({"error": f"Setting {key} is not editable"}), 403

        # Delete setting
        success = settings_manager.delete_setting(key)
        if success:
            invalidate_settings_caches(session["username"])
            return jsonify({"message": f"Setting {key} deleted successfully"})
        return jsonify({"error": f"Failed to delete setting {key}"}), 500
    except Exception:
        logger.exception(f"Error deleting setting {key}")
        return jsonify({"error": "Failed to delete setting"}), 500


@settings_bp.route("/api/import", methods=["POST"])
@login_required
@settings_limit
@with_user_session()
def api_import_settings(db_session=None, settings_manager=None):
    """Import settings from defaults file"""
    try:
        settings_manager.load_from_defaults_file()

        invalidate_settings_caches(session["username"])
        return jsonify({"message": "Settings imported successfully"})
    except Exception:
        logger.exception("Error importing settings")
        return jsonify({"error": "Failed to import settings"}), 500


@settings_bp.route("/api/categories", methods=["GET"])
@login_required
@with_user_session(include_settings_manager=False)
def api_get_categories(db_session=None):
    """Get all setting categories"""
    try:
        # Get all distinct categories
        categories = db_session.query(Setting.category).distinct().all()
        category_list = [c[0] for c in categories if c[0] is not None]

        return jsonify({"categories": category_list})
    except Exception:
        logger.exception("Error getting categories")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/types", methods=["GET"])
@login_required
def api_get_types():
    """Get all setting types"""
    try:
        # Get all setting types
        types = [t.value for t in SettingType]
        return jsonify({"types": types})
    except Exception:
        logger.exception("Error getting types")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/ui_elements", methods=["GET"])
@login_required
def api_get_ui_elements():
    """Get all UI element types"""
    try:
        # Define supported UI element types
        ui_elements = [
            "text",
            "select",
            "checkbox",
            "slider",
            "number",
            "textarea",
            "color",
            "date",
            "file",
            "password",
        ]

        return jsonify({"ui_elements": ui_elements})
    except Exception:
        logger.exception("Error getting UI elements")
        return jsonify({"error": "Failed to retrieve settings"}), 500


@settings_bp.route("/api/available-models", methods=["GET"])
@login_required
def api_get_available_models():
    """Get available LLM models from various providers"""
    try:
        from flask import request

        from ...database.models import ProviderModel

        # Check if force_refresh is requested
        force_refresh = (
            request.args.get("force_refresh", "false").lower() == "true"
        )

        # Get all auto-discovered providers (show all so users can discover
        # and configure providers they haven't set up yet)
        from ...llm.providers import get_discovered_provider_options

        provider_options = get_discovered_provider_options()

        # Add remaining hardcoded providers (complex local providers not yet migrated)
        provider_options.extend(
            [
                {
                    "value": "LLAMACPP",
                    "label": "Llama.cpp (Local GGUF files only)",
                },
            ]
        )

        # Available models by provider
        providers: dict[str, Any] = {}

        # Check database cache first (unless force_refresh is True)
        if not force_refresh:
            try:
                # Define cache expiration (24 hours)
                cache_expiry = datetime.now(UTC) - timedelta(hours=24)

                # Get cached models from database
                username = session["username"]
                with get_user_db_session(username) as db_session:
                    cached_models = (
                        db_session.query(ProviderModel)
                        .filter(ProviderModel.last_updated > cache_expiry)
                        .all()
                    )

                if cached_models:
                    logger.info(
                        f"Found {len(cached_models)} cached models in database"
                    )

                    # Group models by provider
                    for model in cached_models:
                        provider_key = f"{model.provider.lower()}_models"
                        if provider_key not in providers:
                            providers[provider_key] = []

                        providers[provider_key].append(
                            {
                                "value": model.model_key,
                                "label": model.model_label,
                                "provider": model.provider.upper(),
                            }
                        )

                    # If we have cached data for all providers, return it
                    if providers:
                        logger.info("Returning cached models from database")
                        return jsonify(
                            {
                                "provider_options": provider_options,
                                "providers": providers,
                            }
                        )

            except Exception:
                logger.warning("Error reading cached models from database")
                # Continue to fetch fresh data

        # Try to get Ollama models
        ollama_models = []
        try:
            import json
            import re

            import requests

            # Try to query the Ollama API directly
            try:
                logger.info("Attempting to connect to Ollama API")

                raw_base_url = _get_setting_from_session(
                    "llm.ollama.url", DEFAULT_OLLAMA_URL
                )
                base_url = (
                    normalize_url(raw_base_url)
                    if raw_base_url
                    else DEFAULT_OLLAMA_URL
                )

                ollama_response = safe_get(
                    f"{base_url}/api/tags",
                    timeout=5,
                    allow_localhost=True,
                    allow_private_ips=True,
                )

                logger.debug(
                    f"Ollama API response: Status {ollama_response.status_code}"
                )

                # Try to parse the response even if status code is not 200 to help with debugging
                response_text = ollama_response.text
                logger.debug(
                    f"Ollama API raw response: {response_text[:500]}..."
                )

                if ollama_response.status_code == 200:
                    try:
                        ollama_data = ollama_response.json()
                        logger.debug(
                            f"Ollama API JSON data: {json.dumps(ollama_data)[:500]}..."
                        )

                        if "models" in ollama_data:
                            # Format for newer Ollama API
                            logger.info(
                                f"Found {len(ollama_data.get('models', []))} models in newer Ollama API format"
                            )
                            for model in ollama_data.get("models", []):
                                # Extract name correctly from the model object
                                name = model.get("name", "")
                                if name:
                                    # Improved display name formatting
                                    display_name = re.sub(
                                        r"[:/]", " ", name
                                    ).strip()
                                    display_name = " ".join(
                                        word.capitalize()
                                        for word in display_name.split()
                                    )
                                    # Create the model entry with value and label
                                    ollama_models.append(
                                        {
                                            "value": name,  # Original model name as value (for API calls)
                                            "label": f"{display_name} (Ollama)",  # Pretty name as label
                                            "provider": "OLLAMA",  # Add provider field for consistency
                                        }
                                    )
                                    logger.debug(
                                        f"Added Ollama model: {name} -> {display_name}"
                                    )
                        else:
                            # Format for older Ollama API
                            logger.info(
                                f"Found {len(ollama_data)} models in older Ollama API format"
                            )
                            for model in ollama_data:
                                name = model.get("name", "")
                                if name:
                                    # Improved display name formatting
                                    display_name = re.sub(
                                        r"[:/]", " ", name
                                    ).strip()
                                    display_name = " ".join(
                                        word.capitalize()
                                        for word in display_name.split()
                                    )
                                    ollama_models.append(
                                        {
                                            "value": name,
                                            "label": f"{display_name} (Ollama)",
                                            "provider": "OLLAMA",  # Add provider field for consistency
                                        }
                                    )
                                    logger.debug(
                                        f"Added Ollama model: {name} -> {display_name}"
                                    )

                    except json.JSONDecodeError as json_err:
                        logger.exception(
                            f"Failed to parse Ollama API response as JSON: {json_err}"
                        )
                        raise ValueError(
                            f"Ollama API returned invalid JSON: {json_err}"
                        )
                else:
                    logger.warning(
                        f"Ollama API returned non-200 status code: {ollama_response.status_code}"
                    )
                    raise ValueError(
                        f"Ollama API returned status code {ollama_response.status_code}"
                    )

            except requests.exceptions.RequestException:
                logger.warning("Could not connect to Ollama API")
                # No fallback models - just return empty list
                logger.info("Ollama not available - no models to display")
                ollama_models = []

            # Always set the ollama_models in providers, whether we got real or fallback models
            providers["ollama_models"] = ollama_models
            logger.info(f"Final Ollama models count: {len(ollama_models)}")

            # Log some model names for debugging
            if ollama_models:
                model_names = [m["value"] for m in ollama_models[:5]]
                logger.info(f"Sample Ollama models: {', '.join(model_names)}")

        except Exception:
            logger.exception("Error getting Ollama models")
            # No fallback models - just return empty list
            logger.info("Error getting Ollama models - no models to display")
            providers["ollama_models"] = []

        # Note: OpenAI-Compatible Endpoint models are fetched via auto-discovery
        # (see the auto-discovery loop below which handles OPENAI_ENDPOINT provider)

        # Get OpenAI models using the OpenAI package
        openai_models = []
        try:
            logger.info(
                "Attempting to connect to OpenAI API using OpenAI package"
            )

            # Get the API key from settings
            api_key = _get_setting_from_session("llm.openai.api_key", "")

            if api_key:
                import openai
                from openai import OpenAI

                # Create OpenAI client
                client = OpenAI(api_key=api_key)

                try:
                    # Fetch models using the client
                    logger.debug("Fetching models from OpenAI API")
                    models_response = client.models.list()

                    # Process models from the response
                    for model in models_response.data:
                        model_id = model.id
                        if model_id:
                            # Create a clean display name
                            display_name = model_id.replace("-", " ").strip()
                            display_name = " ".join(
                                word.capitalize()
                                for word in display_name.split()
                            )

                            openai_models.append(
                                {
                                    "value": model_id,
                                    "label": f"{display_name} (OpenAI)",
                                    "provider": "OPENAI",
                                }
                            )
                            logger.debug(
                                f"Added OpenAI model: {model_id} -> {display_name}"
                            )

                    # Keep original order from OpenAI - their models are returned in a
                    # meaningful order (newer/more capable models first)

                except openai.APIError as api_err:
                    logger.exception(f"OpenAI API error: {api_err!s}")
                    logger.info("No OpenAI models found due to API error")

            else:
                logger.info(
                    "OpenAI API key not configured, no models available"
                )

        except Exception:
            logger.exception("Error getting OpenAI models")
            logger.info("No OpenAI models available due to error")

        # Always set the openai_models in providers (will be empty array if no models found)
        providers["openai_models"] = openai_models
        logger.info(f"Final OpenAI models count: {len(openai_models)}")

        # Try to get Anthropic models using the Anthropic package
        anthropic_models = []
        try:
            logger.info(
                "Attempting to connect to Anthropic API using Anthropic package"
            )

            # Get the API key from settings
            api_key = _get_setting_from_session("llm.anthropic.api_key", "")

            if api_key:
                # Import Anthropic package here to avoid dependency issues if not installed
                from anthropic import Anthropic

                # Create Anthropic client
                anthropic_client = Anthropic(api_key=api_key)

                try:
                    # Fetch models using the client
                    logger.debug("Fetching models from Anthropic API")
                    models_response = anthropic_client.models.list()

                    # Process models from the response
                    for model in models_response.data:
                        model_id = model.id
                        if model_id:
                            # Create a clean display name
                            display_name = model_id.replace("-", " ").strip()
                            display_name = " ".join(
                                word.capitalize()
                                for word in display_name.split()
                            )

                            anthropic_models.append(
                                {
                                    "value": model_id,
                                    "label": f"{display_name} (Anthropic)",
                                    "provider": "ANTHROPIC",
                                }
                            )
                            logger.debug(
                                f"Added Anthropic model: {model_id} -> {display_name}"
                            )

                except Exception as api_err:
                    logger.exception(f"Anthropic API error: {api_err!s}")
            else:
                logger.info("Anthropic API key not configured")

        except ImportError:
            logger.warning(
                "Anthropic package not installed. No models will be available."
            )
        except Exception:
            logger.exception("Error getting Anthropic models")

        # Set anthropic_models in providers (could be empty if API call failed)
        providers["anthropic_models"] = anthropic_models
        logger.info(f"Final Anthropic models count: {len(anthropic_models)}")

        # Fetch models from auto-discovered providers
        from ...llm.providers import discover_providers

        discovered_providers = discover_providers()

        for provider_key, provider_info in discovered_providers.items():
            provider_models = []
            try:
                logger.info(
                    f"Fetching models from {provider_info.provider_name}"
                )

                # Get the provider class
                provider_class = provider_info.provider_class

                # Get API key if configured
                api_key = _get_setting_from_session(
                    provider_class.api_key_setting, ""
                )

                # Get base URL if provider has configurable URL
                provider_base_url: str | None = None
                if (
                    hasattr(provider_class, "url_setting")
                    and provider_class.url_setting
                ):
                    provider_base_url = _get_setting_from_session(
                        provider_class.url_setting, ""
                    )

                # Use the provider's list_models_for_api method
                models = provider_class.list_models_for_api(
                    api_key, provider_base_url
                )

                # Format models for the API response
                for model in models:
                    provider_models.append(
                        {
                            "value": model["value"],
                            "label": model[
                                "label"
                            ],  # Use provider's label as-is
                            "provider": provider_key,
                        }
                    )

                logger.info(
                    f"Successfully fetched {len(provider_models)} models from {provider_info.provider_name}"
                )

            except Exception:
                logger.exception(
                    f"Error getting {provider_info.provider_name} models"
                )

            # Set models in providers dict using lowercase key
            providers[f"{provider_key.lower()}_models"] = provider_models
            logger.info(
                f"Final {provider_key} models count: {len(provider_models)}"
            )

        # Save fetched models to database cache
        if force_refresh or providers:
            # We fetched fresh data, save it to database
            username = session["username"]
            with get_user_db_session(username) as db_session:
                try:
                    if force_refresh:
                        # When force refresh, clear ALL cached models to remove any stale data
                        # from old code versions or deleted providers
                        deleted_count = db_session.query(ProviderModel).delete()
                        logger.info(
                            f"Force refresh: cleared all {deleted_count} cached models"
                        )
                    else:
                        # Clear old cache entries only for providers we're updating
                        for provider_key in providers:
                            provider_name = provider_key.replace(
                                "_models", ""
                            ).upper()
                            db_session.query(ProviderModel).filter(
                                ProviderModel.provider == provider_name
                            ).delete()

                    # Insert new models
                    for provider_key, models in providers.items():
                        provider_name = provider_key.replace(
                            "_models", ""
                        ).upper()
                        for model in models:
                            if (
                                isinstance(model, dict)
                                and "value" in model
                                and "label" in model
                            ):
                                new_model = ProviderModel(
                                    provider=provider_name,
                                    model_key=model["value"],
                                    model_label=model["label"],
                                    last_updated=datetime.now(UTC),
                                )
                                db_session.add(new_model)

                    db_session.commit()
                    logger.info("Successfully cached models to database")

                except Exception:
                    logger.exception("Error saving models to database cache")
                    db_session.rollback()

        # Return all options
        return jsonify(
            {"provider_options": provider_options, "providers": providers}
        )

    except Exception:
        logger.exception("Error getting available models")
        return jsonify(
            {
                "status": "error",
                "message": "Failed to retrieve available models",
            }
        ), 500


def _get_engine_icon_and_category(
    engine_data: dict, engine_class=None
) -> tuple:
    """
    Get icon emoji and category label for a search engine based on its attributes.

    Args:
        engine_data: Engine configuration dictionary
        engine_class: Optional loaded engine class to check attributes

    Returns:
        Tuple of (icon, category) strings
    """
    # Check attributes from either the class or the engine data
    if engine_class:
        is_scientific = getattr(engine_class, "is_scientific", False)
        is_generic = getattr(engine_class, "is_generic", False)
        is_local = getattr(engine_class, "is_local", False)
        is_news = getattr(engine_class, "is_news", False)
        is_code = getattr(engine_class, "is_code", False)
    else:
        is_scientific = engine_data.get("is_scientific", False)
        is_generic = engine_data.get("is_generic", False)
        is_local = engine_data.get("is_local", False)
        is_news = engine_data.get("is_news", False)
        is_code = engine_data.get("is_code", False)

    # Check books attribute
    if engine_class:
        is_books = getattr(engine_class, "is_books", False)
    else:
        is_books = engine_data.get("is_books", False)

    # Return icon and category based on engine type
    # Priority: local > scientific > news > code > books > generic > default
    if is_local:
        return "📁", "Local RAG"
    if is_scientific:
        return "🔬", "Scientific"
    if is_news:
        return "📰", "News"
    if is_code:
        return "💻", "Code"
    if is_books:
        return "📚", "Books"
    if is_generic:
        return "🌐", "Web Search"
    return "🔍", "Search"


@settings_bp.route("/api/available-search-engines", methods=["GET"])
@login_required
@with_user_session()
def api_get_available_search_engines(db_session=None, settings_manager=None):
    """Get available search engines"""
    try:
        # Get search engines using the same approach as search_engines_config.py
        from ...web_search_engines.search_engines_config import search_config

        username = session["username"]
        search_engines = search_config(username=username, db_session=db_session)

        # Get user's favorites using SettingsManager
        favorites = settings_manager.get_setting("search.favorites", [])
        if not isinstance(favorites, list):
            favorites = []

        # Extract search engines from config
        engines_dict = {}
        engine_options = []

        if search_engines:
            # Format engines for API response with metadata
            from ...security.module_whitelist import (
                get_safe_module_class,
                SecurityError,
            )

            for engine_id, engine_data in search_engines.items():
                # Try to load the engine class to get metadata
                engine_class = None
                try:
                    module_path = engine_data.get("module_path")
                    class_name = engine_data.get("class_name")
                    if module_path and class_name:
                        # Use secure whitelist-validated import
                        engine_class = get_safe_module_class(
                            module_path, class_name
                        )
                except SecurityError:
                    logger.warning(
                        f"Security: Blocked unsafe module for {engine_id}"
                    )
                except Exception as e:
                    logger.debug(
                        f"Could not load engine class for {engine_id}: {e}"
                    )

                # Get icon and category from engine attributes
                icon, category = _get_engine_icon_and_category(
                    engine_data, engine_class
                )

                # Check if engine requires an API key
                requires_api_key = engine_data.get("requires_api_key", False)

                # Build display name with icon, category, and API key status
                base_name = engine_data.get("display_name", engine_id)
                if requires_api_key:
                    label = f"{icon} {base_name} ({category}, API key)"
                else:
                    label = f"{icon} {base_name} ({category}, Free)"

                # Check if engine is a favorite
                is_favorite = engine_id in favorites

                engines_dict[engine_id] = {
                    "display_name": base_name,
                    "description": engine_data.get("description", ""),
                    "strengths": engine_data.get("strengths", []),
                    "icon": icon,
                    "category": category,
                    "requires_api_key": requires_api_key,
                    "is_favorite": is_favorite,
                }

                engine_options.append(
                    {
                        "value": engine_id,
                        "label": label,
                        "icon": icon,
                        "category": category,
                        "requires_api_key": requires_api_key,
                        "is_favorite": is_favorite,
                    }
                )

        # Sort engine_options: favorites first, then alphabetically by label
        engine_options.sort(
            key=lambda x: (
                not x.get("is_favorite", False),
                x.get("label", "").lower(),
            )
        )

        # If no engines found, log the issue but return empty list
        if not engine_options:
            logger.warning("No search engines found in configuration")

        return jsonify(
            {
                "engines": engines_dict,
                "engine_options": engine_options,
                "favorites": favorites,
            }
        )

    except Exception:
        logger.exception("Error getting available search engines")
        return jsonify({"error": "Failed to retrieve search engines"}), 500


@settings_bp.route("/api/search-favorites", methods=["GET"])
@login_required
@with_user_session()
def api_get_search_favorites(db_session=None, settings_manager=None):
    """Get the list of favorite search engines for the current user"""
    try:
        favorites = settings_manager.get_setting("search.favorites", [])
        if not isinstance(favorites, list):
            favorites = []
        return jsonify({"favorites": favorites})

    except Exception:
        logger.exception("Error getting search favorites")
        return jsonify({"error": "Failed to retrieve favorites"}), 500


@settings_bp.route("/api/search-favorites", methods=["PUT"])
@login_required
@require_json_body(error_message="No data provided")
@with_user_session()
def api_update_search_favorites(db_session=None, settings_manager=None):
    """Update the list of favorite search engines for the current user"""
    try:
        data = request.get_json()
        favorites = data.get("favorites")
        if favorites is None:
            return jsonify({"error": "No favorites provided"}), 400

        if not isinstance(favorites, list):
            return jsonify({"error": "Favorites must be a list"}), 400

        if settings_manager.set_setting("search.favorites", favorites):
            invalidate_settings_caches(session["username"])
            return jsonify(
                {
                    "message": "Favorites updated successfully",
                    "favorites": favorites,
                }
            )
        return jsonify({"error": "Failed to update favorites"}), 500

    except Exception:
        logger.exception("Error updating search favorites")
        return jsonify({"error": "Failed to update favorites"}), 500


@settings_bp.route("/api/search-favorites/toggle", methods=["POST"])
@login_required
@require_json_body(error_message="No data provided")
@with_user_session()
def api_toggle_search_favorite(db_session=None, settings_manager=None):
    """Toggle a search engine as favorite"""
    try:
        data = request.get_json()
        engine_id = data.get("engine_id")
        if not engine_id:
            return jsonify({"error": "No engine_id provided"}), 400

        # Get current favorites
        favorites = settings_manager.get_setting("search.favorites", [])
        if not isinstance(favorites, list):
            favorites = []
        else:
            # Make a copy to avoid modifying the original
            favorites = list(favorites)

        # Toggle the engine
        is_favorite = engine_id in favorites
        if is_favorite:
            favorites.remove(engine_id)
            is_favorite = False
        else:
            favorites.append(engine_id)
            is_favorite = True

        # Update the setting
        if settings_manager.set_setting("search.favorites", favorites):
            invalidate_settings_caches(session["username"])
            return jsonify(
                {
                    "message": "Favorite toggled successfully",
                    "engine_id": engine_id,
                    "is_favorite": is_favorite,
                    "favorites": favorites,
                }
            )
        return jsonify({"error": "Failed to toggle favorite"}), 500

    except Exception:
        logger.exception("Error toggling search favorite")
        return jsonify({"error": "Failed to toggle favorite"}), 500


# Legacy routes for backward compatibility - these will redirect to the new routes
@settings_bp.route("/main", methods=["GET"])
@login_required
def main_config_page():
    """Redirect to app settings page"""
    return redirect(url_for("settings.settings_page"))


@settings_bp.route("/collections", methods=["GET"])
@login_required
def collections_config_page():
    """Redirect to app settings page"""
    return redirect(url_for("settings.settings_page"))


@settings_bp.route("/api_keys", methods=["GET"])
@login_required
def api_keys_config_page():
    """Redirect to LLM settings page"""
    return redirect(url_for("settings.settings_page"))


@settings_bp.route("/search_engines", methods=["GET"])
@login_required
def search_engines_config_page():
    """Redirect to search settings page"""
    return redirect(url_for("settings.settings_page"))


@settings_bp.route("/llm", methods=["GET"])
@login_required
def llm_config_page():
    """Redirect to LLM settings page"""
    return redirect(url_for("settings.settings_page"))


@settings_bp.route("/open_file_location", methods=["POST"])
@login_required
def open_file_location():
    """Open the location of a configuration file.

    Security: This endpoint is disabled for server deployments.
    It only makes sense for desktop usage where the server and client are on the same machine.
    """
    return jsonify(
        {
            "status": "error",
            "message": "This feature is disabled. It is only available in desktop mode.",
        }
    ), 403


@settings_bp.context_processor
def inject_csrf_token():
    """Inject CSRF token into the template context for all settings routes."""
    return {"csrf_token": generate_csrf}


@settings_bp.route("/fix_corrupted_settings", methods=["POST"])
@login_required
@settings_limit
@with_user_session(include_settings_manager=False)
def fix_corrupted_settings(db_session=None):
    """Fix corrupted settings in the database"""
    try:
        # Track fixed and removed settings
        fixed_settings = []
        removed_duplicate_settings = []
        # First, find and remove duplicate settings with the same key
        # This happens because of errors in settings import/export
        from sqlalchemy import func as sql_func

        # Find keys with duplicates
        duplicate_keys = (
            db_session.query(Setting.key)
            .group_by(Setting.key)
            .having(sql_func.count(Setting.key) > 1)
            .all()
        )
        duplicate_keys = [key[0] for key in duplicate_keys]

        # For each duplicate key, keep the latest updated one and remove others
        for key in duplicate_keys:
            dupe_settings = (
                db_session.query(Setting)
                .filter(Setting.key == key)
                .order_by(Setting.updated_at.desc())
                .all()
            )

            # Keep the first one (most recently updated) and delete the rest
            for i, setting in enumerate(dupe_settings):
                if i > 0:  # Skip the first one (keep it)
                    db_session.delete(setting)
                    removed_duplicate_settings.append(key)

        # Check for settings with corrupted values
        all_settings = db_session.query(Setting).all()
        for setting in all_settings:
            # Check different types of corruption
            is_corrupted = False

            if (
                setting.value is None
                or (
                    isinstance(setting.value, str)
                    and setting.value
                    in [
                        "{",
                        "[",
                        "{}",
                        "[]",
                        "[object Object]",
                        "null",
                        "undefined",
                    ]
                )
                or (isinstance(setting.value, dict) and len(setting.value) == 0)
            ):
                is_corrupted = True

            # Skip if not corrupted
            if not is_corrupted:
                continue

            default_value: Any = None

            # Try to find a matching default setting based on key
            if setting.key.startswith("llm."):
                if setting.key == "llm.model":
                    default_value = "gpt-3.5-turbo"
                elif setting.key == "llm.provider":
                    default_value = "openai"
                elif setting.key == "llm.temperature":
                    default_value = 0.7
                elif setting.key == "llm.max_tokens":
                    default_value = 1024
            elif setting.key.startswith("search."):
                if setting.key == "search.tool":
                    default_value = "auto"
                elif setting.key == "search.max_results":
                    default_value = 10
                elif setting.key == "search.region":
                    default_value = "us"
                elif setting.key == "search.questions_per_iteration":
                    default_value = 3
                elif setting.key == "search.searches_per_section":
                    default_value = 2
                elif setting.key == "search.skip_relevance_filter":
                    default_value = False
                elif setting.key == "search.safe_search":
                    default_value = True
                elif setting.key == "search.search_language":
                    default_value = "English"
            elif setting.key.startswith("report."):
                if setting.key == "report.searches_per_section":
                    default_value = 2
            elif setting.key.startswith("app."):
                if (
                    setting.key == "app.theme"
                    or setting.key == "app.default_theme"
                ):
                    default_value = "dark"
                elif setting.key == "app.enable_notifications" or (
                    setting.key == "app.enable_web"
                    or setting.key == "app.web_interface"
                ):
                    default_value = True
                elif setting.key == "app.host":
                    default_value = "0.0.0.0"
                elif setting.key == "app.port":
                    default_value = 5000
                elif setting.key == "app.debug":
                    default_value = True

            # Update the setting with the default value if found
            if default_value is not None:
                setting.value = default_value
                fixed_settings.append(setting.key)
            else:
                # If no default found but it's a corrupted JSON, set to empty object
                if setting.key.startswith("report."):
                    setting.value = {}
                    fixed_settings.append(setting.key)

        # Commit changes
        if fixed_settings or removed_duplicate_settings:
            db_session.commit()
            logger.info(
                f"Fixed {len(fixed_settings)} corrupted settings: {', '.join(fixed_settings)}"
            )
            if removed_duplicate_settings:
                logger.info(
                    f"Removed {len(removed_duplicate_settings)} duplicate settings"
                )
            invalidate_settings_caches(session["username"])

        # Return success
        return jsonify(
            {
                "status": "success",
                "message": f"Fixed {len(fixed_settings)} corrupted settings, removed {len(removed_duplicate_settings)} duplicates",
                "fixed_settings": fixed_settings,
                "removed_duplicates": removed_duplicate_settings,
            }
        )

    except Exception:
        logger.exception("Error fixing corrupted settings")
        db_session.rollback()
        return (
            jsonify(
                {
                    "status": "error",
                    "message": "An internal error occurred while fixing corrupted settings. Please try again later.",
                }
            ),
            500,
        )


@settings_bp.route("/api/warnings", methods=["GET"])
@login_required
def api_get_warnings():
    """Get current warnings based on settings"""
    try:
        warnings = calculate_warnings()
        return jsonify({"warnings": warnings})
    except Exception:
        logger.exception("Error getting warnings")
        return jsonify({"error": "Failed to retrieve warnings"}), 500


@settings_bp.route("/api/backup-status", methods=["GET"])
@login_required
def api_get_backup_status():
    """Get backup status for the current user."""
    try:
        from ...config.paths import get_user_backup_directory

        username = session.get("username")
        if not username:
            return jsonify({"error": "Not authenticated"}), 401

        from ...utilities.formatting import human_size

        backup_dir = get_user_backup_directory(username)

        # Sort by modification time (not filename) for robustness
        backup_list = []
        total_size = 0
        for b in backup_dir.glob("ldr_backup_*.db"):
            try:
                stat = b.stat()
                total_size += stat.st_size
                backup_list.append(
                    {
                        "filename": b.name,
                        "size_bytes": stat.st_size,
                        "size_human": human_size(stat.st_size),
                        "created_at": datetime.fromtimestamp(
                            stat.st_mtime, tz=timezone.utc
                        ).isoformat(),
                        "_mtime": stat.st_mtime,
                    }
                )
            except FileNotFoundError:
                continue

        # Sort newest first by mtime, then remove internal field
        backup_list.sort(key=lambda x: x["_mtime"], reverse=True)
        for entry in backup_list:
            del entry["_mtime"]

        backup_enabled = _get_setting_from_session("backup.enabled", True)

        return jsonify(
            {
                "enabled": bool(backup_enabled),
                "count": len(backup_list),
                "backups": backup_list,
                "total_size_bytes": total_size,
                "total_size_human": human_size(total_size),
            }
        )

    except Exception:
        logger.exception("Error getting backup status")
        return jsonify({"error": "Failed to retrieve backup status"}), 500


@settings_bp.route("/api/ollama-status", methods=["GET"])
@login_required
def check_ollama_status():
    """Check if Ollama is running and available"""
    try:
        # Get Ollama URL from settings
        raw_base_url = _get_setting_from_session(
            "llm.ollama.url", DEFAULT_OLLAMA_URL
        )
        base_url = (
            normalize_url(raw_base_url) if raw_base_url else DEFAULT_OLLAMA_URL
        )

        response = safe_get(
            f"{base_url}/api/version",
            timeout=2,
            allow_localhost=True,
            allow_private_ips=True,
        )

        if response.status_code == 200:
            return jsonify(
                {
                    "running": True,
                    "version": response.json().get("version", "unknown"),
                }
            )
        return jsonify(
            {
                "running": False,
                "error": f"Ollama returned status code {response.status_code}",
            }
        )
    except requests.exceptions.RequestException:
        logger.exception("Ollama check failed")
        return jsonify(
            {"running": False, "error": "Failed to check search engine status"}
        )


@settings_bp.route("/api/rate-limiting/status", methods=["GET"])
@login_required
def api_get_rate_limiting_status():
    """Get current rate limiting status and statistics"""
    try:
        from ...web_search_engines.rate_limiting import get_tracker

        tracker = get_tracker()

        # Get basic status
        status = {
            "enabled": tracker.enabled,
            "exploration_rate": tracker.exploration_rate,
            "learning_rate": tracker.learning_rate,
            "memory_window": tracker.memory_window,
        }

        # Get engine statistics
        engine_stats = tracker.get_stats()
        engines = []

        for stat in engine_stats:
            (
                engine_type,
                base_wait,
                min_wait,
                max_wait,
                last_updated,
                total_attempts,
                success_rate,
            ) = stat
            engines.append(
                {
                    "engine_type": engine_type,
                    "base_wait_seconds": round(base_wait, 2),
                    "min_wait_seconds": round(min_wait, 2),
                    "max_wait_seconds": round(max_wait, 2),
                    "last_updated": last_updated,
                    "total_attempts": total_attempts,
                    "success_rate": (
                        round(success_rate * 100, 1) if success_rate else 0.0
                    ),
                }
            )

        return jsonify({"status": status, "engines": engines})

    except Exception:
        logger.exception("Error getting rate limiting status")
        return jsonify({"error": "An internal error occurred"}), 500


@settings_bp.route(
    "/api/rate-limiting/engines/<engine_type>/reset", methods=["POST"]
)
@login_required
def api_reset_engine_rate_limiting(engine_type):
    """Reset rate limiting data for a specific engine"""
    try:
        from ...web_search_engines.rate_limiting import get_tracker

        tracker = get_tracker()
        tracker.reset_engine(engine_type)

        return jsonify(
            {"message": f"Rate limiting data reset for {engine_type}"}
        )

    except Exception:
        logger.exception(f"Error resetting rate limiting for {engine_type}")
        return jsonify({"error": "An internal error occurred"}), 500


@settings_bp.route("/api/rate-limiting/cleanup", methods=["POST"])
@login_required
def api_cleanup_rate_limiting():
    """Clean up old rate limiting data.

    Note: not using @require_json_body because the JSON body is optional
    here — the endpoint works with or without a payload (defaults to 30 days).
    """
    try:
        from ...web_search_engines.rate_limiting import get_tracker

        data = request.get_json() if request.is_json else None
        days = data.get("days", 30) if data is not None else 30

        tracker = get_tracker()
        tracker.cleanup_old_data(days)

        return jsonify(
            {"message": f"Cleaned up rate limiting data older than {days} days"}
        )

    except Exception:
        logger.exception("Error cleaning up rate limiting data")
        return jsonify({"error": "An internal error occurred"}), 500


@settings_bp.route("/api/bulk", methods=["GET"])
@login_required
def get_bulk_settings():
    """Get multiple settings at once for performance."""
    try:
        # Get requested settings from query parameters
        requested = request.args.getlist("keys[]")
        if not requested:
            # Default to common settings if none specified
            requested = [
                "llm.provider",
                "llm.model",
                "search.tool",
                "search.iterations",
                "search.questions_per_iteration",
                "search.search_strategy",
                "benchmark.evaluation.provider",
                "benchmark.evaluation.model",
                "benchmark.evaluation.temperature",
                "benchmark.evaluation.endpoint_url",
            ]

        # Fetch all settings at once
        result = {}
        for key in requested:
            try:
                value = _get_setting_from_session(key)
                result[key] = {"value": value, "exists": value is not None}
            except Exception:
                logger.warning(f"Error getting setting {key}")
                result[key] = {
                    "value": None,
                    "exists": False,
                    "error": "Failed to retrieve setting",
                }

        return jsonify({"success": True, "settings": result})

    except Exception:
        logger.exception("Error getting bulk settings")
        return jsonify(
            {"success": False, "error": "An internal error occurred"}
        ), 500


@settings_bp.route("/api/data-location", methods=["GET"])
@login_required
def api_get_data_location():
    """Get information about data storage location and security"""
    try:
        # Get the data directory path
        data_dir = get_data_directory()
        # Get the encrypted databases path
        encrypted_db_path = get_encrypted_database_path()

        # Check if LDR_DATA_DIR environment variable is set
        from local_deep_research.settings.manager import SettingsManager

        settings_manager = SettingsManager()
        custom_data_dir = settings_manager.get_setting("bootstrap.data_dir")

        # Get platform-specific default location info
        platform_info = {
            "Windows": "C:\\Users\\Username\\AppData\\Local\\local-deep-research",
            "macOS": "~/Library/Application Support/local-deep-research",
            "Linux": "~/.local/share/local-deep-research",
        }

        # Current platform
        current_platform = platform.system()
        if current_platform == "Darwin":
            current_platform = "macOS"

        # Get SQLCipher settings from environment
        from ...database.sqlcipher_utils import get_sqlcipher_settings

        # Debug logging
        logger.info(f"db_manager type: {type(db_manager)}")
        logger.info(
            f"db_manager.has_encryption: {getattr(db_manager, 'has_encryption', 'ATTRIBUTE NOT FOUND')}"
        )

        cipher_settings = (
            get_sqlcipher_settings() if db_manager.has_encryption else {}
        )

        return jsonify(
            {
                "data_directory": str(data_dir),
                "database_path": str(encrypted_db_path),
                "encrypted_database_path": str(encrypted_db_path),
                "is_custom": custom_data_dir is not None,
                "custom_env_var": "LDR_DATA_DIR",
                "custom_env_value": custom_data_dir,
                "platform": current_platform,
                "platform_default": platform_info.get(
                    current_platform, str(data_dir)
                ),
                "platform_info": platform_info,
                "security_notice": {
                    "encrypted": db_manager.has_encryption,
                    "warning": "All data including API keys stored in the database are securely encrypted."
                    if db_manager.has_encryption
                    else "All data including API keys stored in the database are currently unencrypted. Please ensure appropriate file system permissions are set.",
                    "recommendation": "Your data is protected with database encryption."
                    if db_manager.has_encryption
                    else "Consider using environment variables for sensitive API keys instead of storing them in the database.",
                },
                "encryption_settings": cipher_settings,
            }
        )

    except Exception:
        logger.exception("Error getting data location information")
        return jsonify({"error": "Failed to retrieve data location"}), 500


@settings_bp.route("/api/notifications/test-url", methods=["POST"])
@login_required
def api_test_notification_url():
    """
    Test a notification service URL.

    This endpoint creates a temporary NotificationService instance to test
    the provided URL. No database session or password is required because:
    - The service URL is provided directly in the request body
    - Test notifications use a temporary Apprise instance
    - No user settings or database queries are performed

    Security note: Rate limiting is not applied here because users need to
    test URLs when configuring notifications. Abuse is mitigated by the
    @login_required decorator and the fact that users can only spam their
    own notification services.
    """
    try:
        from ...notifications.service import NotificationService

        data = request.get_json()
        if not data or "service_url" not in data:
            return jsonify(
                {"success": False, "error": "service_url is required"}
            ), 400

        service_url = data["service_url"]

        # Create notification service instance and test the URL
        # No password/session needed - URL provided directly, no DB access
        notification_service = NotificationService()
        result = notification_service.test_service(service_url)

        # Only return expected fields to prevent information leakage
        safe_response = {
            "success": result.get("success", False),
            "message": result.get("message", ""),
            "error": result.get("error", ""),
        }
        return jsonify(safe_response)

    except Exception:
        logger.exception("Error testing notification URL")
        return jsonify(
            {
                "success": False,
                "error": "Failed to test notification service. Check logs for details.",
            }
        ), 500
