import json
import re
import time
from abc import ABC, abstractmethod
from datetime import datetime, UTC
from typing import Any, Dict, List, Optional, Set, Union

from langchain_core.language_models import BaseLLM
from loguru import logger
from tenacity import (
    RetryError,
    retry,
    retry_if_exception_type,
    stop_after_attempt,
)
from tenacity.wait import wait_base

from ..advanced_search_system.filters.base_filter import BaseFilter
from ..config.thread_settings import get_setting_from_snapshot
from ..utilities.json_utils import extract_json, get_llm_response_text
from ..utilities.thread_context import clear_search_context, set_search_context

from .rate_limiting import RateLimitError, get_tracker


class AdaptiveWait(wait_base):
    """Custom wait strategy that uses adaptive rate limiting."""

    def __init__(self, get_wait_func):
        self.get_wait_func = get_wait_func

    def __call__(self, retry_state):
        return self.get_wait_func()


class BaseSearchEngine(ABC):
    """
    Abstract base class for search engines with two-phase retrieval capability.
    Handles common parameters and implements the two-phase search approach.
    """

    # Class attribute to indicate if this engine searches public internet sources
    # Should be overridden by subclasses - defaults to False for safety
    is_public = False

    # Class attribute to indicate if this is a generic search engine (vs specialized)
    # Generic engines are general web search (Google, Bing, etc) vs specialized (arXiv, PubMed)
    is_generic = False

    # Class attribute to indicate if this is a scientific/academic search engine
    # Scientific engines include arXiv, PubMed, Semantic Scholar, etc.
    is_scientific = False

    # Class attribute to indicate if this is a local RAG/document search engine
    # Local engines search private document collections stored locally
    is_local = False

    # Class attribute to indicate if this is a news search engine
    # News engines specialize in news articles and current events
    is_news = False

    # Class attribute to indicate if this is a code search engine
    # Code engines specialize in searching code repositories
    is_code = False

    # Class attribute to indicate if this is a book/literature search engine
    # Book engines search libraries and literary archives
    is_books = False

    # Class attribute for rate limit detection patterns
    # Subclasses can override to add engine-specific patterns
    rate_limit_patterns: Set[str] = {
        "rate limit",
        "rate_limit",
        "ratelimit",
        "too many requests",
        "throttl",
        "quota exceeded",
        "quota_exceeded",
        "limit exceeded",
        "request limit",
        "api limit",
        "usage limit",
    }

    @staticmethod
    def _ensure_list(value, *, default=None):
        """Normalize a value that should be a list.

        Handles JSON-encoded strings, comma-separated strings, and
        already-parsed lists.  Returns *default* (empty list when not
        supplied) for ``None`` or empty/unparseable input.
        """
        if default is None:
            default = []
        if value is None:
            return default
        if isinstance(value, list):
            return value
        if isinstance(value, str):
            stripped = value.strip()
            if not stripped:
                return default
            if stripped.startswith("["):
                try:
                    parsed = json.loads(stripped)
                    if isinstance(parsed, list):
                        return [str(item) for item in parsed]
                except (json.JSONDecodeError, ValueError, RecursionError):
                    pass
            return [
                item.strip() for item in stripped.split(",") if item.strip()
            ]
        return default

    @classmethod
    def _load_engine_class(cls, name: str, config: Dict[str, Any]):
        """
        Helper method to load an engine class dynamically.

        Args:
            name: Engine name
            config: Engine configuration dict with module_path and class_name

        Returns:
            Tuple of (success: bool, engine_class or None, error_msg or None)
        """
        from ..security.module_whitelist import (
            ModuleNotAllowedError,
            get_safe_module_class,
        )

        try:
            module_path = config.get("module_path")
            class_name = config.get("class_name")

            if not module_path or not class_name:
                return (
                    False,
                    None,
                    f"Missing module_path or class_name for {name}",
                )

            # Use whitelist-validated safe import
            engine_class = get_safe_module_class(module_path, class_name)

            return True, engine_class, None

        except ModuleNotAllowedError as e:
            return (
                False,
                None,
                f"Security error loading engine class for {name}: {e}",
            )
        except Exception as e:
            return False, None, f"Could not load engine class for {name}: {e}"

    @classmethod
    def _check_api_key_availability(
        cls, name: str, config: Dict[str, Any]
    ) -> bool:
        """
        Helper method to check if an engine's API key is available and valid.

        Args:
            name: Engine name
            config: Engine configuration dict

        Returns:
            True if API key is not required or is available and valid
        """
        from loguru import logger

        if not config.get("requires_api_key", False):
            return True

        api_key = config.get("api_key", "").strip()

        # Check for common placeholder values
        if (
            not api_key
            or api_key in ["", "None", "PLACEHOLDER", "YOUR_API_KEY_HERE"]
            or api_key.endswith(
                "_API_KEY"
            )  # Default placeholders like BRAVE_API_KEY
            or api_key.startswith("YOUR_")
            or api_key == "null"
        ):
            logger.debug(
                f"Skipping {name} - requires API key but none configured"
            )
            return False

        return True

    def __init__(
        self,
        llm: Optional[BaseLLM] = None,
        max_filtered_results: Optional[int] = None,
        max_results: Optional[int] = 10,  # Default value if not provided
        preview_filters: List[BaseFilter] | None = None,
        content_filters: List[BaseFilter] | None = None,
        search_snippets_only: bool = True,  # New parameter with default
        include_full_content: bool = False,
        settings_snapshot: Optional[Dict[str, Any]] = None,
        programmatic_mode: bool = False,
        **kwargs,
    ):
        """
        Initialize the search engine with common parameters.

        Args:
            llm: Optional language model for relevance filtering
            max_filtered_results: Maximum number of results to keep after filtering
            max_results: Maximum number of search results to return
            preview_filters: Filters that will be applied to all previews
                produced by the search engine, before relevancy checks.
            content_filters: Filters that will be applied to the full content
                produced by the search engine, after relevancy checks.
            search_snippets_only: Whether to return only snippets or full content
            include_full_content: Whether to use FullSearchResults for full webpage content
            settings_snapshot: Settings snapshot for configuration
            programmatic_mode: If True, disables database operations and uses memory-only tracking
            **kwargs: Additional engine-specific parameters
        """
        if max_filtered_results is None:
            max_filtered_results = 5
        if max_results is None:
            max_results = 10
        self._preview_filters: List[BaseFilter] = (
            preview_filters if preview_filters is not None else []
        )
        self._content_filters: List[BaseFilter] = (
            content_filters if content_filters is not None else []
        )

        self.llm = llm  # LLM for relevance filtering
        self._max_filtered_results = int(
            max_filtered_results
        )  # Ensure it's an integer
        self._max_results = max(
            1, int(max_results)
        )  # Ensure it's a positive integer
        self.search_snippets_only = search_snippets_only  # Store the setting
        self.include_full_content = include_full_content
        self.settings_snapshot = (
            settings_snapshot or {}
        )  # Store settings snapshot
        self.programmatic_mode = programmatic_mode

        # Rate limiting attributes
        self.engine_type = self.__class__.__name__
        # Create a tracker with our settings if in programmatic mode
        if self.programmatic_mode:
            from .rate_limiting.tracker import AdaptiveRateLimitTracker

            self.rate_tracker = AdaptiveRateLimitTracker(
                settings_snapshot=self.settings_snapshot,
                programmatic_mode=self.programmatic_mode,
            )
        else:
            self.rate_tracker = get_tracker()
        self._last_wait_time = (
            0.0  # Default to 0 for successful searches without rate limiting
        )
        self._last_results_count = 0

    @property
    def max_filtered_results(self) -> int:
        """Get the maximum number of filtered results."""
        return self._max_filtered_results

    @max_filtered_results.setter
    def max_filtered_results(self, value: int) -> None:
        """Set the maximum number of filtered results."""
        if value is None:
            value = 5
            logger.warning("Setting max_filtered_results to 5")
        self._max_filtered_results = int(value)

    @property
    def max_results(self) -> int:
        """Get the maximum number of search results."""
        return self._max_results

    @max_results.setter
    def max_results(self, value: int) -> None:
        """Set the maximum number of search results."""
        if value is None:
            value = 10
        self._max_results = max(1, int(value))

    def _get_adaptive_wait(self) -> float:
        """Get adaptive wait time from tracker."""
        wait_time = self.rate_tracker.get_wait_time(self.engine_type)
        self._last_wait_time = wait_time
        logger.debug(
            f"{self.engine_type} waiting {wait_time:.2f}s before retry"
        )
        return wait_time

    def _record_retry_outcome(self, retry_state) -> None:
        """Record outcome after retry completes."""
        success = (
            not retry_state.outcome.failed if retry_state.outcome else False
        )
        self.rate_tracker.record_outcome(
            self.engine_type,
            self._last_wait_time or 0,
            success,
            retry_state.attempt_number,
            error_type="RateLimitError" if not success else None,
            search_result_count=self._last_results_count if success else 0,
        )

    def run(
        self, query: str, research_context: Dict[str, Any] | None = None
    ) -> List[Dict[str, Any]]:
        """
        Run the search engine with a given query, retrieving and filtering results.
        This implements a two-phase retrieval approach:
        1. Get preview information for many results
        2. Filter the previews for relevance
        3. Get full content for only the relevant results

        Args:
            query: The search query
            research_context: Context from previous research to use.

        Returns:
            List of search results with full content (if available)
        """
        logger.info(f"---Execute a search using {self.__class__.__name__}---")

        # Track search call for metrics (if available and not in programmatic mode)
        should_record_metrics = False
        context_was_set = False
        if not self.programmatic_mode:
            from ..metrics.search_tracker import SearchTracker

            should_record_metrics = True

            # For thread-safe context propagation: if we have research_context parameter, use it
            # Otherwise, try to inherit from current thread context (normal case)
            # This allows strategies running in threads to explicitly pass context when needed
            if research_context:
                # Explicit context provided - use it and set it for this thread
                set_search_context(research_context)
                context_was_set = True

        engine_name = self.__class__.__name__.replace(
            "SearchEngine", ""
        ).lower()
        start_time = time.time()

        success = True
        error_message = None
        results_count = 0

        # Define the core search function with retry logic
        if self.rate_tracker.enabled:
            # Rate limiting enabled - use retry with adaptive wait
            @retry(
                stop=stop_after_attempt(3),
                wait=AdaptiveWait(lambda: self._get_adaptive_wait()),
                retry=retry_if_exception_type((RateLimitError,)),
                after=self._record_retry_outcome,
                reraise=True,
            )
            def _run_with_retry():
                nonlocal success, error_message, results_count
                return _execute_search()
        else:
            # Rate limiting disabled - run without retry
            def _run_with_retry():
                nonlocal success, error_message, results_count
                return _execute_search()

        def _execute_search():
            nonlocal success, error_message, results_count

            try:
                # Step 1: Get preview information for items
                previews = self._get_previews(query)
                if not previews:
                    logger.info(
                        f"Search engine {self.__class__.__name__} returned no preview results for query: {query}"
                    )
                    results_count = 0
                    return []

                for preview_filter in self._preview_filters:
                    previews = preview_filter.filter_results(previews, query)

                # Step 2: Filter previews for relevance with LLM
                # Check if LLM relevance filtering should be enabled
                enable_llm_filter = getattr(
                    self, "enable_llm_relevance_filter", False
                )

                logger.info(
                    f"BaseSearchEngine: Relevance filter check - enable_llm_relevance_filter={enable_llm_filter}, has_llm={self.llm is not None}, engine_type={type(self).__name__}"
                )

                if enable_llm_filter and self.llm:
                    logger.info(
                        f"Applying LLM relevance filter to {len(previews)} previews"
                    )
                    filtered_items = self._filter_for_relevance(previews, query)
                    logger.info(
                        f"LLM filter kept {len(filtered_items)} of {len(previews)} results"
                    )
                else:
                    filtered_items = previews
                    if not enable_llm_filter:
                        logger.info(
                            f"LLM relevance filtering disabled (enable_llm_relevance_filter={enable_llm_filter}) - returning all {len(previews)} previews"
                        )
                    elif not self.llm:
                        logger.info(
                            f"No LLM available for relevance filtering - returning all {len(previews)} previews"
                        )

                # Step 3: Get full content for filtered items
                if self.search_snippets_only:
                    logger.info("Returning snippet-only results as per config")
                    results = filtered_items
                else:
                    results = self._get_full_content(filtered_items)

                for content_filter in self._content_filters:
                    results = content_filter.filter_results(results, query)

                results_count = len(results)
                self._last_results_count = results_count

                # Record success if we get here and rate limiting is enabled
                if self.rate_tracker.enabled:
                    logger.info(
                        f"Recording successful search for {self.engine_type}: wait_time={self._last_wait_time}s, results={results_count}"
                    )
                    self.rate_tracker.record_outcome(
                        self.engine_type,
                        self._last_wait_time,
                        success=True,
                        retry_count=1,  # First attempt succeeded
                        search_result_count=results_count,
                    )
                else:
                    logger.info(
                        f"Rate limiting disabled, not recording search for {self.engine_type}"
                    )

                return results

            except RateLimitError:
                # Only re-raise if rate limiting is enabled
                if self.rate_tracker.enabled:
                    raise
                # If rate limiting is disabled, treat as regular error
                success = False
                error_message = "Rate limit hit but rate limiting disabled"
                logger.warning(
                    f"Rate limit hit on {self.__class__.__name__} but rate limiting is disabled"
                )
                results_count = 0
                return []
            except Exception as e:
                # Other errors - don't retry
                success = False
                error_message = str(e)
                logger.exception(
                    f"Search engine {self.__class__.__name__} failed"
                )
                results_count = 0
                return []

        try:
            return _run_with_retry()  # type: ignore[no-any-return]
        except RetryError as e:
            # All retries exhausted
            success = False
            error_message = f"Rate limited after all retries: {e}"
            logger.exception(
                f"{self.__class__.__name__} failed after all retries"
            )
            return []
        except Exception as e:
            success = False
            error_message = str(e)
            logger.exception(f"Search engine {self.__class__.__name__} error")
            return []
        finally:
            try:
                # Record search metrics BEFORE clearing context (record_search needs it)
                if should_record_metrics:
                    response_time_ms = int((time.time() - start_time) * 1000)
                    SearchTracker.record_search(
                        engine_name=engine_name,
                        query=query,
                        results_count=results_count,
                        response_time_ms=response_time_ms,
                        success=success,
                        error_message=error_message,
                    )
            finally:
                # Clean up temporary search result storage
                for attr in self._temp_attributes():
                    if hasattr(self, attr):
                        delattr(self, attr)
                # ALWAYS clean up search context, even if recording fails
                if context_was_set:
                    clear_search_context()

    def invoke(self, query: str) -> List[Dict[str, Any]]:
        """Compatibility method for LangChain tools"""
        return self.run(query)

    def _filter_for_relevance(
        self, previews: List[Dict[str, Any]], query: str
    ) -> List[Dict[str, Any]]:
        """
        Filter search results by relevance to the query using the LLM.

        Args:
            previews: List of preview dictionaries
            query: The original search query

        Returns:
            Filtered list of preview dictionaries
        """
        # If no LLM or too few previews, return all
        if not self.llm or len(previews) <= 1:
            return previews

        # Log the number of previews we're processing
        logger.info(f"Filtering {len(previews)} previews for relevance")

        # Create a simple context for LLM
        preview_context = []
        indices_used = []
        for i, preview in enumerate(previews):
            title = preview.get("title", "Untitled").strip()
            snippet = preview.get("snippet", "").strip()
            url = preview.get("url", "").strip()

            # Clean up snippet if too long
            if len(snippet) > 300:
                snippet = snippet[:300] + "..."

            preview_context.append(
                f"[{i}] Title: {title}\nURL: {url}\nSnippet: {snippet}"
            )
            indices_used.append(i)

        # Log the indices we're presenting to the LLM
        logger.info(
            f"Created preview context with indices 0-{len(previews) - 1}"
        )
        logger.info(
            f"First 5 indices in prompt: {indices_used[:5]}, Last 5: {indices_used[-5:] if len(indices_used) > 5 else 'N/A'}"
        )

        # Join all previews with clear separation
        preview_text = "\n\n".join(preview_context)

        # Log a sample of what we're sending to the LLM
        logger.debug(
            f"First preview in prompt: {preview_context[0] if preview_context else 'None'}"
        )
        if len(preview_context) > 1:
            logger.debug(f"Last preview in prompt: {preview_context[-1]}")

        # Set a reasonable limit on context length
        current_date = datetime.now(UTC).strftime("%Y-%m-%d")
        prompt = f"""Analyze these search results and select the most relevant ones for the query.

Query: "{query}"
Current date: {current_date}
Total results: {len(previews)}

Criteria for selection (in order of importance):
1. Direct relevance - MUST directly address the specific query topic, not just mention keywords
2. Quality - from reputable sources with substantive information about the query
3. Recency - prefer recent information when relevant

Search results to evaluate:
{preview_text}

Return a JSON array of indices (0-based) for results that are highly relevant to the query.
Valid indices are 0 to {len(previews) - 1}.
Only include results that directly help answer the specific question asked.
Be selective - it's better to return fewer high-quality results than many mediocre ones.
Maximum results to return: {self.max_filtered_results}
Example response: [4, 0, 2, 7]

Respond with ONLY the JSON array, no other text."""

        try:
            # Get LLM's evaluation
            response = self.llm.invoke(prompt)

            # Log the raw response for debugging
            logger.info(f"Raw LLM response for relevance filtering: {response}")

            response_text = get_llm_response_text(response)
            logger.debug(f"Cleaned response text: {response_text}")

            ranked_indices = extract_json(response_text, expected_type=list)

            if ranked_indices is not None:
                logger.info(f"LLM returned indices: {ranked_indices}")

                # Validate that ranked_indices is a list of integers
                if not isinstance(ranked_indices, list):
                    logger.warning(
                        "LLM response is not a list, returning empty results"
                    )
                    return []

                if not all(isinstance(idx, int) for idx in ranked_indices):
                    logger.warning(
                        "LLM response contains non-integer indices, returning empty results"
                    )
                    return []

                # Log analysis of the indices
                max_index = max(ranked_indices) if ranked_indices else -1
                min_index = min(ranked_indices) if ranked_indices else -1
                logger.info(
                    f"Index analysis: min={min_index}, max={max_index}, "
                    f"valid_range=0-{len(previews) - 1}, count={len(ranked_indices)}"
                )

                # Return the results in ranked order
                ranked_results = []
                out_of_range = []
                for idx in ranked_indices:
                    if 0 <= idx < len(previews):
                        ranked_results.append(previews[idx])
                    else:
                        out_of_range.append(idx)
                        logger.warning(
                            f"Index {idx} out of range (valid: 0-{len(previews) - 1}), skipping"
                        )

                if out_of_range:
                    logger.error(
                        f"Out of range indices: {out_of_range}. "
                        f"Total previews: {len(previews)}, "
                        f"All returned indices: {ranked_indices}"
                    )

                # Limit to max_filtered_results if specified
                if (
                    self.max_filtered_results
                    and len(ranked_results) > self.max_filtered_results
                ):
                    logger.info(
                        f"Limiting filtered results to top {self.max_filtered_results}"
                    )
                    return ranked_results[: self.max_filtered_results]

                return ranked_results
            logger.warning(
                "Could not find JSON array in response, returning original previews"
            )
            logger.debug(f"Response text without JSON array: {response_text}")
            return previews[: min(5, len(previews))]

        except Exception:
            logger.exception("Relevance filtering error")
            # Fall back to returning top results on error
            return previews[: min(5, len(previews))]

    # =========================================================================
    # Shared Helper Methods for Subclasses
    # =========================================================================

    @staticmethod
    def _is_valid_api_key(api_key: Optional[str]) -> bool:
        """
        Check if an API key is valid (not a placeholder value).

        Args:
            api_key: The API key to validate

        Returns:
            True if the key appears to be a real API key, False if it's a placeholder

        Example:
            >>> BaseSearchEngine._is_valid_api_key("sk-abc123")
            True
            >>> BaseSearchEngine._is_valid_api_key("YOUR_API_KEY_HERE")
            False
        """
        if not api_key:
            return False

        api_key = api_key.strip()

        # Empty or whitespace-only
        if not api_key:
            return False

        # Common placeholder values
        placeholders = {
            "",
            "None",
            "null",
            "PLACEHOLDER",
            "YOUR_API_KEY_HERE",
            "YOUR_API_KEY",
            "API_KEY",
            "your_api_key",
            "your-api-key",
        }

        if api_key in placeholders:
            return False

        # Patterns that indicate placeholders
        if api_key.endswith("_API_KEY"):
            return False
        if api_key.startswith("YOUR_"):
            return False
        if api_key.startswith("<") and api_key.endswith(">"):
            return False
        if api_key.startswith("${") and api_key.endswith("}"):
            return False

        return True

    def _resolve_api_key(
        self,
        api_key: Optional[str],
        setting_key: str,
        engine_name: str = "search engine",
        settings_snapshot: Optional[Dict[str, Any]] = None,
    ) -> str:
        """
        Resolve an API key from multiple sources with priority order.

        Environment variables are handled automatically by SettingsManager
        when building the settings snapshot, so they don't need to be
        checked separately here.

        Priority order:
        1. Direct parameter (api_key argument)
        2. Settings snapshot (via setting_key)

        Args:
            api_key: API key passed directly as parameter
            setting_key: Key to look up in settings snapshot (e.g., "search.brave_api_key")
            engine_name: Human-readable engine name for error messages
            settings_snapshot: Optional settings snapshot dict (uses self.settings_snapshot if not provided)

        Returns:
            The resolved API key string

        Raises:
            ValueError: If no valid API key is found from any source

        Example:
            >>> engine._resolve_api_key(
            ...     api_key=None,
            ...     setting_key="search.brave_api_key",
            ...     engine_name="Brave Search"
            ... )
            "sk-abc123..."
        """
        # Use instance settings snapshot if not provided
        if settings_snapshot is None:
            settings_snapshot = self.settings_snapshot

        # Priority 1: Direct parameter
        if self._is_valid_api_key(api_key) and api_key is not None:
            return api_key.strip()

        # Priority 2: Settings snapshot (includes env var overrides via SettingsManager)
        if settings_snapshot:
            settings_value = get_setting_from_snapshot(
                setting_key,
                default=None,
                settings_snapshot=settings_snapshot,
            )
            if self._is_valid_api_key(settings_value):
                return settings_value.strip() if settings_value else ""

        # No valid API key found
        masked_key = self._mask_api_key(str(api_key)) if api_key else "None"
        raise ValueError(
            f"No valid API key found for {engine_name}. "
            f"Checked: direct parameter ({masked_key}), "
            f"settings key '{setting_key}'. "
            f"Please provide a valid API key."
        )

    def _is_rate_limit_error(
        self,
        error: Union[Exception, str, int],
        additional_patterns: Optional[Set[str]] = None,
    ) -> bool:
        """
        Detect if an error is a rate limit error.

        Checks multiple sources for rate limit indicators:
        - HTTP status code 429
        - HTTPError response objects
        - Error messages containing rate limit phrases

        Args:
            error: The error to check (Exception, string, or HTTP status code)
            additional_patterns: Optional set of additional patterns to match

        Returns:
            True if the error appears to be a rate limit error

        Example:
            >>> engine._is_rate_limit_error(429)
            True
            >>> engine._is_rate_limit_error("Rate limit exceeded")
            True
            >>> engine._is_rate_limit_error(ValueError("Invalid input"))
            False
        """
        # Combine default and additional patterns
        patterns = self.rate_limit_patterns.copy()
        if additional_patterns:
            patterns.update(additional_patterns)

        # Check integer status code directly
        if isinstance(error, int):
            return error == 429

        # Convert to string for pattern matching
        error_str = ""
        status_code = None

        if isinstance(error, str):
            error_str = error
        elif isinstance(error, Exception):
            error_str = str(error)

            # Check for HTTP status code in common HTTP error types
            if hasattr(error, "status_code"):
                status_code = error.status_code
            elif hasattr(error, "response"):
                response = error.response
                if hasattr(response, "status_code"):
                    status_code = response.status_code

        # Check status code first
        if status_code == 429:
            return True

        # Case-insensitive pattern matching
        error_lower = error_str.lower()
        for pattern in patterns:
            if pattern.lower() in error_lower:
                return True

        return False

    def _raise_if_rate_limit(
        self,
        error: Union[Exception, str, int],
        additional_patterns: Optional[Set[str]] = None,
    ) -> None:
        """
        Raise RateLimitError if the given error is a rate limit error.

        Convenience method that combines _is_rate_limit_error check with
        raising RateLimitError.

        Args:
            error: The error to check
            additional_patterns: Optional set of additional patterns to match

        Raises:
            RateLimitError: If the error is detected as a rate limit error

        Example:
            >>> try:
            ...     response = make_api_call()
            ... except Exception as e:
            ...     engine._raise_if_rate_limit(e)
        """
        if self._is_rate_limit_error(error, additional_patterns):
            error_msg = str(error) if not isinstance(error, str) else error
            raise RateLimitError(
                f"Rate limit detected: {self._sanitize_error_message(error_msg)}"
            )

    def _extract_full_result(self, item: Dict[str, Any]) -> Dict[str, Any]:
        """
        Extract the full result from an item that may contain a _full_result key.

        This is a helper for the default _get_full_content implementation.
        It extracts data from the _full_result key if present, otherwise uses
        the item directly, and removes the internal _full_result key.

        Args:
            item: A search result item that may contain a _full_result key

        Returns:
            A dictionary with the full result data, without the _full_result key

        Example:
            >>> engine._extract_full_result({"title": "A", "_full_result": {"title": "A", "content": "Full"}})
            {"title": "A", "content": "Full"}
        """
        source = item.get("_full_result")
        if source is None:
            source = item
        return {k: v for k, v in source.items() if k != "_full_result"}

    def _get_full_content(
        self, relevant_items: List[Dict[str, Any]]
    ) -> List[Dict[str, Any]]:
        """
        Get full content for the relevant items.

        Default implementation extracts data from _full_result keys if present.
        Subclasses can override this method to fetch additional content from
        external sources (e.g., web scraping, API calls).

        Args:
            relevant_items: List of relevant preview dictionaries

        Returns:
            List of result dictionaries with full content

        Example:
            >>> engine._get_full_content([
            ...     {"title": "A", "_full_result": {"title": "A", "content": "Full A"}},
            ...     {"title": "B"}
            ... ])
            [{"title": "A", "content": "Full A"}, {"title": "B"}]
        """
        if not relevant_items:
            return []
        return [self._extract_full_result(item) for item in relevant_items]

    def _init_full_search(
        self,
        web_search=None,
        language="en",
        max_results=10,
        region=None,
        time_period=None,
        safe_search=None,
    ):
        """Initialize FullSearchResults if include_full_content is True.

        Call this at the end of your __init__ after setting up your search wrapper.

        Args:
            web_search: The search wrapper/engine to pass to FullSearchResults
            language: Language for search results
            max_results: Maximum number of results
            region: Region/country code for results
            time_period: Time period filter
            safe_search: Safe search setting (string value for FullSearchResults)
        """
        if self.include_full_content and self.llm:
            try:
                from .engines.full_search import FullSearchResults

                self.full_search = FullSearchResults(
                    llm=self.llm,
                    web_search=web_search,
                    language=language,
                    max_results=max_results,
                    region=region,
                    time=time_period,
                    safesearch=safe_search,
                )
            except ImportError:
                logger.warning(
                    "FullSearchResults not available. "
                    "Full content retrieval disabled."
                )
                self.include_full_content = False

    def _temp_attributes(self):
        """Return list of temporary attribute names to clean up after run().

        Override in subclasses that store additional temporary data.
        """
        return ["_search_results"]

    def _sanitize_error_message(self, message: str) -> str:
        """
        Remove/mask API keys, tokens, and secrets from error messages.

        Uses pattern matching for common credential formats.

        Args:
            message: The error message to sanitize

        Returns:
            Sanitized message with sensitive data redacted

        Example:
            >>> engine._sanitize_error_message("Error with key sk-abc123xyz")
            "Error with key [REDACTED]"
        """
        if not message:
            return message

        sanitized = message

        # Additional regex patterns for common credential formats
        patterns = [
            # Bearer tokens
            (r"Bearer\s+[A-Za-z0-9\-._~+/]+=*", "Bearer [REDACTED]"),
            # API keys in URLs
            (
                r"([?&])(api_key|apikey|key|token|secret)=([A-Za-z0-9\-._~+/]+)",
                r"\1\2=[REDACTED]",
            ),
            # URL credentials (user:pass@host)
            (r"(https?://)([^:\s]+):([^@\s]+)@", r"\1[REDACTED]:[REDACTED]@"),
            # Common API key patterns (sk-*, pk-*, etc.)
            (r"\b(sk-[A-Za-z0-9]{20,})\b", "[REDACTED_KEY]"),
            (r"\b(pk-[A-Za-z0-9]{20,})\b", "[REDACTED_KEY]"),
        ]

        for pattern, replacement in patterns:
            sanitized = re.sub(pattern, replacement, sanitized)

        return sanitized

    def _mask_api_key(self, api_key: str, visible_chars: int = 4) -> str:
        """
        Mask an API key for safe logging, showing only first and last characters.

        Args:
            api_key: The API key to mask
            visible_chars: Number of characters to show at start and end

        Returns:
            Masked API key in format "sk-1...nop" or "***" for short keys

        Example:
            >>> engine._mask_api_key("sk-abcdefghijklmnop123456")
            "sk-a...3456"
            >>> engine._mask_api_key("short")
            "***"
        """
        if not api_key:
            return "***"

        api_key = str(api_key).strip()

        if len(api_key) <= visible_chars * 2:
            return "***"

        return f"{api_key[:visible_chars]}...{api_key[-visible_chars:]}"

    @abstractmethod
    def _get_previews(self, query: str) -> List[Dict[str, Any]]:
        """
        Get preview information (titles, summaries) for initial search results.

        Args:
            query: The search query

        Returns:
            List of preview dictionaries with at least 'id', 'title', and 'snippet' keys
        """
        pass

    def close(self) -> None:
        """
        Close any resources held by this search engine.

        Subclasses with HTTP sessions or other resources should override this.
        The base implementation safely closes any 'session' attribute if present
        and closes content filters that hold resources.
        """
        from ..utilities.resource_utils import safe_close

        if hasattr(self, "session") and self.session is not None:
            safe_close(self.session, "HTTP session")
        if hasattr(self, "_content_filters"):
            for content_filter in self._content_filters:
                safe_close(content_filter, "content filter")

    def __enter__(self):
        """Support context manager usage."""
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        """Cleanup on context exit."""
        self.close()
        return False
