"""
BrowseComp benchmark evaluator.

This module provides a benchmark evaluator implementation for the BrowseComp
benchmark, which tests browsing comprehension capabilities.
"""

from typing import Any, Dict

from loguru import logger

from ..runners import run_browsecomp_benchmark
from .base import BaseBenchmarkEvaluator


class BrowseCompEvaluator(BaseBenchmarkEvaluator):
    """
    Evaluator for the BrowseComp benchmark.

    This evaluator runs the BrowseComp benchmark, which tests a system's ability
    to accurately comprehend and answer questions from web browsing.
    """

    def __init__(self):
        """Initialize the BrowseComp evaluator."""
        super().__init__("browsecomp")

    def evaluate(
        self,
        system_config: Dict[str, Any],
        num_examples: int,
        output_dir: str,
    ) -> Dict[str, Any]:
        """
        Run BrowseComp benchmark and return metrics.

        Args:
            system_config: Search and LLM configuration parameters
            num_examples: Number of benchmark examples to run
            output_dir: Directory to save evaluation results

        Returns:
            Dictionary with metrics including quality_score based on accuracy
        """
        # Create benchmark-specific directory
        benchmark_dir = self._create_subdirectory(output_dir)

        # Log benchmark execution
        logger.info(
            f"Running BrowseComp benchmark with {num_examples} examples"
        )

        try:
            # Run BrowseComp benchmark
            results = run_browsecomp_benchmark(
                num_examples=num_examples,
                output_dir=benchmark_dir,
                search_config=system_config,
                run_evaluation=True,
            )

            # Extract metrics
            metrics = results.get("metrics", {})
            accuracy = metrics.get("accuracy", 0.0)

            # Return evaluation results with quality score
            return {
                "benchmark_type": self.name,
                "accuracy": accuracy,
                "quality_score": accuracy,  # Map accuracy directly to quality score
                "raw_results": results,
                "report_path": results.get("report_path"),
            }

        except Exception as e:
            logger.exception("Error in BrowseComp evaluation")

            # Return error information
            return {
                "benchmark_type": self.name,
                "error": str(e),
                "quality_score": 0.0,
                "accuracy": 0.0,
            }
