"""
API functions for benchmarking.

This module provides functions for running benchmarks programmatically.
"""

from loguru import logger
from pathlib import Path
from typing import Any, Dict, List, Optional

from ..benchmarks import (
    calculate_metrics,
    generate_report,
    run_benchmark,
    run_browsecomp_benchmark,
    run_simpleqa_benchmark,
    run_xbench_deepsearch_benchmark,
)


def evaluate_simpleqa(
    num_examples: int = 100,
    search_iterations: int = 3,
    questions_per_iteration: int = 3,
    search_tool: str = "searxng",
    human_evaluation: bool = False,
    evaluation_model: Optional[str] = None,
    evaluation_provider: Optional[str] = None,
    output_dir: str = "benchmark_results",
) -> Dict[str, Any]:
    """
    Run SimpleQA benchmark evaluation.

    Args:
        num_examples: Number of examples to evaluate
        search_iterations: Number of search iterations per query
        questions_per_iteration: Number of questions per iteration
        search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
        human_evaluation: Whether to use human evaluation
        evaluation_model: Optional custom model for evaluation
        evaluation_provider: Optional custom provider for evaluation
        output_dir: Directory to save results

    Returns:
        Dictionary with benchmark results
    """
    logger.info(f"Starting SimpleQA benchmark with {num_examples} examples")

    # Set up search configuration
    search_config = {
        "iterations": search_iterations,
        "questions_per_iteration": questions_per_iteration,
        "search_tool": search_tool,
    }

    # Set up evaluation configuration if needed
    evaluation_config = None
    if evaluation_model or evaluation_provider:
        evaluation_config = {}
        if evaluation_model:
            evaluation_config["model_name"] = evaluation_model
        if evaluation_provider:
            evaluation_config["provider"] = evaluation_provider

    # Run the benchmark
    results = run_simpleqa_benchmark(
        num_examples=num_examples,
        output_dir=output_dir,
        search_config=search_config,
        evaluation_config=evaluation_config,
        human_evaluation=human_evaluation,
    )

    return results


def evaluate_browsecomp(
    num_examples: int = 100,
    search_iterations: int = 3,
    questions_per_iteration: int = 3,
    search_tool: str = "searxng",
    human_evaluation: bool = False,
    evaluation_model: Optional[str] = None,
    evaluation_provider: Optional[str] = None,
    output_dir: str = "benchmark_results",
) -> Dict[str, Any]:
    """
    Run BrowseComp benchmark evaluation.

    Args:
        num_examples: Number of examples to evaluate
        search_iterations: Number of search iterations per query
        questions_per_iteration: Number of questions per iteration
        search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
        human_evaluation: Whether to use human evaluation
        evaluation_model: Optional custom model for evaluation
        evaluation_provider: Optional custom provider for evaluation
        output_dir: Directory to save results

    Returns:
        Dictionary with benchmark results
    """
    logger.info(f"Starting BrowseComp benchmark with {num_examples} examples")

    # Set up search configuration
    search_config = {
        "iterations": search_iterations,
        "questions_per_iteration": questions_per_iteration,
        "search_tool": search_tool,
    }

    # Set up evaluation configuration if needed
    evaluation_config = None
    if evaluation_model or evaluation_provider:
        evaluation_config = {}
        if evaluation_model:
            evaluation_config["model_name"] = evaluation_model
        if evaluation_provider:
            evaluation_config["provider"] = evaluation_provider

    # Run the benchmark
    results = run_browsecomp_benchmark(
        num_examples=num_examples,
        output_dir=output_dir,
        search_config=search_config,
        evaluation_config=evaluation_config,
        human_evaluation=human_evaluation,
    )

    return results


def evaluate_xbench_deepsearch(
    num_examples: int = 100,
    search_iterations: int = 4,
    questions_per_iteration: int = 3,
    search_tool: str = "searxng",
    human_evaluation: bool = False,
    evaluation_model: Optional[str] = None,
    evaluation_provider: Optional[str] = None,
    output_dir: str = "benchmark_results",
) -> Dict[str, Any]:
    """
    Run xbench-DeepSearch benchmark evaluation.

    Args:
        num_examples: Number of examples to evaluate (default 100 - full dataset)
        search_iterations: Number of search iterations per query
        questions_per_iteration: Number of questions per iteration
        search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
        human_evaluation: Whether to use human evaluation
        evaluation_model: Optional custom model for evaluation
        evaluation_provider: Optional custom provider for evaluation
        output_dir: Directory to save results

    Returns:
        Dictionary with benchmark results
    """
    logger.info(
        f"Starting xbench-DeepSearch benchmark with {num_examples} examples"
    )

    # Set up search configuration
    search_config = {
        "iterations": search_iterations,
        "questions_per_iteration": questions_per_iteration,
        "search_tool": search_tool,
    }

    # Set up evaluation configuration if needed
    evaluation_config = None
    if evaluation_model or evaluation_provider:
        evaluation_config = {}
        if evaluation_model:
            evaluation_config["model_name"] = evaluation_model
        if evaluation_provider:
            evaluation_config["provider"] = evaluation_provider

    # Run the benchmark
    results = run_xbench_deepsearch_benchmark(
        num_examples=num_examples,
        output_dir=output_dir,
        search_config=search_config,
        evaluation_config=evaluation_config,
        human_evaluation=human_evaluation,
    )

    return results


def get_available_benchmarks() -> List[Dict[str, str]]:
    """
    Get information about available benchmarks.

    Returns:
        List of dictionaries with benchmark information
    """
    return [
        {
            "id": "simpleqa",
            "name": "SimpleQA",
            "description": "Benchmark for factual question answering",
            "recommended_examples": 100,
        },
        {
            "id": "browsecomp",
            "name": "BrowseComp",
            "description": "Benchmark for web browsing comprehension",
            "recommended_examples": 100,
        },
        {
            "id": "xbench_deepsearch",
            "name": "xbench-DeepSearch",
            "description": "Benchmark for deep search and investigation queries",
            "recommended_examples": 100,
        },
    ]


def compare_configurations(
    dataset_type: str = "simpleqa",
    num_examples: int = 20,
    configurations: List[Dict[str, Any]] = None,
    output_dir: str = "benchmark_comparisons",
) -> Dict[str, Any]:
    """
    Compare multiple search configurations on the same benchmark.

    Args:
        dataset_type: Type of dataset to use
        num_examples: Number of examples to evaluate
        configurations: List of search configurations to compare
        output_dir: Directory to save results

    Returns:
        Dictionary with comparison results
    """
    if not configurations:
        # Default configurations to compare
        configurations = [
            {
                "name": "Base Config",
                "search_tool": "searxng",
                "iterations": 1,
                "questions_per_iteration": 3,
            },
            {
                "name": "More Iterations",
                "search_tool": "searxng",
                "iterations": 3,
                "questions_per_iteration": 3,
            },
            {
                "name": "More Questions",
                "search_tool": "searxng",
                "iterations": 1,
                "questions_per_iteration": 5,
            },
        ]

    # Create output directory
    import os

    os.makedirs(output_dir, exist_ok=True)

    # Run benchmarks for each configuration
    results = []
    for config in configurations:
        config_name = config.pop("name", f"Config-{len(results)}")

        logger.info(f"Running benchmark with configuration: {config_name}")

        search_config = {
            "iterations": config.pop("iterations", 1),
            "questions_per_iteration": config.pop("questions_per_iteration", 3),
            "search_tool": config.pop("search_tool", "searxng"),
        }

        # Add any remaining config items
        for key, value in config.items():
            search_config[key] = value

        # Run benchmark with this configuration
        benchmark_result = run_benchmark(
            dataset_type=dataset_type,
            num_examples=num_examples,
            output_dir=str(Path(output_dir) / config_name.replace(" ", "_")),
            search_config=search_config,
            run_evaluation=True,
        )

        # Add configuration name to results
        benchmark_result["configuration_name"] = config_name
        benchmark_result["search_config"] = search_config

        results.append(benchmark_result)

    # Generate comparison report
    import time
    from ...security.file_write_verifier import write_file_verified

    timestamp = time.strftime("%Y%m%d_%H%M%S")
    report_file = str(
        Path(output_dir) / f"comparison_{dataset_type}_{timestamp}.md"
    )

    # Build report content
    content = f"# Configuration Comparison - {dataset_type.capitalize()}\n\n"

    # Write summary table
    content += "## Summary\n\n"
    content += "| Configuration | Accuracy | Avg. Time | Examples |\n"
    content += "|---------------|----------|-----------|----------|\n"

    for result in results:
        accuracy = result.get("metrics", {}).get("accuracy", 0)
        avg_time = result.get("metrics", {}).get("average_processing_time", 0)
        examples = result.get("total_examples", 0)

        content += f"| {result['configuration_name']} | {accuracy:.3f} | {avg_time:.2f}s | {examples} |\n"

    content += "\n## Configuration Details\n\n"

    for result in results:
        content += f"### {result['configuration_name']}\n\n"

        config = result.get("search_config", {})
        content += "```\n"
        for key, value in config.items():
            content += f"{key}: {value}\n"
        content += "```\n\n"

    write_file_verified(
        report_file,
        content,
        "benchmark.allow_file_output",
        context="benchmark comparison report",
    )

    logger.info(f"Comparison report saved to {report_file}")

    return {
        "status": "complete",
        "dataset_type": dataset_type,
        "configurations_tested": len(configurations),
        "report_path": report_file,
        "results": results,
    }


# Export the API functions
__all__ = [
    "calculate_metrics",
    "compare_configurations",
    "evaluate_browsecomp",
    "evaluate_simpleqa",
    "evaluate_xbench_deepsearch",
    "generate_report",
    "get_available_benchmarks",
    "run_benchmark",  # For advanced users
]
