"""
API functions for benchmarking.

This module provides functions for running benchmarks programmatically.
"""

import logging
import os
from typing import Any, Dict, List, Optional

from ..benchmarks import (
    calculate_metrics,
    generate_report,
    run_benchmark,
    run_browsecomp_benchmark,
    run_simpleqa_benchmark,
)

logger = logging.getLogger(__name__)


def evaluate_simpleqa(
    num_examples: int = 100,
    search_iterations: int = 3,
    questions_per_iteration: int = 3,
    search_tool: str = "searxng",
    human_evaluation: bool = False,
    evaluation_model: Optional[str] = None,
    evaluation_provider: Optional[str] = None,
    output_dir: str = "benchmark_results",
    search_model: Optional[str] = None,
    search_provider: Optional[str] = None,
    endpoint_url: Optional[str] = None,
    search_strategy: str = "source_based",
) -> Dict[str, Any]:
    """
    Run SimpleQA benchmark evaluation.

    Args:
        num_examples: Number of examples to evaluate
        search_iterations: Number of search iterations per query
        questions_per_iteration: Number of questions per iteration
        search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
        human_evaluation: Whether to use human evaluation
        evaluation_model: Optional custom model for evaluation
        evaluation_provider: Optional custom provider for evaluation
        output_dir: Directory to save results
        search_model: Optional model to use for the search system
        search_provider: Optional provider to use for the search system
        endpoint_url: Optional endpoint URL for OpenRouter or other API services
        search_strategy: Search strategy to use (default: 'source_based')

    Returns:
        Dictionary with benchmark results
    """
    logger.info(f"Starting SimpleQA benchmark with {num_examples} examples")

    # Set up search configuration
    search_config = {
        "iterations": search_iterations,
        "questions_per_iteration": questions_per_iteration,
        "search_tool": search_tool,
        "search_strategy": search_strategy,
    }

    # Add model configurations if provided
    if search_model:
        search_config["model_name"] = search_model
    if search_provider:
        search_config["provider"] = search_provider
    if endpoint_url:
        search_config["openai_endpoint_url"] = endpoint_url

    # Check environment variables for additional configuration
    if env_model := os.environ.get("LDR_SEARCH_MODEL"):
        search_config["model_name"] = env_model
    if env_provider := os.environ.get("LDR_SEARCH_PROVIDER"):
        search_config["provider"] = env_provider
    if env_url := os.environ.get("LDR_ENDPOINT_URL"):
        search_config["openai_endpoint_url"] = env_url

    # Set up evaluation configuration if needed
    evaluation_config = None
    if evaluation_model or evaluation_provider:
        evaluation_config = {
            "temperature": 0  # Always use zero temperature for evaluation
        }
        if evaluation_model:
            evaluation_config["model_name"] = evaluation_model
        if evaluation_provider:
            evaluation_config["provider"] = evaluation_provider
            # Add OpenRouter URL if using openai_endpoint
            if evaluation_provider == "openai_endpoint":
                evaluation_config["openai_endpoint_url"] = (
                    "https://openrouter.ai/api/v1"
                )

    # Run the benchmark
    results = run_simpleqa_benchmark(
        num_examples=num_examples,
        output_dir=output_dir,
        search_config=search_config,
        evaluation_config=evaluation_config,
        human_evaluation=human_evaluation,
    )

    return results


def evaluate_browsecomp(
    num_examples: int = 100,
    search_iterations: int = 3,
    questions_per_iteration: int = 3,
    search_tool: str = "searxng",
    human_evaluation: bool = False,
    evaluation_model: Optional[str] = None,
    evaluation_provider: Optional[str] = None,
    output_dir: str = "benchmark_results",
    search_model: Optional[str] = None,
    search_provider: Optional[str] = None,
    endpoint_url: Optional[str] = None,
    search_strategy: str = "source_based",
) -> Dict[str, Any]:
    """
    Run BrowseComp benchmark evaluation.

    Args:
        num_examples: Number of examples to evaluate
        search_iterations: Number of search iterations per query
        questions_per_iteration: Number of questions per iteration
        search_tool: Search engine to use (e.g., 'searxng', 'wikipedia')
        human_evaluation: Whether to use human evaluation
        evaluation_model: Optional custom model for evaluation
        evaluation_provider: Optional custom provider for evaluation
        output_dir: Directory to save results
        search_model: Optional model to use for the search system
        search_provider: Optional provider to use for the search system
        endpoint_url: Optional endpoint URL for OpenRouter or other API services
        search_strategy: Search strategy to use (default: 'source_based')

    Returns:
        Dictionary with benchmark results
    """
    logger.info(f"Starting BrowseComp benchmark with {num_examples} examples")

    # Set up search configuration
    search_config = {
        "iterations": search_iterations,
        "questions_per_iteration": questions_per_iteration,
        "search_tool": search_tool,
        "search_strategy": search_strategy,
    }

    # Add model configurations if provided
    if search_model:
        search_config["model_name"] = search_model
    if search_provider:
        search_config["provider"] = search_provider
    if endpoint_url:
        search_config["openai_endpoint_url"] = endpoint_url

    # Check environment variables for additional configuration
    if env_model := os.environ.get("LDR_SEARCH_MODEL"):
        search_config["model_name"] = env_model
    if env_provider := os.environ.get("LDR_SEARCH_PROVIDER"):
        search_config["provider"] = env_provider
    if env_url := os.environ.get("LDR_ENDPOINT_URL"):
        search_config["openai_endpoint_url"] = env_url

    # Set up evaluation configuration if needed
    evaluation_config = None
    if evaluation_model or evaluation_provider:
        evaluation_config = {
            "temperature": 0  # Always use zero temperature for evaluation
        }
        if evaluation_model:
            evaluation_config["model_name"] = evaluation_model
        if evaluation_provider:
            evaluation_config["provider"] = evaluation_provider
            # Add OpenRouter URL if using openai_endpoint
            if evaluation_provider == "openai_endpoint":
                evaluation_config["openai_endpoint_url"] = (
                    "https://openrouter.ai/api/v1"
                )

    # Run the benchmark
    results = run_browsecomp_benchmark(
        num_examples=num_examples,
        output_dir=output_dir,
        search_config=search_config,
        evaluation_config=evaluation_config,
        human_evaluation=human_evaluation,
    )

    return results


def get_available_benchmarks() -> List[Dict[str, str]]:
    """
    Get information about available benchmarks.

    Returns:
        List of dictionaries with benchmark information
    """
    return [
        {
            "id": "simpleqa",
            "name": "SimpleQA",
            "description": "Benchmark for factual question answering",
            "recommended_examples": 100,
        },
        {
            "id": "browsecomp",
            "name": "BrowseComp",
            "description": "Benchmark for web browsing comprehension",
            "recommended_examples": 100,
        },
    ]


def compare_configurations(
    dataset_type: str = "simpleqa",
    num_examples: int = 20,
    configurations: List[Dict[str, Any]] = None,
    output_dir: str = "benchmark_comparisons",
) -> Dict[str, Any]:
    """
    Compare multiple search configurations on the same benchmark.

    Args:
        dataset_type: Type of dataset to use
        num_examples: Number of examples to evaluate
        configurations: List of search configurations to compare
        output_dir: Directory to save results

    Returns:
        Dictionary with comparison results
    """
    if not configurations:
        # Default configurations to compare
        configurations = [
            {
                "name": "Base Config",
                "search_tool": "searxng",
                "iterations": 1,
                "questions_per_iteration": 3,
            },
            {
                "name": "More Iterations",
                "search_tool": "searxng",
                "iterations": 3,
                "questions_per_iteration": 3,
            },
            {
                "name": "More Questions",
                "search_tool": "searxng",
                "iterations": 1,
                "questions_per_iteration": 5,
            },
        ]

    # Create output directory
    import os

    os.makedirs(output_dir, exist_ok=True)

    # Run benchmarks for each configuration
    results = []
    for config in configurations:
        config_name = config.pop("name", f"Config-{len(results)}")

        logger.info(f"Running benchmark with configuration: {config_name}")

        search_config = {
            "iterations": config.pop("iterations", 1),
            "questions_per_iteration": config.pop("questions_per_iteration", 3),
            "search_tool": config.pop("search_tool", "searxng"),
        }

        # Add any remaining config items
        for key, value in config.items():
            search_config[key] = value

        # Run benchmark with this configuration
        benchmark_result = run_benchmark(
            dataset_type=dataset_type,
            num_examples=num_examples,
            output_dir=os.path.join(output_dir, config_name.replace(" ", "_")),
            search_config=search_config,
            run_evaluation=True,
        )

        # Add configuration name to results
        benchmark_result["configuration_name"] = config_name
        benchmark_result["search_config"] = search_config

        results.append(benchmark_result)

    # Generate comparison report
    import time

    timestamp = time.strftime("%Y%m%d_%H%M%S")
    report_file = os.path.join(
        output_dir, f"comparison_{dataset_type}_{timestamp}.md"
    )

    with open(report_file, "w") as f:
        f.write(f"# Configuration Comparison - {dataset_type.capitalize()}\n\n")

        # Write summary table
        f.write("## Summary\n\n")
        f.write("| Configuration | Accuracy | Avg. Time | Examples |\n")
        f.write("|---------------|----------|-----------|----------|\n")

        for result in results:
            accuracy = result.get("metrics", {}).get("accuracy", 0)
            avg_time = result.get("metrics", {}).get(
                "average_processing_time", 0
            )
            examples = result.get("total_examples", 0)

            f.write(
                f"| {result['configuration_name']} | {accuracy:.3f} | {avg_time:.2f}s | {examples} |\n"
            )

        f.write("\n## Configuration Details\n\n")

        for result in results:
            f.write(f"### {result['configuration_name']}\n\n")

            config = result.get("search_config", {})
            f.write("```\n")
            for key, value in config.items():
                f.write(f"{key}: {value}\n")
            f.write("```\n\n")

    logger.info(f"Comparison report saved to {report_file}")

    return {
        "status": "complete",
        "dataset_type": dataset_type,
        "configurations_tested": len(configurations),
        "report_path": report_file,
        "results": results,
    }


# Export the API functions
__all__ = [
    "evaluate_simpleqa",
    "evaluate_browsecomp",
    "get_available_benchmarks",
    "compare_configurations",
    "run_benchmark",  # For advanced users
    "calculate_metrics",
    "generate_report",
]
