"""
Branch-coverage tests for benchmarks/comparison/evaluator.py.

Targets the remaining 44 missing lines, focusing on branches not yet
exercised by test_evaluator_coverage.py or test_comparison_evaluator.py:

- compare_configurations: empty list early-return, all-runs-failed branch,
  exception-during-run path, repetitions averaging
- _evaluate_single_configuration: full exception handler (error dict returned)
- _calculate_average_metrics: multi-result averaging across all three metric
  categories, None-value filtering, disjoint metric keys
- _create_comparison_visualizations: matplotlib calls with successful results
- comparison with repetitions: multiple reps correctly produce avg_metrics
"""

import pytest
from unittest.mock import MagicMock, patch

MODULE = "local_deep_research.benchmarks.comparison.evaluator"

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

_SUCCESSFUL_RUN = {
    "success": True,
    "quality_metrics": {"overall_quality": 0.8},
    "speed_metrics": {"total_duration": 20.0},
    "resource_metrics": {"memory_mb": 128.0},
}

_FAILED_RUN = {"success": False, "error": "boom"}


def _avg_metrics_from(run):
    return {
        "quality_metrics": run.get("quality_metrics", {}),
        "speed_metrics": run.get("speed_metrics", {}),
        "resource_metrics": run.get("resource_metrics", {}),
    }


# ---------------------------------------------------------------------------
# 1. test_compare_configurations_empty_list
# ---------------------------------------------------------------------------


class TestCompareConfigurationsEmptyList:
    """compare_configurations returns an error dict for an empty list."""

    @patch(f"{MODULE}.os.makedirs")
    def test_empty_list_returns_error_dict(self, mock_makedirs):
        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        result = compare_configurations(query="anything", configurations=[])

        assert isinstance(result, dict)
        assert "error" in result
        assert result["error"] == "No configurations provided"

    @patch(f"{MODULE}.os.makedirs")
    def test_empty_list_does_not_call_evaluate(self, mock_makedirs):
        """_evaluate_single_configuration must not be called for empty input."""
        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        with patch(f"{MODULE}._evaluate_single_configuration") as mock_eval:
            compare_configurations(query="q", configurations=[])
            mock_eval.assert_not_called()

    @patch(f"{MODULE}.os.makedirs")
    def test_empty_list_still_creates_output_dir(self, mock_makedirs):
        """makedirs is called even before the empty-list guard."""
        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        compare_configurations(
            query="q", configurations=[], output_dir="/tmp/out_branch_test"
        )
        mock_makedirs.assert_called_once_with(
            "/tmp/out_branch_test", exist_ok=True
        )


# ---------------------------------------------------------------------------
# 2. test_compare_configurations_all_runs_failed
# ---------------------------------------------------------------------------


class TestCompareConfigurationsAllRunsFailed:
    """When every repetition of a config fails the summary has success=False."""

    @patch(f"{MODULE}.os.makedirs")
    @patch(f"{MODULE}._evaluate_single_configuration", return_value=_FAILED_RUN)
    @patch(f"{MODULE}._create_comparison_visualizations")
    @patch(
        "local_deep_research.security.file_write_verifier.write_json_verified"
    )
    def test_all_runs_failed_success_false(
        self, mock_write, mock_viz, mock_eval, mock_makedirs, tmp_path
    ):
        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        result = compare_configurations(
            query="q",
            configurations=[{"name": "BadConfig"}],
            output_dir=str(tmp_path),
            repetitions=2,
        )

        assert result["successful_configurations"] == 0
        assert result["failed_configurations"] == 1

    @patch(f"{MODULE}.os.makedirs")
    @patch(f"{MODULE}._evaluate_single_configuration", return_value=_FAILED_RUN)
    @patch(f"{MODULE}._create_comparison_visualizations")
    @patch(
        "local_deep_research.security.file_write_verifier.write_json_verified"
    )
    def test_all_runs_failed_result_summary_shape(
        self, mock_write, mock_viz, mock_eval, mock_makedirs, tmp_path
    ):
        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        result = compare_configurations(
            query="q",
            configurations=[{"name": "BadConfig"}],
            output_dir=str(tmp_path),
        )

        summary = result["results"][0]
        assert summary["success"] is False
        assert summary["runs_completed"] == 0
        assert summary["runs_failed"] == 1
        assert "error" in summary
        assert summary["error"] == "All runs failed"

    @patch(f"{MODULE}.os.makedirs")
    @patch(f"{MODULE}._evaluate_single_configuration")
    @patch(f"{MODULE}._create_comparison_visualizations")
    @patch(
        "local_deep_research.security.file_write_verifier.write_json_verified"
    )
    def test_exception_in_evaluate_treated_as_failed_run(
        self, mock_write, mock_viz, mock_eval, mock_makedirs, tmp_path
    ):
        """An exception raised by _evaluate_single_configuration counts as a failure."""
        mock_eval.side_effect = RuntimeError("network error")
        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        result = compare_configurations(
            query="q",
            configurations=[{"name": "Crasher"}],
            output_dir=str(tmp_path),
        )

        assert result["failed_configurations"] == 1
        # The individual_results entry must contain the error string
        individual = result["results"][0]["individual_results"]
        assert len(individual) == 1
        assert individual[0]["success"] is False
        assert "network error" in individual[0]["error"]


# ---------------------------------------------------------------------------
# 3. test_evaluate_single_configuration_exception
# ---------------------------------------------------------------------------


class TestEvaluateSingleConfigurationException:
    """_evaluate_single_configuration returns a proper error dict on exception."""

    @patch(f"{MODULE}.SpeedProfiler")
    @patch(f"{MODULE}.ResourceMonitor")
    @patch(f"{MODULE}.get_llm")
    def test_exception_returns_success_false(
        self, mock_get_llm, mock_monitor_cls, mock_profiler_cls
    ):
        mock_get_llm.side_effect = ValueError("bad model name")
        mock_profiler = MagicMock()
        mock_profiler_cls.return_value = mock_profiler
        mock_monitor = MagicMock()
        mock_monitor_cls.return_value = mock_monitor

        from local_deep_research.benchmarks.comparison.evaluator import (
            _evaluate_single_configuration,
        )

        result = _evaluate_single_configuration(
            query="test query", config={"name": "cfg1"}
        )

        assert result["success"] is False

    @patch(f"{MODULE}.SpeedProfiler")
    @patch(f"{MODULE}.ResourceMonitor")
    @patch(f"{MODULE}.get_llm")
    def test_exception_message_in_result(
        self, mock_get_llm, mock_monitor_cls, mock_profiler_cls
    ):
        mock_get_llm.side_effect = ValueError("bad model name")
        mock_profiler = MagicMock()
        mock_profiler_cls.return_value = mock_profiler
        mock_monitor = MagicMock()
        mock_monitor_cls.return_value = mock_monitor

        from local_deep_research.benchmarks.comparison.evaluator import (
            _evaluate_single_configuration,
        )

        result = _evaluate_single_configuration(
            query="test query", config={"name": "cfg1"}
        )

        assert "error" in result
        assert "bad model name" in result["error"]

    @patch(f"{MODULE}.SpeedProfiler")
    @patch(f"{MODULE}.ResourceMonitor")
    @patch(f"{MODULE}.get_llm")
    def test_exception_result_contains_timing_and_resource_details(
        self, mock_get_llm, mock_monitor_cls, mock_profiler_cls
    ):
        """Even on error the result includes timing_details and resource_details."""
        mock_get_llm.side_effect = Exception("oops")

        profiler_instance = MagicMock()
        profiler_instance.get_timings.return_value = {"llm_init": 0.5}
        mock_profiler_cls.return_value = profiler_instance

        monitor_instance = MagicMock()
        monitor_instance.get_combined_stats.return_value = {"cpu": 10.0}
        mock_monitor_cls.return_value = monitor_instance

        from local_deep_research.benchmarks.comparison.evaluator import (
            _evaluate_single_configuration,
        )

        result = _evaluate_single_configuration(query="q", config={})

        assert "timing_details" in result
        assert "resource_details" in result

    @patch(f"{MODULE}.SpeedProfiler")
    @patch(f"{MODULE}.ResourceMonitor")
    @patch(f"{MODULE}.get_llm")
    def test_exception_stops_profiling(
        self, mock_get_llm, mock_monitor_cls, mock_profiler_cls
    ):
        """stop() must be called on both profiler and monitor even on error."""
        mock_get_llm.side_effect = Exception("fail")

        profiler_instance = MagicMock()
        mock_profiler_cls.return_value = profiler_instance

        monitor_instance = MagicMock()
        mock_monitor_cls.return_value = monitor_instance

        from local_deep_research.benchmarks.comparison.evaluator import (
            _evaluate_single_configuration,
        )

        _evaluate_single_configuration(query="q", config={})

        profiler_instance.stop.assert_called()
        monitor_instance.stop.assert_called()


# ---------------------------------------------------------------------------
# 4. test_calculate_average_metrics
# ---------------------------------------------------------------------------


class TestCalculateAverageMetrics:
    """_calculate_average_metrics averages correctly across all metric types."""

    def test_quality_metrics_averaged(self):
        from local_deep_research.benchmarks.comparison.evaluator import (
            _calculate_average_metrics,
        )

        results = [
            {
                "quality_metrics": {"score": 0.4},
                "speed_metrics": {},
                "resource_metrics": {},
            },
            {
                "quality_metrics": {"score": 0.6},
                "speed_metrics": {},
                "resource_metrics": {},
            },
        ]
        avg = _calculate_average_metrics(results)
        assert avg["quality_metrics"]["score"] == pytest.approx(0.5)

    def test_speed_metrics_averaged(self):
        from local_deep_research.benchmarks.comparison.evaluator import (
            _calculate_average_metrics,
        )

        results = [
            {
                "quality_metrics": {},
                "speed_metrics": {"total_duration": 10.0},
                "resource_metrics": {},
            },
            {
                "quality_metrics": {},
                "speed_metrics": {"total_duration": 30.0},
                "resource_metrics": {},
            },
        ]
        avg = _calculate_average_metrics(results)
        assert avg["speed_metrics"]["total_duration"] == pytest.approx(20.0)

    def test_resource_metrics_averaged(self):
        from local_deep_research.benchmarks.comparison.evaluator import (
            _calculate_average_metrics,
        )

        results = [
            {
                "quality_metrics": {},
                "speed_metrics": {},
                "resource_metrics": {"memory_mb": 100.0},
            },
            {
                "quality_metrics": {},
                "speed_metrics": {},
                "resource_metrics": {"memory_mb": 200.0},
            },
        ]
        avg = _calculate_average_metrics(results)
        assert avg["resource_metrics"]["memory_mb"] == pytest.approx(150.0)

    def test_none_values_filtered_out(self):
        """None values are skipped so the average uses only real numbers."""
        from local_deep_research.benchmarks.comparison.evaluator import (
            _calculate_average_metrics,
        )

        results = [
            {
                "quality_metrics": {"x": None},
                "speed_metrics": {},
                "resource_metrics": {},
            },
            {
                "quality_metrics": {"x": 0.8},
                "speed_metrics": {},
                "resource_metrics": {},
            },
        ]
        avg = _calculate_average_metrics(results)
        # Only the non-None value contributes
        assert avg["quality_metrics"]["x"] == pytest.approx(0.8)

    def test_disjoint_metric_keys_all_present(self):
        """Keys present in only one result still appear in the output."""
        from local_deep_research.benchmarks.comparison.evaluator import (
            _calculate_average_metrics,
        )

        results = [
            {
                "quality_metrics": {"key_a": 0.9},
                "speed_metrics": {},
                "resource_metrics": {},
            },
            {
                "quality_metrics": {"key_b": 0.3},
                "speed_metrics": {},
                "resource_metrics": {},
            },
        ]
        avg = _calculate_average_metrics(results)
        assert avg["quality_metrics"]["key_a"] == pytest.approx(0.9)
        assert avg["quality_metrics"]["key_b"] == pytest.approx(0.3)

    def test_empty_input_returns_empty_dict(self):
        from local_deep_research.benchmarks.comparison.evaluator import (
            _calculate_average_metrics,
        )

        assert _calculate_average_metrics([]) == {}

    def test_three_results_averaged_correctly(self):
        from local_deep_research.benchmarks.comparison.evaluator import (
            _calculate_average_metrics,
        )

        results = [
            {
                "quality_metrics": {"q": 0.2},
                "speed_metrics": {},
                "resource_metrics": {},
            },
            {
                "quality_metrics": {"q": 0.5},
                "speed_metrics": {},
                "resource_metrics": {},
            },
            {
                "quality_metrics": {"q": 0.8},
                "speed_metrics": {},
                "resource_metrics": {},
            },
        ]
        avg = _calculate_average_metrics(results)
        assert avg["quality_metrics"]["q"] == pytest.approx(0.5)

    def test_output_always_has_three_categories(self):
        """Result always contains quality_metrics, speed_metrics, resource_metrics keys."""
        from local_deep_research.benchmarks.comparison.evaluator import (
            _calculate_average_metrics,
        )

        results = [
            {
                "quality_metrics": {"x": 1.0},
                "speed_metrics": {},
                "resource_metrics": {},
            }
        ]
        avg = _calculate_average_metrics(results)
        assert "quality_metrics" in avg
        assert "speed_metrics" in avg
        assert "resource_metrics" in avg


# ---------------------------------------------------------------------------
# 5. test_create_comparison_visualizations
# ---------------------------------------------------------------------------


class TestCreateComparisonVisualizations:
    """_create_comparison_visualizations makes the expected matplotlib calls."""

    @patch(f"{MODULE}.plt")
    def test_no_successful_results_skips_all_plots(self, mock_plt, tmp_path):
        """Early return when there are no successful results."""
        from local_deep_research.benchmarks.comparison.evaluator import (
            _create_comparison_visualizations,
        )

        report = {"results": [{"success": False, "error": "failed"}]}
        _create_comparison_visualizations(
            report, output_dir=str(tmp_path), timestamp="20240101_120000"
        )

        mock_plt.figure.assert_not_called()
        mock_plt.barh.assert_not_called()
        mock_plt.savefig.assert_not_called()

    @patch(f"{MODULE}._create_spider_chart")
    @patch(f"{MODULE}._create_pareto_chart")
    @patch(f"{MODULE}._create_metric_comparison_chart")
    @patch(f"{MODULE}.plt")
    def test_successful_results_calls_savefig(
        self, mock_plt, mock_metric_chart, mock_pareto, mock_spider, tmp_path
    ):
        """With one successful result all sub-chart helpers are called."""
        from local_deep_research.benchmarks.comparison.evaluator import (
            _create_comparison_visualizations,
        )

        mock_fig = MagicMock()
        mock_plt.figure.return_value = mock_fig

        report = {
            "results": [
                {
                    "success": True,
                    "name": "Config A",
                    "overall_score": 0.75,
                    "avg_metrics": {
                        "quality_metrics": {"overall_quality": 0.75},
                        "speed_metrics": {"total_duration": 15.0},
                        "resource_metrics": {},
                    },
                }
            ]
        }

        _create_comparison_visualizations(
            report, output_dir=str(tmp_path), timestamp="20240101_000000"
        )

        # Overall score bar chart triggers figure/barh/savefig
        mock_plt.figure.assert_called()
        mock_plt.savefig.assert_called()
        mock_plt.close.assert_called()

        # Sub-chart helpers must have been invoked once each
        mock_metric_chart.assert_called()
        mock_spider.assert_called_once()
        mock_pareto.assert_called_once()

    @patch(f"{MODULE}._create_spider_chart")
    @patch(f"{MODULE}._create_pareto_chart")
    @patch(f"{MODULE}._create_metric_comparison_chart")
    @patch(f"{MODULE}.plt")
    def test_config_names_extracted_correctly(
        self, mock_plt, mock_metric_chart, mock_pareto, mock_spider, tmp_path
    ):
        """Config names used in barh must come from result['name']."""
        from local_deep_research.benchmarks.comparison.evaluator import (
            _create_comparison_visualizations,
        )

        mock_plt.figure.return_value = MagicMock()

        report = {
            "results": [
                {
                    "success": True,
                    "name": "MySpecialConfig",
                    "overall_score": 0.5,
                    "avg_metrics": {
                        "quality_metrics": {},
                        "speed_metrics": {},
                        "resource_metrics": {},
                    },
                }
            ]
        }

        _create_comparison_visualizations(
            report, output_dir=str(tmp_path), timestamp="ts1"
        )

        # barh should have been called with the name in the y-axis list
        barh_call_args = mock_plt.barh.call_args
        assert barh_call_args is not None
        config_names_arg = barh_call_args[0][0]
        assert "MySpecialConfig" in config_names_arg


# ---------------------------------------------------------------------------
# 6. test_comparison_with_repetitions
# ---------------------------------------------------------------------------


class TestComparisonWithRepetitions:
    """compare_configurations with multiple repetitions averages metrics correctly."""

    @patch(f"{MODULE}.os.makedirs")
    @patch(f"{MODULE}._evaluate_single_configuration")
    @patch(f"{MODULE}.calculate_combined_score", return_value=0.7)
    @patch(f"{MODULE}._create_comparison_visualizations")
    @patch(
        "local_deep_research.security.file_write_verifier.write_json_verified"
    )
    def test_three_repetitions_runs_completed(
        self,
        mock_write,
        mock_viz,
        mock_score,
        mock_eval,
        mock_makedirs,
        tmp_path,
    ):
        mock_eval.return_value = _SUCCESSFUL_RUN.copy()

        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        result = compare_configurations(
            query="q",
            configurations=[{"name": "Cfg"}],
            output_dir=str(tmp_path),
            repetitions=3,
        )

        summary = result["results"][0]
        assert summary["runs_completed"] == 3
        assert summary["runs_failed"] == 0
        assert summary["success"] is True

    @patch(f"{MODULE}.os.makedirs")
    @patch(f"{MODULE}._evaluate_single_configuration")
    @patch(f"{MODULE}.calculate_combined_score", return_value=0.7)
    @patch(f"{MODULE}._create_comparison_visualizations")
    @patch(
        "local_deep_research.security.file_write_verifier.write_json_verified"
    )
    def test_partial_repetition_failure(
        self,
        mock_write,
        mock_viz,
        mock_score,
        mock_eval,
        mock_makedirs,
        tmp_path,
    ):
        """Only successful repetitions count toward runs_completed."""
        mock_eval.side_effect = [
            _SUCCESSFUL_RUN.copy(),
            _FAILED_RUN.copy(),
            _SUCCESSFUL_RUN.copy(),
        ]

        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        result = compare_configurations(
            query="q",
            configurations=[{"name": "Cfg"}],
            output_dir=str(tmp_path),
            repetitions=3,
        )

        summary = result["results"][0]
        assert summary["runs_completed"] == 2
        assert summary["runs_failed"] == 1
        assert summary["success"] is True

    @patch(f"{MODULE}.os.makedirs")
    @patch(f"{MODULE}._evaluate_single_configuration")
    @patch(f"{MODULE}._calculate_average_metrics")
    @patch(f"{MODULE}.calculate_combined_score", return_value=0.65)
    @patch(f"{MODULE}._create_comparison_visualizations")
    @patch(
        "local_deep_research.security.file_write_verifier.write_json_verified"
    )
    def test_repetitions_pass_all_successful_runs_to_avg(
        self,
        mock_write,
        mock_viz,
        mock_score,
        mock_avg,
        mock_eval,
        mock_makedirs,
        tmp_path,
    ):
        """_calculate_average_metrics is called with the list of successful runs."""
        run1 = _SUCCESSFUL_RUN.copy()
        run2 = _SUCCESSFUL_RUN.copy()
        mock_eval.side_effect = [run1, run2]
        mock_avg.return_value = {
            "quality_metrics": {},
            "speed_metrics": {},
            "resource_metrics": {},
        }

        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        compare_configurations(
            query="q",
            configurations=[{"name": "Cfg"}],
            output_dir=str(tmp_path),
            repetitions=2,
        )

        mock_avg.assert_called_once_with([run1, run2])

    @patch(f"{MODULE}.os.makedirs")
    @patch(f"{MODULE}._evaluate_single_configuration")
    @patch(f"{MODULE}.calculate_combined_score", return_value=0.8)
    @patch(f"{MODULE}._create_comparison_visualizations")
    @patch(
        "local_deep_research.security.file_write_verifier.write_json_verified"
    )
    def test_repetitions_field_in_report(
        self,
        mock_write,
        mock_viz,
        mock_score,
        mock_eval,
        mock_makedirs,
        tmp_path,
    ):
        """The top-level 'repetitions' key reflects the requested count."""
        mock_eval.return_value = _SUCCESSFUL_RUN.copy()

        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        result = compare_configurations(
            query="q",
            configurations=[{"name": "Cfg"}],
            output_dir=str(tmp_path),
            repetitions=5,
        )

        assert result["repetitions"] == 5

    @patch(f"{MODULE}.os.makedirs")
    @patch(f"{MODULE}._evaluate_single_configuration")
    @patch(f"{MODULE}.calculate_combined_score")
    @patch(f"{MODULE}._create_comparison_visualizations")
    @patch(
        "local_deep_research.security.file_write_verifier.write_json_verified"
    )
    def test_two_configs_sorted_by_score(
        self,
        mock_write,
        mock_viz,
        mock_score,
        mock_eval,
        mock_makedirs,
        tmp_path,
    ):
        """Results are sorted descending by overall_score."""
        mock_eval.return_value = _SUCCESSFUL_RUN.copy()
        mock_score.side_effect = [0.3, 0.9]

        from local_deep_research.benchmarks.comparison.evaluator import (
            compare_configurations,
        )

        result = compare_configurations(
            query="q",
            configurations=[{"name": "Low"}, {"name": "High"}],
            output_dir=str(tmp_path),
        )

        scores = [
            r["overall_score"] for r in result["results"] if r.get("success")
        ]
        assert scores == sorted(scores, reverse=True)
