"""Report generator — aggregates results into structured reports with fix suggestions."""

from __future__ import annotations

from collections import defaultdict

from evalkit.judges.base import BaseJudge
from evalkit.metrics.base import BaseMetric
from evalkit.models import (
    EvaluationReport,
    EvaluationResult,
    FailureGroup,
    MetricSummary,
    ReportSummary,
)
from evalkit.reports.analyzer import analyze_failure_group
from evalkit.reports.patterns import run_all_detectors


class ReportGenerator:
    SUGGESTIONS: dict[str, dict[str, str]] = {
        "faithfulness": {
            "critical": (
                "Fundamental faithfulness failure: answers are largely ungrounded in the retrieved "
                "context. Check your prompt template immediately — ensure the LLM is explicitly "
                "instructed to only use information from the provided context and nothing else."
            ),
            "warning": (
                "Moderate faithfulness issues: some answers contain claims not fully supported "
                "by the retrieved context. Consider strengthening your prompt template to emphasize "
                "grounding answers in the provided context."
            ),
            "mild": (
                "Near-miss faithfulness: answers are mostly grounded but occasionally include "
                "minor unsupported details. Fine-tune your prompt to reinforce strict adherence "
                "to the retrieved context."
            ),
        },
        "answer_relevancy": {
            "critical": (
                "Fundamental relevancy failure: responses are not addressing the user's question "
                "at all. Check your prompt template and retrieval pipeline immediately — the model "
                "may be ignoring the query entirely."
            ),
            "warning": (
                "Moderate relevancy issues: responses partially address the question but include "
                "off-topic content. Consider adding few-shot examples that demonstrate focused, "
                "on-topic answers."
            ),
            "mild": (
                "Near-miss relevancy: responses mostly address the question but could be more "
                "focused. Fine-tune your prompt to encourage more direct answers."
            ),
        },
        "contextual_relevancy": {
            "critical": (
                "Fundamental retrieval failure: retrieved chunks are largely irrelevant to the "
                "query. Check your embedding model and retrieval pipeline immediately — consider "
                "switching to a better embedding model or reindexing your documents."
            ),
            "warning": (
                "Moderate retrieval issues: some retrieved chunks are not relevant to the query. "
                "Consider improving your chunking strategy or adjusting retrieval parameters "
                "(top-k, similarity threshold)."
            ),
            "mild": (
                "Near-miss retrieval: most chunks are relevant but a few are marginally useful. "
                "Fine-tune your similarity threshold or top-k to filter out borderline chunks."
            ),
        },
        "hallucination": {
            "critical": (
                "Fundamental hallucination problem: the LLM is fabricating facts not present in "
                "the retrieved context. Check your system prompt immediately — use a more "
                "instruction-following model or drastically lower the temperature."
            ),
            "warning": (
                "Moderate hallucination rate: the LLM occasionally adds facts not present in the "
                "context. Consider adding explicit anti-hallucination instructions to your system "
                "prompt and lowering the temperature."
            ),
            "mild": (
                "Near-miss hallucination: the LLM rarely fabricates but occasionally embellishes "
                "details. Fine-tune your temperature setting and add gentle anti-hallucination "
                "reminders to your prompt."
            ),
        },
        "completeness": {
            "critical": (
                "Fundamental completeness failure: answers are missing most key information from "
                "expected responses. Check your retrieval pipeline immediately — consider "
                "increasing top-k significantly or improving chunking to avoid splitting related content."
            ),
            "warning": (
                "Moderate completeness issues: answers are missing some key information. Consider "
                "retrieving more chunks (higher top-k) or improving chunking to keep related "
                "content together."
            ),
            "mild": (
                "Near-miss completeness: answers cover most key points but miss minor details. "
                "Fine-tune your chunking granularity or slightly increase top-k for better coverage."
            ),
        },
        "tool_selection": {
            "critical": (
                "Fundamental tool selection failure: the agent is calling completely wrong tools. "
                "Check your tool descriptions and agent system prompt immediately — ensure the "
                "agent clearly understands when to use each tool."
            ),
            "warning": (
                "Moderate tool selection issues: the agent sometimes picks incorrect tools or "
                "uses them in the wrong order. Consider improving tool descriptions and adding "
                "selection guidance to the system prompt."
            ),
            "mild": (
                "Near-miss tool selection: the agent mostly picks the right tools but occasionally "
                "makes suboptimal choices. Fine-tune tool descriptions to clarify edge cases."
            ),
        },
        "tool_parameter": {
            "critical": (
                "Fundamental tool parameter failure: the agent is passing completely incorrect "
                "parameters to tools. Check your tool parameter descriptions immediately — ensure "
                "parameter schemas and examples cover all expected input formats."
            ),
            "warning": (
                "Moderate tool parameter issues: the agent sometimes passes incorrect parameters. "
                "Consider improving parameter descriptions and adding validation examples to "
                "your tool definitions."
            ),
            "mild": (
                "Near-miss tool parameters: parameters are mostly correct with minor formatting "
                "differences. Fine-tune your parameter descriptions to clarify expected formats."
            ),
        },
        "tool_parameter_similarity": {
            "critical": (
                "Fundamental parameter value mismatch: parameter values are semantically very "
                "different from expected. Check how your agent extracts and formats values from "
                "user input immediately."
            ),
            "warning": (
                "Moderate parameter similarity issues: some parameter values differ semantically "
                "from expected. Consider improving how your agent extracts and normalizes values "
                "from user input."
            ),
            "mild": (
                "Near-miss parameter similarity: parameter values are close but not quite matching "
                "expected semantics. Fine-tune your agent's value extraction and formatting logic."
            ),
        },
        "exact_match": {
            "critical": (
                "Fundamental exact match failure: outputs are completely different from expected "
                "text. Check your output formatting and normalization pipeline immediately."
            ),
            "warning": (
                "Moderate exact match issues: outputs differ from expected text in formatting or "
                "structure. Consider adding output normalization or post-processing steps."
            ),
            "mild": (
                "Near-miss exact match: outputs are very close but differ in minor whitespace "
                "or formatting. Fine-tune your output normalization to handle edge cases."
            ),
        },
        "string_containment": {
            "critical": (
                "Fundamental containment failure: expected substrings are completely absent from "
                "the output. Check your generation pipeline immediately — the model may not be "
                "receiving the right context."
            ),
            "warning": (
                "Moderate containment issues: some expected substrings are not found in the "
                "output. Consider improving your prompt to ensure key terms are included in "
                "responses."
            ),
            "mild": (
                "Near-miss containment: most expected substrings are present but a few are "
                "missing. Fine-tune your prompt to encourage inclusion of all key terms."
            ),
        },
        "string_similarity": {
            "critical": (
                "Fundamental string similarity failure: outputs have very low edit distance "
                "similarity to expected text. Check your generation pipeline immediately."
            ),
            "warning": (
                "Moderate string similarity issues: outputs show noticeable differences from "
                "expected text. Consider improving your prompt or adding output constraints."
            ),
            "mild": (
                "Near-miss string similarity: outputs are close to expected text with minor "
                "differences. Fine-tune your output formatting for better consistency."
            ),
        },
        "bleu": {
            "critical": (
                "Fundamental BLEU score failure: very low n-gram precision indicates outputs "
                "share almost no phrasing with expected text. Check your generation pipeline "
                "immediately."
            ),
            "warning": (
                "Moderate BLEU score issues: low n-gram precision suggests outputs use different "
                "phrasing than expected. Consider adding few-shot examples to guide output style."
            ),
            "mild": (
                "Near-miss BLEU score: n-gram precision is close to target but could improve. "
                "Fine-tune your prompt or add style guidance for better phrasing alignment."
            ),
        },
        "rouge": {
            "critical": (
                "Fundamental ROUGE score failure: very low n-gram recall indicates outputs are "
                "missing most expected content. Check your retrieval and generation pipeline "
                "immediately."
            ),
            "warning": (
                "Moderate ROUGE score issues: low n-gram recall suggests outputs are missing "
                "some expected content. Consider improving retrieval coverage or prompt design."
            ),
            "mild": (
                "Near-miss ROUGE score: n-gram recall is close to target but could improve. "
                "Fine-tune your prompt to encourage more comprehensive coverage of key content."
            ),
        },
    }

    def generate(
        self,
        results: list[EvaluationResult],
        suite_name: str,
        *,
        metrics: list[BaseMetric] | None = None,
        judge: BaseJudge | None = None,
    ) -> EvaluationReport:
        thresholds = {m.name: m.threshold for m in metrics} if metrics else None
        summary = self._compute_summary(results, thresholds=thresholds)
        failures = self._group_failures(
            results,
            thresholds=thresholds,
            score_by_metric=summary.score_by_metric,
            judge=judge,
        )
        return EvaluationReport(
            suite_name=suite_name,
            results=results,
            summary=summary,
            failures=failures,
        )

    def _classify_severity(self, avg_score: float, threshold: float) -> str:
        if avg_score >= threshold * 0.85:
            return "mild"
        if avg_score >= threshold * 0.5:
            return "warning"
        return "critical"

    def _get_suggestion(self, metric_name: str, severity: str) -> str:
        metric_suggestions = self.SUGGESTIONS.get(metric_name)
        if metric_suggestions is None:
            return f"Review your {metric_name} metric scores."
        return metric_suggestions.get(severity, metric_suggestions.get("warning", ""))

    def _compute_summary(
        self,
        results: list[EvaluationResult],
        *,
        thresholds: dict[str, float] | None = None,
    ) -> ReportSummary:
        total_cases = len(results)
        passed = sum(1 for r in results if r.overall_score >= 0.5)
        failed = total_cases - passed
        average_score = sum(r.overall_score for r in results) / total_cases if total_cases > 0 else 0.0

        metric_scores: dict[str, list[float]] = defaultdict(list)
        for result in results:
            for ms in result.scores:
                metric_scores[ms.metric_name].append(ms.score)

        score_by_metric: dict[str, float] = {
            metric_name: sum(scores) / len(scores) for metric_name, scores in metric_scores.items()
        }

        metric_summaries: dict[str, MetricSummary] = {}
        for metric_name, scores in metric_scores.items():
            t = (thresholds or {}).get(metric_name, 0.5)
            passed_count = sum(1 for s in scores if s >= t)
            metric_summaries[metric_name] = MetricSummary(
                average_score=sum(scores) / len(scores),
                pass_rate=passed_count / len(scores),
                min_score=min(scores),
                max_score=max(scores),
                total_count=len(scores),
                failed_count=len(scores) - passed_count,
            )

        return ReportSummary(
            total_cases=total_cases,
            passed=passed,
            failed=failed,
            average_score=average_score,
            score_by_metric=score_by_metric,
            metric_summaries=metric_summaries,
        )

    def _group_failures(
        self,
        results: list[EvaluationResult],
        *,
        thresholds: dict[str, float] | None = None,
        score_by_metric: dict[str, float] | None = None,
        judge: BaseJudge | None = None,
    ) -> list[FailureGroup]:
        metric_failures: dict[str, list[tuple[float, EvaluationResult]]] = defaultdict(list)

        for result in results:
            for ms in result.scores:
                if not ms.passed:
                    metric_failures[ms.metric_name].append((ms.score, result))

        failure_groups: list[FailureGroup] = []
        for metric_name, failures in metric_failures.items():
            failures_sorted = sorted(failures, key=lambda t: t[0])
            examples = [result for _, result in failures_sorted[:3]]
            failing_scores = [score for score, _ in failures]
            avg_score = sum(failing_scores) / len(failing_scores)
            t = (thresholds or {}).get(metric_name, 0.5)
            severity = self._classify_severity(avg_score, t)
            suggestion = self._get_suggestion(metric_name, severity)

            overall_avg = score_by_metric.get(metric_name, 0.0) if score_by_metric else 0.0
            patterns = run_all_detectors(metric_name, failures, overall_avg)
            llm_analysis = analyze_failure_group(metric_name, failures, judge) if judge else None

            failure_groups.append(
                FailureGroup(
                    failure_type=metric_name,
                    count=len(failures),
                    average_score=avg_score,
                    examples=examples,
                    suggestion=suggestion,
                    severity=severity,
                    patterns=patterns,
                    llm_analysis=llm_analysis,
                )
            )

        failure_groups.sort(key=lambda fg: fg.count, reverse=True)
        return failure_groups
