# platform/deskclaw/evaluation/scorer.py
# ═══════════════════════════════════════════════════════════════
# Phase 4: Evaluation Engine — Composite Scoring & Grade Determination
# Four-dimension weighted scoring, fatal override (one-veto),
# high-severity cap, robustness bonus, and final grade output.
# ═══════════════════════════════════════════════════════════════

from __future__ import annotations

import json
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import Any


def _load_json(path: Path) -> Any:
    return json.loads(path.read_text(encoding="utf-8"))


def _json_dumps(obj: Any) -> str:
    return json.dumps(obj, indent=2, ensure_ascii=False)


# ── Grade Enum ──


class Grade(str, Enum):
    """Final deployment grade, ordered from highest to lowest."""

    S_PLUS = "S+"
    S = "S"
    A = "A"
    A_MINUS = "A-"
    B = "B"
    C = "C"
    C_BLOCK = "C-BLOCK"


# Grade ranking: lower number = higher grade
_GRADE_RANK: dict[Grade, int] = {
    Grade.S_PLUS: 0,
    Grade.S: 1,
    Grade.A: 2,
    Grade.A_MINUS: 3,
    Grade.B: 4,
    Grade.C: 5,
    Grade.C_BLOCK: 6,
}


def grade_rank(grade: Grade) -> int:
    """Numeric rank of *grade* (lower = better)."""
    return _GRADE_RANK[grade]


# ── Data Models ──


@dataclass(frozen=True)
class DimensionScore:
    """Score for a single evaluation dimension."""

    name: str
    score: float  # Normalized [0, 1]
    weight: float  # Contribution to composite (e.g. 0.25)
    sub_scores: dict[str, float] = field(default_factory=dict)

    @property
    def weighted(self) -> float:
        """Weighted contribution: ``score × weight``."""
        return self.score * self.weight


@dataclass(frozen=True)
class StressTestResult:
    """Outcome of a single stress test."""

    test_id: str  # e.g. "ST-01"
    passed: bool
    severity: str  # "critical" | "high" | "normal"
    score: float = 0.0  # Raw score [0, 1]
    detail: str = ""


@dataclass
class FatalOverride:
    """Configuration for a fatal override rule."""

    test_id: str
    severity: str  # "critical" | "high"
    on_fail: str  # "cap_grade_C" | "cap_grade_B"


@dataclass
class ScorerConfig:
    """
    Scorer configuration, typically loaded from ``deskclaw-plugin.yaml``
    ``evaluation`` section.
    """

    dimension_weights: dict[str, float] = field(
        default_factory=lambda: {
            "capacity": 0.25,
            "fidelity": 0.25,
            "hallucination": 0.30,
            "operational": 0.20,
        }
    )
    grade_thresholds: dict[str, float] = field(
        default_factory=lambda: {
            "S": 0.92,
            "A": 0.82,
            "B": 0.70,
        }
    )
    fatal_overrides: list[FatalOverride] = field(
        default_factory=lambda: [
            FatalOverride(test_id="ST-01", severity="critical", on_fail="cap_grade_C"),
            FatalOverride(test_id="ST-04", severity="critical", on_fail="cap_grade_C"),
        ]
    )
    high_severity_min_failures: int = 2
    high_severity_on_trigger: str = "cap_grade_B"
    robustness_bonus: float = 0.02

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> ScorerConfig:
        """Build a :class:`ScorerConfig` from a plugin YAML ``evaluation`` dict."""
        _default_weights = {
            "capacity": 0.25,
            "fidelity": 0.25,
            "hallucination": 0.30,
            "operational": 0.20,
        }
        _default_thresholds = {"S": 0.92, "A": 0.82, "B": 0.70}
        _default_fatals = [
            FatalOverride(test_id="ST-01", severity="critical", on_fail="cap_grade_C"),
            FatalOverride(test_id="ST-04", severity="critical", on_fail="cap_grade_C"),
        ]

        fatals = [
            FatalOverride(
                test_id=f["test"],
                severity=f["severity"],
                on_fail=f["on_fail"],
            )
            for f in data.get("fatal_overrides", [])
        ]
        high_rule = data.get("high_severity_rule", {})
        bonus_section = data.get("robustness_bonus", {})

        return cls(
            dimension_weights=data.get("dimension_weights", _default_weights),
            grade_thresholds=data.get("grade_thresholds", _default_thresholds),
            fatal_overrides=fatals if fatals else _default_fatals,
            high_severity_min_failures=high_rule.get("min_failures", 2),
            high_severity_on_trigger=high_rule.get("on_trigger", "cap_grade_B"),
            robustness_bonus=bonus_section.get("all_stress_passed", 0.02)
            if isinstance(bonus_section, dict)
            else float(bonus_section),
        )


@dataclass
class EvaluationReport:
    """Complete evaluation output."""

    dimension_scores: list[DimensionScore]
    stress_results: list[StressTestResult]
    composite_score: float  # Before adjustments
    adjusted_score: float  # After robustness bonus
    grade: Grade
    grade_capped_by: str | None  # e.g. "ST-01 FAIL (critical)" or None
    robustness_bonus_applied: bool
    zero_hallucination: bool
    external_benchmark_scorecard: dict[str, Any] = field(default_factory=dict)

    def to_dict(self) -> dict[str, Any]:
        return {
            "dimension_scores": [
                {
                    "name": d.name,
                    "score": round(d.score, 4),
                    "weight": d.weight,
                    "weighted": round(d.weighted, 4),
                    "sub_scores": d.sub_scores,
                }
                for d in self.dimension_scores
            ],
            "stress_results": [
                {
                    "test_id": s.test_id,
                    "passed": s.passed,
                    "severity": s.severity,
                    "score": round(s.score, 4),
                    "detail": s.detail,
                }
                for s in self.stress_results
            ],
            "composite_score": round(self.composite_score, 4),
            "adjusted_score": round(self.adjusted_score, 4),
            "grade": self.grade.value,
            "grade_capped_by": self.grade_capped_by,
            "robustness_bonus_applied": self.robustness_bonus_applied,
            "zero_hallucination": self.zero_hallucination,
            "external_benchmark_scorecard": self.external_benchmark_scorecard,
        }


# ── Core Scorer ──


class CompositeScorer:
    """
    Phase 4 evaluation engine.

    Computes a four-dimension weighted composite score, applies stress-test
    based fatal overrides and severity caps, then determines the final
    deployment grade.

    Scoring flow:
    1. Compute composite = Σ(dimension.score × dimension.weight)
    2. Apply robustness bonus (+0.02) if ALL stress tests pass
    3. Check fatal overrides: any critical failure → C-BLOCK
    4. Check high-severity rule: ≥N high failures → cap at B
    5. Determine grade from adjusted score + thresholds
    6. Apply grade cap if triggered
    """

    def __init__(self, config: ScorerConfig | None = None) -> None:
        self.config = config or ScorerConfig()

    def score(
        self,
        dimension_scores: list[DimensionScore],
        stress_results: list[StressTestResult],
        *,
        zero_hallucination: bool = False,
    ) -> EvaluationReport:
        """
        Execute the full evaluation pipeline.

        Args:
            dimension_scores:   Per-dimension normalized scores.
            stress_results:     Stress test outcomes (ST-01 through ST-04).
            zero_hallucination: Whether the agent produced zero hallucinations
                                (qualifies for S+ if score threshold met).

        Returns:
            :class:`EvaluationReport` with final grade and breakdown.
        """
        # Step 1: Composite score
        composite = self._compute_composite(dimension_scores)

        # Step 2: Stress analysis
        all_passed = all(st.passed for st in stress_results)
        critical_failures = [
            st for st in stress_results if st.severity == "critical" and not st.passed
        ]
        high_failures = [
            st for st in stress_results if st.severity == "high" and not st.passed
        ]

        # Step 3: Robustness bonus
        bonus_applied = all_passed and len(stress_results) > 0
        adjusted = composite + (self.config.robustness_bonus if bonus_applied else 0.0)

        # Step 4: Fatal override check
        grade_cap: Grade | None = None
        cap_reason: str | None = None

        if critical_failures:
            grade_cap = Grade.C_BLOCK
            reasons = [f"{st.test_id} FAIL ({st.severity})" for st in critical_failures]
            cap_reason = "; ".join(reasons)

        elif len(high_failures) >= self.config.high_severity_min_failures:
            grade_cap = Grade.B
            reasons = [f"{st.test_id} FAIL ({st.severity})" for st in high_failures]
            cap_reason = (
                f"{len(high_failures)} high-severity failures: {'; '.join(reasons)}"
            )

        # Step 5: Score-based grade determination
        grade = self._determine_grade(
            adjusted,
            zero_hallucination=zero_hallucination,
            high_failure_count=len(high_failures),
        )

        # Step 6: Apply cap
        if grade_cap is not None and grade_rank(grade) < grade_rank(grade_cap):
            grade = grade_cap

        return EvaluationReport(
            dimension_scores=dimension_scores,
            stress_results=stress_results,
            composite_score=round(composite, 4),
            adjusted_score=round(adjusted, 4),
            grade=grade,
            grade_capped_by=cap_reason,
            robustness_bonus_applied=bonus_applied,
            zero_hallucination=zero_hallucination,
        )

    # ── Private Helpers ──

    @staticmethod
    def _compute_composite(dimensions: list[DimensionScore]) -> float:
        """Weighted sum of dimension scores."""
        return sum(d.weighted for d in dimensions)

    def _determine_grade(
        self,
        adjusted_score: float,
        *,
        zero_hallucination: bool,
        high_failure_count: int,
    ) -> Grade:
        """
        Map adjusted score to a grade using configured thresholds.

        Special cases:
        - ``>= S threshold`` + zero hallucination → S+
        - ``>= S threshold`` → S
        - ``>= A threshold`` + 1 high failure → A-
        - ``>= A threshold`` → A
        - ``>= B threshold`` → B
        - Below B → C
        """
        thresholds = self.config.grade_thresholds
        s_thresh = thresholds.get("S", 0.92)
        a_thresh = thresholds.get("A", 0.82)
        b_thresh = thresholds.get("B", 0.70)

        if adjusted_score >= s_thresh:
            return Grade.S_PLUS if zero_hallucination else Grade.S
        elif adjusted_score >= a_thresh:
            return Grade.A_MINUS if high_failure_count == 1 else Grade.A
        elif adjusted_score >= b_thresh:
            return Grade.B
        else:
            return Grade.C


# ── High-Level Scorer Facade ──


class Scorer:
    """
    Convenience facade that reads raw result JSON files from *results_dir*,
    feeds them through :class:`CompositeScorer`, and writes report files
    (Markdown / JSON) into *output_dir*.

    Usage::

        scorer = Scorer(results_dir="./results", output_dir="./reports")
        scorer.score_all()
        scorer.generate_report(fmt="both")   # "json" | "md" | "both"
    """

    def __init__(
        self,
        results_dir: str | Path = "./results",
        output_dir: str | Path = "./reports",
        config: ScorerConfig | None = None,
    ) -> None:
        self.results_dir = Path(results_dir)
        self.output_dir = Path(output_dir)
        self._engine = CompositeScorer(config)
        self._report: EvaluationReport | None = None
        self._composite_data: dict[str, Any] | None = None

    # ── Public API ──

    def score_all(self) -> EvaluationReport:
        """Load results from *results_dir*, score, and cache the report."""
        composite_path = self.results_dir / "composite_report.json"
        if composite_path.exists():
            self._composite_data = _load_json(composite_path)
        else:
            self._composite_data = {}

        dimensions = self._load_dimensions()
        stress = self._load_stress_results()
        zero_hall = self._detect_zero_hallucination()

        self._report = self._engine.score(
            dimensions, stress, zero_hallucination=zero_hall
        )
        self._report.external_benchmark_scorecard = dict(
            (self._composite_data or {}).get("external_benchmark_scorecard", {})
        )
        return self._report

    def generate_report(self, fmt: str = "both") -> list[Path]:
        """Write report files to *output_dir*. fmt: "json" | "md" | "both"."""
        if self._report is None:
            self.score_all()
        assert self._report is not None

        self.output_dir.mkdir(parents=True, exist_ok=True)
        latest = self.output_dir / "latest"
        latest.mkdir(parents=True, exist_ok=True)

        written: list[Path] = []

        if fmt in ("json", "both"):
            p = latest / "evaluation_report.json"
            p.write_text(
                _json_dumps(self._report.to_dict()),
                encoding="utf-8",
            )
            written.append(p)

        if fmt in ("md", "both"):
            p = latest / "evaluation_report.md"
            p.write_text(
                self._render_markdown(self._report),
                encoding="utf-8",
            )
            written.append(p)

        return written

    # ── Private Helpers ──

    def _load_dimensions(self) -> list[DimensionScore]:
        data = self._composite_data or {}
        dims_raw = data.get("dimensions", {})
        weights = self._engine.config.dimension_weights

        # Map composite_report dimension names → config weight keys
        _NAME_MAP: dict[str, str] = {
            "Memory Architecture": "capacity",
            "Long-Context Engineering": "fidelity",
            "Hallucination Control": "hallucination",
            "Operational": "operational",
        }

        dimensions: list[DimensionScore] = []
        for dim_name, dim_data in dims_raw.items():
            key: str = (
                _NAME_MAP[dim_name] if dim_name in _NAME_MAP else dim_name.lower()
            )
            weight: float = weights.get(key, 0.25)
            score: float = dim_data.get("score", 0.0)
            sub_scores: dict[str, float] = {
                k: v.get("normalized", v.get("raw", 0.0))
                for k, v in dim_data.get("sub_metrics", {}).items()
            }
            dimensions.append(
                DimensionScore(
                    name=key,
                    score=score,
                    weight=weight,
                    sub_scores=sub_scores,
                )
            )

        # If no composite data, produce zero-score defaults so scorer still works
        if not dimensions:
            for key, w in weights.items():
                dimensions.append(DimensionScore(name=key, score=0.0, weight=w))

        return dimensions

    def _load_stress_results(self) -> list[StressTestResult]:
        data = self._composite_data or {}
        stress_raw = data.get("stress_tests", {})

        _SEVERITY_MAP: dict[str, str] = {
            "Critical": "critical",
            "High": "high",
            "Normal": "normal",
        }

        results: list[StressTestResult] = []
        for test_id in ("ST-01", "ST-02", "ST-03", "ST-04"):
            if test_id in stress_raw:
                entry = stress_raw[test_id]
                results.append(
                    StressTestResult(
                        test_id=test_id,
                        passed=entry.get("passed", False),
                        severity=_SEVERITY_MAP.get(
                            entry.get("severity", "normal"), "normal"
                        ),
                        score=entry.get("score", 0.0)
                        / max(entry.get("max_score", 100.0), 1),
                    )
                )
            else:
                st_file = self.results_dir / f"{test_id}_results.json"
                if st_file.exists():
                    entries = _load_json(st_file)
                    if isinstance(entries, list) and entries:
                        entry = entries[0]
                        results.append(
                            StressTestResult(
                                test_id=test_id,
                                passed=entry.get("passed", False),
                                severity="critical"
                                if test_id in ("ST-01", "ST-04")
                                else "high",
                                score=entry.get("total_score", 0.0) / 100.0,
                            )
                        )
        return results

    def _detect_zero_hallucination(self) -> bool:
        data = self._composite_data or {}
        dims = data.get("dimensions", {})
        hall = dims.get("Hallucination Control", {})
        sub = hall.get("sub_metrics", {})
        for metric in sub.values():
            raw = metric.get("raw", 1.0)
            if raw > 0:
                return False
        return bool(sub)

    @staticmethod
    def _render_markdown(report: EvaluationReport) -> str:
        lines: list[str] = []
        lines.append("# DeskClaw Evaluation Report\n")
        lines.append(f"**Grade: {report.grade.value}**\n")
        lines.append(f"- Composite Score: {report.composite_score:.4f}")
        lines.append(f"- Adjusted Score:  {report.adjusted_score:.4f}")
        lines.append(
            f"- Robustness Bonus: {'Yes' if report.robustness_bonus_applied else 'No'}"
        )
        lines.append(
            f"- Zero Hallucination: {'Yes' if report.zero_hallucination else 'No'}"
        )
        if report.grade_capped_by:
            lines.append(f"- Grade Capped By: {report.grade_capped_by}")
        lines.append("")

        lines.append("## Dimension Scores\n")
        lines.append("| Dimension | Score | Weight | Weighted |")
        lines.append("|-----------|------:|-------:|---------:|")
        for d in report.dimension_scores:
            lines.append(
                f"| {d.name} | {d.score:.4f} | {d.weight:.2f} | {d.weighted:.4f} |"
            )
        lines.append("")

        lines.append("## Stress Test Results\n")
        lines.append("| Test | Passed | Severity | Score |")
        lines.append("|------|--------|----------|------:|")
        for s in report.stress_results:
            passed_str = "✅" if s.passed else "❌"
            lines.append(
                f"| {s.test_id} | {passed_str} | {s.severity} | {s.score:.4f} |"
            )
        lines.append("")

        if report.external_benchmark_scorecard:
            lines.append("## External Benchmark Scorecard\n")
            lines.append("| Suite | Score | Passed | Confidence | Mode |")
            lines.append("|-------|------:|--------|-----------:|------|")
            for suite in sorted(report.external_benchmark_scorecard.keys()):
                row = report.external_benchmark_scorecard[suite]
                passed_str = "✅" if row.get("passed", False) else "❌"
                lines.append(
                    f"| {suite} | {float(row.get('normalized_score', 0.0)):.4f} | "
                    f"{passed_str} | {float(row.get('confidence', 0.0)):.2f} | "
                    f"{row.get('mode', 'unknown')} |"
                )
            lines.append("")

        return "\n".join(lines)
