#!/usr/bin/env python3
"""Compare two cell-output manifests and produce dual-format drift reports.

Takes manifests generated by ``capture_notebook_outputs.py`` plus the
run-level YAML artefacts (snapshot_grid, project_context, feature profiles)
and produces both Markdown (LLM-optimized) and HTML (human-optimized) reports.

Usage:
    python scripts/experiments/compare_exploration_runs.py \
        --old-manifest experiments/runs/baseline/cell_outputs.md \
        --new-manifest experiments/runs/current/cell_outputs.md \
        --old-run-dir experiments/runs/baseline \
        --new-run-dir experiments/runs/current \
        --output experiments/drift_report.md \
        --format both
"""

from __future__ import annotations

import argparse
import difflib
import html as html_mod
import io
import re
import sys
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import Any, Iterable, Optional

try:
    import yaml
except ImportError:
    yaml = None  # type: ignore[assignment]

try:
    import pandas as pd
except ImportError:
    pd = None  # type: ignore[assignment]

# Cell profiling support — lazy import to avoid hard dependency
try:
    from cell_profiling import CellProfileManifest, NotebookProfile, load_cell_profiles
except ImportError:
    try:
        # When invoked from a different working directory
        _here = Path(__file__).resolve().parent
        import importlib.util

        _spec = importlib.util.spec_from_file_location("cell_profiling", _here / "cell_profiling.py")
        _mod = importlib.util.module_from_spec(_spec)  # type: ignore[arg-type]
        _spec.loader.exec_module(_mod)  # type: ignore[union-attr]
        CellProfileManifest = _mod.CellProfileManifest
        NotebookProfile = _mod.NotebookProfile
        load_cell_profiles = _mod.load_cell_profiles
    except Exception:
        CellProfileManifest = None  # type: ignore[assignment,misc]
        NotebookProfile = None  # type: ignore[assignment,misc]
        load_cell_profiles = None  # type: ignore[assignment]

# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------

_DRIFT_THRESHOLD = 0.005
_REGRESSION_THRESHOLD = 0.01
_JUPYTER_DEFAULT_ID_RE = re.compile(r"^cell-\d+$")
_NB_HEADER_RE = re.compile(r"^# (\S+)\s*$", re.MULTILINE)
_CELL_HEADER_RE = re.compile(
    r"^## (?:code|config):\s*(\S+)\s+\(id:\s*([\w-]+)\)(?:\s+\(section:\s*([^\)]+)\))?",
    re.MULTILINE,
)
_SECTION_MAP_RE = re.compile(r"<!--\s*(\S+)\s+\((\d+)\s+cells,\s+(\d+)\s+with output\)\s*-->")
_CELL_CONTENT_MAX = 500


# ---------------------------------------------------------------------------
# Data Layer
# ---------------------------------------------------------------------------


class CellMatchStatus(str, Enum):
    UNCHANGED = "unchanged"
    MODIFIED = "modified"
    ADDED = "added"
    REMOVED = "removed"
    RENAMED = "renamed"


@dataclass(frozen=True)
class ManifestCell:
    name: str
    cell_id: str
    section: str
    content: str


@dataclass
class ParsedManifest:
    notebooks: dict[str, list[ManifestCell]]
    section_map_raw: str

    @property
    def notebook_names(self) -> set[str]:
        return set(self.notebooks)

    def cells_by_id(self, notebook: str) -> dict[str, ManifestCell]:
        return {c.cell_id: c for c in self.notebooks.get(notebook, [])}

    def cells_by_name(self, notebook: str) -> dict[str, ManifestCell]:
        return {c.name: c for c in self.notebooks.get(notebook, [])}


@dataclass(frozen=True)
class RunMetadata:
    sha: str
    version: str
    date: str

    @property
    def label(self) -> str:
        return f"{self.sha}  v{self.version}  ({self.date})"


@dataclass(frozen=True)
class CellMatch:
    old_cell: Optional[ManifestCell]
    new_cell: Optional[ManifestCell]
    status: CellMatchStatus
    diff_text: str


@dataclass
class NotebookDrift:
    notebook_name: str
    cell_matches: list[CellMatch]

    @property
    def has_changes(self) -> bool:
        return any(m.status != CellMatchStatus.UNCHANGED for m in self.cell_matches)

    @property
    def changed_matches(self) -> list[CellMatch]:
        return [m for m in self.cell_matches if m.status != CellMatchStatus.UNCHANGED]


@dataclass
class DriftResult:
    old_meta: RunMetadata
    new_meta: RunMetadata
    old_manifest: ParsedManifest
    new_manifest: ParsedManifest
    notebook_drifts: list[NotebookDrift]
    snapshot_grid_old: Optional[dict]
    snapshot_grid_new: Optional[dict]
    project_context_old: Optional[dict]
    project_context_new: Optional[dict]
    feature_profile_old: Optional[dict]
    feature_profile_new: Optional[dict]
    old_timings: dict[str, str]
    new_timings: dict[str, str]
    metric_tables: list = field(default_factory=list)
    old_cell_profiles: Optional[Any] = None  # CellProfileManifest when available
    new_cell_profiles: Optional[Any] = None


# ---------------------------------------------------------------------------
# Manifest Parsing
# ---------------------------------------------------------------------------


def parse_manifest(text: str) -> ParsedManifest:
    section_map_lines = []
    for line in text.split("\n"):
        if line.startswith("<!--"):
            section_map_lines.append(line)
        elif line.strip() and not line.startswith("<!--"):
            break
    section_map_raw = "\n".join(section_map_lines)

    notebooks: dict[str, list[ManifestCell]] = {}
    parts = _NB_HEADER_RE.split(text)
    i = 1
    while i < len(parts) - 1:
        nb_name = parts[i].strip()
        nb_body = parts[i + 1]
        i += 2
        cells: list[ManifestCell] = []
        cell_parts = _CELL_HEADER_RE.split(nb_body)
        j = 1
        while j < len(cell_parts) - 3:
            name = cell_parts[j]
            cell_id = cell_parts[j + 1]
            section = cell_parts[j + 2] or ""
            content = cell_parts[j + 3].strip()
            cells.append(ManifestCell(name=name, cell_id=cell_id, section=section, content=content))
            j += 4
        notebooks[nb_name] = cells
    return ParsedManifest(notebooks=notebooks, section_map_raw=section_map_raw)


# ---------------------------------------------------------------------------
# Cell Matching
# ---------------------------------------------------------------------------


def _is_jupyter_default_id(cell_id: str) -> bool:
    return bool(_JUPYTER_DEFAULT_ID_RE.match(cell_id))


def _compute_diff(old_content: str, new_content: str) -> str:
    return "\n".join(difflib.unified_diff(old_content.split("\n"), new_content.split("\n"), lineterm="", n=3))


def _pair_lists(
    old_list: list[tuple[int, ManifestCell]],
    new_list: list[tuple[int, ManifestCell]],
    matched_old: set[int],
    matched_new: set[int],
    results: list[CellMatch],
) -> None:
    for i in range(min(len(old_list), len(new_list))):
        oidx, oc = old_list[i]
        nidx, nc = new_list[i]
        matched_old.add(oidx)
        matched_new.add(nidx)
        if oc.content == nc.content:
            results.append(CellMatch(oc, nc, CellMatchStatus.RENAMED, ""))
        else:
            results.append(CellMatch(oc, nc, CellMatchStatus.MODIFIED, _compute_diff(oc.content, nc.content)))


def _positional_pair(
    leftover_old: list[tuple[int, ManifestCell]],
    leftover_new: list[tuple[int, ManifestCell]],
    matched_old: set[int],
    matched_new: set[int],
    results: list[CellMatch],
) -> None:
    new_sections = {c.section for _, c in leftover_new if c.section}
    old_sections = {c.section for _, c in leftover_old if c.section}
    shared_sections = old_sections & new_sections
    if shared_sections:
        for section in sorted(shared_sections):
            sec_old = [(i, c) for i, c in leftover_old if c.section == section and i not in matched_old]
            sec_new = [(i, c) for i, c in leftover_new if c.section == section and i not in matched_new]
            _pair_lists(sec_old, sec_new, matched_old, matched_new, results)
        remaining_old = [(i, c) for i, c in leftover_old if i not in matched_old]
        remaining_new = [(i, c) for i, c in leftover_new if i not in matched_new]
        _pair_lists(remaining_old, remaining_new, matched_old, matched_new, results)
    else:
        _pair_lists(leftover_old, leftover_new, matched_old, matched_new, results)


def match_cells(old_cells: list[ManifestCell], new_cells: list[ManifestCell]) -> list[CellMatch]:
    matched_old: set[int] = set()
    matched_new: set[int] = set()
    results: list[CellMatch] = []

    old_by_id: dict[str, tuple[int, ManifestCell]] = {}
    for idx, c in enumerate(old_cells):
        if not _is_jupyter_default_id(c.cell_id):
            old_by_id[c.cell_id] = (idx, c)

    for nidx, nc in enumerate(new_cells):
        if _is_jupyter_default_id(nc.cell_id):
            continue
        if nc.cell_id in old_by_id:
            oidx, oc = old_by_id[nc.cell_id]
            matched_old.add(oidx)
            matched_new.add(nidx)
            if oc.content == nc.content and oc.name == nc.name:
                results.append(CellMatch(oc, nc, CellMatchStatus.UNCHANGED, ""))
            elif oc.content == nc.content:
                results.append(CellMatch(oc, nc, CellMatchStatus.RENAMED, ""))
            else:
                diff = _compute_diff(oc.content, nc.content)
                status = CellMatchStatus.RENAMED if oc.name != nc.name else CellMatchStatus.MODIFIED
                results.append(CellMatch(oc, nc, status, diff))

    remaining_old: dict[str, tuple[int, ManifestCell]] = {}
    for idx, c in enumerate(old_cells):
        if idx not in matched_old:
            remaining_old[c.name] = (idx, c)

    for nidx, nc in enumerate(new_cells):
        if nidx in matched_new:
            continue
        if nc.name in remaining_old:
            oidx, oc = remaining_old.pop(nc.name)
            matched_old.add(oidx)
            matched_new.add(nidx)
            if oc.content == nc.content:
                results.append(CellMatch(oc, nc, CellMatchStatus.UNCHANGED, ""))
            else:
                results.append(CellMatch(oc, nc, CellMatchStatus.MODIFIED, _compute_diff(oc.content, nc.content)))

    leftover_old = [(idx, c) for idx, c in enumerate(old_cells) if idx not in matched_old]
    leftover_new = [(idx, c) for idx, c in enumerate(new_cells) if idx not in matched_new]
    _positional_pair(leftover_old, leftover_new, matched_old, matched_new, results)

    for idx, c in enumerate(old_cells):
        if idx not in matched_old:
            results.append(CellMatch(c, None, CellMatchStatus.REMOVED, ""))
    for idx, c in enumerate(new_cells):
        if idx not in matched_new:
            results.append(CellMatch(None, c, CellMatchStatus.ADDED, ""))

    return results


# ---------------------------------------------------------------------------
# YAML / Metric helpers
# ---------------------------------------------------------------------------


def _load_yaml(path: Path) -> Optional[dict]:
    if yaml is None or not path.exists():
        return None
    with open(path, encoding="utf-8") as f:
        return yaml.safe_load(f)


def _extract_html_tables(cell_content: str) -> list:
    if pd is None:
        return []
    tables = re.findall(r"<table[\s\S]*?</table>", cell_content, re.IGNORECASE)
    result = []
    for t in tables:
        try:
            result.extend(pd.read_html(io.StringIO(t)))
        except Exception:
            pass
    return result


def _find_metric_tables(manifest: ParsedManifest, nb_name: str = "08_baseline_experiments") -> list:
    cells = manifest.notebooks.get(nb_name, [])
    result = []
    for cell in cells:
        for df in _extract_html_tables(cell.content):
            cols = [c.lower() for c in df.columns]
            if any("auc" in c or "f1" in c for c in cols):
                result.append((cell.name, df))
    return result


def _read_run_metadata(run_dir: Optional[Path]) -> RunMetadata:
    if run_dir is None:
        return RunMetadata("unknown", "unknown", "unknown")
    meta = {"sha": "unknown", "version": "unknown", "date": "unknown"}
    for key, filename in [("sha", "git_sha.txt"), ("version", "version.txt"), ("date", "commit_date.txt")]:
        p = run_dir / filename
        if p.exists():
            meta[key] = p.read_text(encoding="utf-8").strip()
    if meta["sha"] != "unknown":
        meta["sha"] = meta["sha"][:7]
    return RunMetadata(**meta)


_LOG_OK_RE = re.compile(r"\s*\[?(\w+)\]?\s+.*?OK\s+\((\d+\.?\d*)s\)")
_LOG_FAIL_RE = re.compile(r"\s*---\s*Notebook:\s*(\w+)\s*\|.*?\((\d+\.?\d*)s\)\s*---")
_LOG_STATUS_RE = re.compile(r"\s*\[?(\w+)\]?\s+.*?(FAILED|SKIPPED)")


def _parse_timing_log(path: Path) -> dict[str, str]:
    if not path.exists():
        return {}
    timings: dict[str, str] = {}
    for line in path.read_text(encoding="utf-8").split("\n"):
        m = _LOG_OK_RE.match(line)
        if m:
            timings[m.group(1)] = m.group(2)
            continue
        m = _LOG_FAIL_RE.match(line)
        if m:
            timings[m.group(1)] = f"FAILED ({m.group(2)}s)"
            continue
        m = _LOG_STATUS_RE.match(line)
        if m:
            timings[m.group(1)] = m.group(2)
    return timings


def _notebook_has_errors(cells: list[ManifestCell]) -> bool:
    return any("### Error:" in c.content for c in cells)


def _build_metric_comparisons(old_manifest: ParsedManifest, new_manifest: ParsedManifest) -> list:
    old_tables = _find_metric_tables(old_manifest)
    new_tables = _find_metric_tables(new_manifest)
    if not old_tables and not new_tables:
        return []
    result = []
    for old_item, new_item in zip(old_tables or [(None, None)], new_tables or [(None, None)]):
        old_name, old_df = old_item if old_item else (None, None)
        new_name, new_df = new_item if new_item else (None, None)
        result.append({"old_name": old_name, "old_df": old_df, "new_name": new_name, "new_df": new_df})
    return result


# ---------------------------------------------------------------------------
# compute_drift_result
# ---------------------------------------------------------------------------


def compute_drift_result(
    old_manifest_path: Path,
    new_manifest_path: Path,
    old_run_dir: Optional[Path],
    new_run_dir: Optional[Path],
) -> DriftResult:
    old_m = parse_manifest(old_manifest_path.read_text(encoding="utf-8"))
    new_m = parse_manifest(new_manifest_path.read_text(encoding="utf-8"))
    old_meta = _read_run_metadata(old_run_dir)
    new_meta = _read_run_metadata(new_run_dir)

    all_nbs = sorted(old_m.notebook_names | new_m.notebook_names)
    notebook_drifts = []
    for nb in all_nbs:
        old_cells = old_m.notebooks.get(nb, [])
        new_cells = new_m.notebooks.get(nb, [])
        matches = match_cells(old_cells, new_cells)
        notebook_drifts.append(NotebookDrift(nb, matches))

    sg_old = sg_new = pc_old = pc_new = fp_old = fp_new = None
    old_timings: dict[str, str] = {}
    new_timings: dict[str, str] = {}
    if old_run_dir and new_run_dir:
        sg_old = _load_yaml(old_run_dir / "snapshot_grid.yaml")
        sg_new = _load_yaml(new_run_dir / "snapshot_grid.yaml")
        pc_old = _load_yaml(old_run_dir / "project_context.yaml")
        pc_new = _load_yaml(new_run_dir / "project_context.yaml")
        fp_old = _load_yaml(old_run_dir / "merged" / "exploration_feature_profile.yaml")
        fp_new = _load_yaml(new_run_dir / "merged" / "exploration_feature_profile.yaml")
        for run_dir, target in [(old_run_dir, "old_timings"), (new_run_dir, "new_timings")]:
            for base in [
                run_dir,
                run_dir / "notebooks",
                run_dir.parent.parent / "exploration_notebooks",
                run_dir.parent.parent,
            ]:
                log = base / "run_exploration.log"
                if log.exists():
                    if target == "old_timings":
                        old_timings = _parse_timing_log(log)
                    else:
                        new_timings = _parse_timing_log(log)
                    break

    # Load cell profiles if available
    old_cell_profiles = None
    new_cell_profiles = None
    if load_cell_profiles is not None:
        if old_run_dir:
            old_cell_profiles = load_cell_profiles(old_run_dir / "cell_profiles.json")
        if new_run_dir:
            new_cell_profiles = load_cell_profiles(new_run_dir / "cell_profiles.json")

    return DriftResult(
        old_meta=old_meta,
        new_meta=new_meta,
        old_manifest=old_m,
        new_manifest=new_m,
        notebook_drifts=notebook_drifts,
        snapshot_grid_old=sg_old,
        snapshot_grid_new=sg_new,
        project_context_old=pc_old,
        project_context_new=pc_new,
        feature_profile_old=fp_old,
        feature_profile_new=fp_new,
        old_timings=old_timings,
        new_timings=new_timings,
        metric_tables=_build_metric_comparisons(old_m, new_m),
        old_cell_profiles=old_cell_profiles,
        new_cell_profiles=new_cell_profiles,
    )


# ---------------------------------------------------------------------------
# Rendering Layer — Base
# ---------------------------------------------------------------------------


class DriftReportRenderer(ABC):
    REPORT_SECTIONS = [
        "title",
        "summary",
        "section_map_changes",
        "model_metrics",
        "snapshot_grid",
        "project_context",
        "feature_profile",
        "timing",
        "cell_profiling",
        "cell_by_cell_diff",
    ]
    SNAPSHOT_GRID_KEYS = [
        "grid_start",
        "grid_end",
        "cadence_interval",
        "observation_window_days",
        "purge_gap_days",
        "label_window_days",
        "max_grid_dates",
    ]
    PROJECT_CONTEXT_KEYS = [
        "prediction_horizons",
        "recent_window_days",
        "observation_window_days",
        "purge_gap_days",
        "label_window_days",
        "split_strategy",
        "cadence_interval",
        "temporal_posture",
    ]

    def __init__(self, drift: DriftResult):
        self.drift = drift

    def render(self) -> str:
        return self.join_sections(getattr(self, f"render_{s}")() for s in self.REPORT_SECTIONS)

    def write(self, path: Path) -> None:
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_text(self.render(), encoding="utf-8")

    @abstractmethod
    def join_sections(self, sections: Iterable[str]) -> str: ...

    @abstractmethod
    def render_title(self) -> str: ...

    @abstractmethod
    def render_summary(self) -> str: ...

    @abstractmethod
    def render_section_map_changes(self) -> str: ...

    @abstractmethod
    def render_model_metrics(self) -> str: ...

    @abstractmethod
    def render_snapshot_grid(self) -> str: ...

    @abstractmethod
    def render_project_context(self) -> str: ...

    @abstractmethod
    def render_feature_profile(self) -> str: ...

    @abstractmethod
    def render_timing(self) -> str: ...

    @abstractmethod
    def render_cell_profiling(self) -> str: ...

    @abstractmethod
    def render_cell_by_cell_diff(self) -> str: ...

    def yaml_diff_rows(
        self, old: Optional[dict], new: Optional[dict], keys: list[str]
    ) -> list[tuple[str, str, str, bool]]:
        rows = []
        for key in keys:
            old_val = str(old.get(key, "N/A")) if old else "N/A"
            new_val = str(new.get(key, "N/A")) if new else "N/A"
            rows.append((key, old_val, new_val, old_val != new_val))
        return rows

    def feature_profile_sets(self) -> tuple[set[str], set[str], set[str]]:
        old_f = set(self.drift.feature_profile_old.keys()) if self.drift.feature_profile_old else set()
        new_f = set(self.drift.feature_profile_new.keys()) if self.drift.feature_profile_new else set()
        return new_f - old_f, old_f - new_f, old_f & new_f

    def section_map_parsed(self) -> dict[str, tuple[tuple[int, int], tuple[int, int]]]:
        old_parsed = {
            m.group(1): (int(m.group(2)), int(m.group(3)))
            for line in self.drift.old_manifest.section_map_raw.split("\n")
            for m in [_SECTION_MAP_RE.match(line.strip())]
            if m
        }
        new_parsed = {
            m.group(1): (int(m.group(2)), int(m.group(3)))
            for line in self.drift.new_manifest.section_map_raw.split("\n")
            for m in [_SECTION_MAP_RE.match(line.strip())]
            if m
        }
        all_nbs = sorted(set(old_parsed) | set(new_parsed))
        return {nb: (old_parsed.get(nb, (0, 0)), new_parsed.get(nb, (0, 0))) for nb in all_nbs}

    def notebook_run_status(self, nb_name: str) -> tuple[str, str]:
        d = self.drift
        old_cells = d.old_manifest.notebooks.get(nb_name, [])
        new_cells = d.new_manifest.notebooks.get(nb_name, [])
        old_timing = d.old_timings.get(nb_name, "")
        new_timing = d.new_timings.get(nb_name, "")

        def _status(timing: str, cells: list[ManifestCell]) -> str:
            if "FAILED" in timing:
                return "FAILED"
            if "SKIPPED" in timing:
                return "SKIPPED"
            if _notebook_has_errors(cells):
                return "ERROR"
            if timing:
                return "OK"
            if cells:
                return "OK"
            return "N/A"

        return _status(old_timing, old_cells), _status(new_timing, new_cells)

    def notebook_timing(self, nb_name: str) -> tuple[str, str]:
        def _seconds(raw: str) -> str:
            if not raw:
                return ""
            m = re.match(r"[\d.]+", raw)
            if m:
                secs = float(m.group())
                return f"{secs:.0f}s" if secs < 60 else f"{secs / 60:.1f}m"
            if "FAILED" in raw:
                m2 = re.search(r"([\d.]+)s", raw)
                return m2.group() if m2 else ""
            return ""

        return _seconds(self.drift.old_timings.get(nb_name, "")), _seconds(self.drift.new_timings.get(nb_name, ""))

    # --- Cell profile helpers ---

    def _cell_profile_index(self, profiles: Optional[Any], nb_name: str) -> dict[str, Any]:
        """Build cell_id -> profile entry index for a notebook."""
        if profiles is None or not hasattr(profiles, "notebooks"):
            return {}
        nb_profile = profiles.notebooks.get(nb_name)
        if nb_profile is None:
            return {}
        return {c.cell_id: c for c in nb_profile.cells}

    def _cell_profile_name_index(self, profiles: Optional[Any], nb_name: str) -> dict[str, Any]:
        """Build cell_name -> profile entry index (fallback matching)."""
        if profiles is None or not hasattr(profiles, "notebooks"):
            return {}
        nb_profile = profiles.notebooks.get(nb_name)
        if nb_profile is None:
            return {}
        return {c.cell_name: c for c in nb_profile.cells}

    def _nb_profile_total(self, profiles: Optional[Any], nb_name: str) -> Optional[float]:
        """Get total_elapsed for a notebook, or None."""
        if profiles is None or not hasattr(profiles, "notebooks"):
            return None
        nb_profile = profiles.notebooks.get(nb_name)
        return nb_profile.total_elapsed if nb_profile else None

    def _cell_timing_annotation(self, cell_name: str, cell_id: str, nb_name: str) -> str:
        """Return timing annotation string for a cell, e.g. '3.2s -> 2.8s (-0.4s)'."""
        old_idx = self._cell_profile_index(self.drift.old_cell_profiles, nb_name)
        new_idx = self._cell_profile_index(self.drift.new_cell_profiles, nb_name)
        # Try by cell_id first, then by name
        old_entry = old_idx.get(cell_id)
        new_entry = new_idx.get(cell_id)
        if old_entry is None:
            old_name_idx = self._cell_profile_name_index(self.drift.old_cell_profiles, nb_name)
            old_entry = old_name_idx.get(cell_name)
        if new_entry is None:
            new_name_idx = self._cell_profile_name_index(self.drift.new_cell_profiles, nb_name)
            new_entry = new_name_idx.get(cell_name)
        if old_entry is None and new_entry is None:
            return ""
        parts = []
        old_sec = old_entry.elapsed_sec if old_entry else None
        new_sec = new_entry.elapsed_sec if new_entry else None
        if old_sec is not None and new_sec is not None:
            delta = new_sec - old_sec
            parts.append(f"{self._fmt_sec(old_sec)} \u2192 {self._fmt_sec(new_sec)} ({self._fmt_delta(delta)})")
        elif old_sec is not None:
            parts.append(f"old: {self._fmt_sec(old_sec)}")
        elif new_sec is not None:
            parts.append(f"new: {self._fmt_sec(new_sec)}")
        return " \u2014 ".join(parts) if parts else ""

    def _nb_timing_annotation(self, nb_name: str) -> str:
        """Return notebook-level profile timing, e.g. '45.2s -> 38.1s (-7.1s)'."""
        old_total = self._nb_profile_total(self.drift.old_cell_profiles, nb_name)
        new_total = self._nb_profile_total(self.drift.new_cell_profiles, nb_name)
        if old_total is None and new_total is None:
            return ""
        if old_total is not None and new_total is not None:
            delta = new_total - old_total
            return f"{self._fmt_sec(old_total)} \u2192 {self._fmt_sec(new_total)} ({self._fmt_delta(delta)})"
        if old_total is not None:
            return f"old: {self._fmt_sec(old_total)}"
        return f"new: {self._fmt_sec(new_total)}"  # type: ignore[arg-type]

    @staticmethod
    def _fmt_sec(seconds: float) -> str:
        if seconds < 60:
            return f"{seconds:.1f}s"
        return f"{seconds / 60:.1f}m"

    @staticmethod
    def _fmt_delta(delta: float) -> str:
        if abs(delta) < 60:
            return f"{delta:+.1f}s"
        return f"{delta / 60:+.1f}m"

    def grid_dates_summary(self) -> Optional[str]:
        sg_old, sg_new = self.drift.snapshot_grid_old, self.drift.snapshot_grid_new
        if not sg_old and not sg_new:
            return None
        old_dates = sg_old.get("grid_dates", []) if sg_old else []
        new_dates = sg_new.get("grid_dates", []) if sg_new else []
        parts = [f"Grid dates: {len(old_dates)} (old) vs {len(new_dates)} (new)"]
        if old_dates and new_dates and (old_dates[0] != new_dates[0] or old_dates[-1] != new_dates[-1]):
            parts.append(f"Range: [{old_dates[0]} .. {old_dates[-1]}] -> [{new_dates[0]} .. {new_dates[-1]}]")
        return "\n".join(parts)

    def metric_comparison_data(self) -> list[dict[str, Any]]:
        enriched = []
        for item in self.drift.metric_tables:
            old_df, new_df = item["old_df"], item["new_df"]
            entry: dict[str, Any] = {**item, "merged_df": None, "warnings": []}
            if old_df is not None and new_df is not None and pd is not None:
                model_col = next((c for c in old_df.columns if "model" in c.lower() or "name" in c.lower()), None)
                if model_col:
                    merged = old_df.merge(new_df, on=model_col, suffixes=(" (old)", " (new)"), how="outer")
                    for base_col in old_df.columns:
                        if base_col == model_col:
                            continue
                        old_c, new_c = f"{base_col} (old)", f"{base_col} (new)"
                        if old_c in merged.columns and new_c in merged.columns:
                            try:
                                merged[f"{base_col} delta"] = (
                                    merged[new_c].astype(float) - merged[old_c].astype(float)
                                ).round(4)
                            except (ValueError, TypeError):
                                pass
                    entry["merged_df"] = merged
                    for base_col in old_df.columns:
                        delta_col = f"{base_col} delta"
                        if delta_col in merged.columns:
                            try:
                                min_delta = merged[delta_col].min()
                                if min_delta < -_REGRESSION_THRESHOLD:
                                    entry["warnings"].append(f"REGRESSION: {base_col} dropped by {abs(min_delta):.4f}")
                                elif min_delta < -_DRIFT_THRESHOLD:
                                    entry["warnings"].append(f"DRIFT: {base_col} shifted by {abs(min_delta):.4f}")
                            except (ValueError, TypeError):
                                pass
            enriched.append(entry)
        return enriched


# ---------------------------------------------------------------------------
# Markdown Renderer
# ---------------------------------------------------------------------------


class MarkdownDriftRenderer(DriftReportRenderer):
    def join_sections(self, sections: Iterable[str]) -> str:
        return "\n\n".join(s for s in sections if s) + "\n"

    def render_title(self) -> str:
        return f"# Drift Report: {self.drift.old_meta.sha} -> {self.drift.new_meta.sha}"

    def render_summary(self) -> str:
        d = self.drift
        old_nb = len(d.old_manifest.notebooks)
        new_nb = len(d.new_manifest.notebooks)
        return (
            f"## Summary\n"
            f"- Old: {d.old_meta.label}\n"
            f"- New: {d.new_meta.label}\n"
            f"- Notebooks captured: {old_nb} (old), {new_nb} (new)"
        )

    def _status_icon(self, status: str) -> str:
        return {"OK": "\u2705", "FAILED": "\u274c", "ERROR": "\u26a0\ufe0f", "SKIPPED": "\u23ed", "N/A": "\u2014"}.get(
            status, status
        )

    def render_section_map_changes(self) -> str:
        rows = [
            "| Notebook | Old status | New status | Old cells (outputs) | New cells (outputs) | Cell delta | Output delta |",
            "|----------|-----------|-----------|--------------------|--------------------|------------|--------------|",
        ]
        for nb, ((oc, oo), (nc, no)) in self.section_map_parsed().items():
            old_st, new_st = self.notebook_run_status(nb)
            cd = nc - oc if oc or nc else "N/A"
            od = no - oo if oc or nc else "N/A"
            rows.append(
                f"| {nb} | {self._status_icon(old_st)} {old_st} | {self._status_icon(new_st)} {new_st} | {oc} ({oo}) | {nc} ({no}) | {cd} | {od} |"
            )
        return "## Section Map Changes\n\n" + "\n".join(rows)

    def render_model_metrics(self) -> str:
        data = self.metric_comparison_data()
        if not data:
            return "## Model Metrics (NB08)\n\n_No metric tables found in either run._"
        parts = ["## Model Metrics (NB08)"]
        for entry in data:
            if entry["merged_df"] is not None:
                parts.append(entry["merged_df"].to_markdown(index=False))
                for w in entry["warnings"]:
                    parts.append(f"\n\u26a0 **{w}**")
            elif entry["old_df"] is not None:
                parts.append(f"**Old only ({entry['old_name']}):**\n{entry['old_df'].to_markdown(index=False)}")
            elif entry["new_df"] is not None:
                parts.append(f"**New only ({entry['new_name']}):**\n{entry['new_df'].to_markdown(index=False)}")
        return "\n\n".join(parts)

    def _render_yaml_section(
        self, title: str, old_d: Optional[dict], new_d: Optional[dict], keys: list[str], extra: str = ""
    ) -> str:
        if old_d is None and new_d is None:
            return f"## {title}\n\n_Not available._"
        rows = ["| Parameter | Old | New | Changed |", "|-----------|-----|-----|---------|"]
        for key, ov, nv, changed in self.yaml_diff_rows(old_d, new_d, keys):
            flag = "\u26a0" if changed else ""
            rows.append(f"| {key} | {ov} | {nv} | {flag} |")
        result = f"## {title}\n\n" + "\n".join(rows)
        if extra:
            result += f"\n\n{extra}"
        return result

    def render_snapshot_grid(self) -> str:
        return self._render_yaml_section(
            "Snapshot Grid",
            self.drift.snapshot_grid_old,
            self.drift.snapshot_grid_new,
            self.SNAPSHOT_GRID_KEYS,
            self.grid_dates_summary() or "",
        )

    def render_project_context(self) -> str:
        return self._render_yaml_section(
            "Project Context / Intent",
            self.drift.project_context_old,
            self.drift.project_context_new,
            self.PROJECT_CONTEXT_KEYS,
        )

    def render_feature_profile(self) -> str:
        added, removed, common = self.feature_profile_sets()
        if not added and not removed and not common:
            return "## Feature Profile\n\n_Not found in either run._"
        parts = [
            "## Feature Profile",
            f"- Old features: {len(removed) + len(common)}",
            f"- New features: {len(added) + len(common)}",
            f"- Added: {len(added)}, Removed: {len(removed)}, Common: {len(common)}",
        ]
        if added:
            parts.append(f"\n**Added:** {', '.join(sorted(added)[:20])}")
        if removed:
            parts.append(f"\n**Removed:** {', '.join(sorted(removed)[:20])}")
        return "\n".join(parts)

    def render_timing(self) -> str:
        ot, nt = self.drift.old_timings, self.drift.new_timings
        if not ot and not nt:
            return "## Per-Notebook Timing\n\n_No timing logs found._"
        rows = ["| Notebook | Old | New | Delta |", "|----------|-----|-----|-------|"]
        for nb in dict.fromkeys(list(ot) + list(nt)):
            ov, nv = ot.get(nb, "N/A"), nt.get(nb, "N/A")
            try:
                delta = f"{float(nv) - float(ov):+.1f}s"
            except (ValueError, TypeError):
                delta = "N/A"
            rows.append(f"| {nb} | {ov} | {nv} | {delta} |")
        table = "\n".join(rows)
        return f"## Per-Notebook Timing\n\n<details>\n<summary>Show timing table</summary>\n\n{table}\n\n</details>"

    def render_cell_profiling(self) -> str:
        op, np_ = self.drift.old_cell_profiles, self.drift.new_cell_profiles
        if op is None and np_ is None:
            return "## Per-Cell Profiling\n\n_No cell profiles found._"
        parts = ["## Per-Cell Profiling"]
        # Determine which columns have data
        has_spark = False
        has_mem = False
        for profiles in [op, np_]:
            if profiles is None:
                continue
            for nb_prof in profiles.notebooks.values():
                for c in nb_prof.cells:
                    if c.spark_jobs is not None:
                        has_spark = True
                    if c.peak_memory_mb is not None:
                        has_mem = True
        # Cross-environment note
        if op and np_ and op.environment != np_.environment:
            parts.append(
                f"\n> _Cross-environment comparison ({op.environment} vs {np_.environment}) "
                f"\u2014 elapsed times reflect different hardware._\n"
            )

        all_nbs = sorted(set(list(op.notebooks if op else {}) + list(np_.notebooks if np_ else {})))
        for nb_name in all_nbs:
            nb_annotation = self._nb_timing_annotation(nb_name)
            summary = nb_name
            if nb_annotation:
                summary += f" ({nb_annotation})"

            cols = ["Cell Name", "Cell ID", "Old", "New", "Delta"]
            if has_spark:
                cols.append("Spark Jobs")
            if has_mem:
                cols.append("Mem MB")
            header_row = "| " + " | ".join(cols) + " |"
            sep_row = "|" + "|".join("---" for _ in cols) + "|"

            old_idx = self._cell_profile_index(op, nb_name)
            new_idx = self._cell_profile_index(np_, nb_name)
            old_name_idx = self._cell_profile_name_index(op, nb_name)
            new_name_idx = self._cell_profile_name_index(np_, nb_name)

            # Collect all cells in order (new side order, then old-only)
            seen_ids: set[str] = set()
            ordered_cells: list[tuple[str, str]] = []  # (cell_name, cell_id)
            for profiles in [np_, op]:
                if profiles is None:
                    continue
                nb_prof = profiles.notebooks.get(nb_name)
                if nb_prof is None:
                    continue
                for c in nb_prof.cells:
                    if c.cell_id not in seen_ids:
                        seen_ids.add(c.cell_id)
                        ordered_cells.append((c.cell_name, c.cell_id))

            rows = [header_row, sep_row]
            for cell_name, cell_id in ordered_cells:
                old_e = old_idx.get(cell_id) or old_name_idx.get(cell_name)
                new_e = new_idx.get(cell_id) or new_name_idx.get(cell_name)
                old_sec = old_e.elapsed_sec if old_e else None
                new_sec = new_e.elapsed_sec if new_e else None

                old_str = self._fmt_sec(old_sec) if old_sec is not None else "\u2014"
                new_str = self._fmt_sec(new_sec) if new_sec is not None else "\u2014"
                if old_sec is not None and new_sec is not None:
                    delta_str = self._fmt_delta(new_sec - old_sec)
                else:
                    delta_str = "\u2014"

                row_parts = [cell_name, cell_id[:8], old_str, new_str, delta_str]
                if has_spark:
                    old_sj = str(old_e.spark_jobs) if old_e and old_e.spark_jobs is not None else "\u2014"
                    new_sj = str(new_e.spark_jobs) if new_e and new_e.spark_jobs is not None else "\u2014"
                    row_parts.append(f"{old_sj}/{new_sj}")
                if has_mem:
                    old_mm = f"{old_e.peak_memory_mb:.1f}" if old_e and old_e.peak_memory_mb is not None else "\u2014"
                    new_mm = f"{new_e.peak_memory_mb:.1f}" if new_e and new_e.peak_memory_mb is not None else "\u2014"
                    row_parts.append(f"{old_mm}/{new_mm}")
                rows.append("| " + " | ".join(row_parts) + " |")
            table = "\n".join(rows)
            parts.append(f"<details>\n<summary>{summary}</summary>\n\n{table}\n\n</details>")
        return "\n\n".join(parts)

    def render_cell_by_cell_diff(self) -> str:
        parts = ["## Cell-by-Cell Diff"]
        any_diff = False
        for nd in self.drift.notebook_drifts:
            old_st, new_st = self.notebook_run_status(nd.notebook_name)
            old_time, new_time = self.notebook_timing(nd.notebook_name)
            old_part = (
                f"{self._status_icon(old_st)} {old_st} ({old_time})"
                if old_time
                else f"{self._status_icon(old_st)} {old_st}"
            )
            new_part = (
                f"{self._status_icon(new_st)} {new_st} ({new_time})"
                if new_time
                else f"{self._status_icon(new_st)} {new_st}"
            )
            status_line = f"Old: {old_part} | New: {new_part}"
            if not nd.has_changes:
                continue
            any_diff = True
            nb_timing = self._nb_timing_annotation(nd.notebook_name)
            nb_header = f"### {nd.notebook_name}"
            if nb_timing:
                nb_header += f" ({nb_timing})"
            nb_parts = [f"---\n\n{nb_header}\n\n> {status_line}"]
            for m in nd.changed_matches:
                cell = m.new_cell or m.old_cell
                name = cell.name
                cell_id = cell.cell_id
                label = m.status.value.upper()
                timing_ann = self._cell_timing_annotation(name, cell_id, nd.notebook_name)
                cell_header = f"#### {name} ({label})"
                if timing_ann:
                    cell_header += f" \u2014 {timing_ann}"
                nb_parts.append(cell_header)
                if m.status == CellMatchStatus.ADDED:
                    nb_parts.append(f"```\n{m.new_cell.content[:_CELL_CONTENT_MAX]}\n```")
                elif m.status == CellMatchStatus.REMOVED:
                    nb_parts.append(f"```\n{m.old_cell.content[:_CELL_CONTENT_MAX]}\n```")
                elif m.diff_text:
                    nb_parts.append(f"```diff\n{m.diff_text}\n```")
                elif m.status == CellMatchStatus.RENAMED:
                    nb_parts.append(f"_Renamed from `{m.old_cell.name}` to `{m.new_cell.name}`_")
            parts.append("\n\n".join(nb_parts))
        if not any_diff:
            parts.append("_No cell-level differences detected._")
        return "\n\n".join(parts)


# ---------------------------------------------------------------------------
# HTML Renderer
# ---------------------------------------------------------------------------


class HtmlDriftRenderer(DriftReportRenderer):
    def join_sections(self, sections: Iterable[str]) -> str:
        body = "\n".join(s for s in sections if s)
        return f'<!DOCTYPE html>\n<html lang="en">\n<head>\n<meta charset="utf-8">\n<title>Drift Report: {self.drift.old_meta.sha} vs {self.drift.new_meta.sha}</title>\n{self._render_style()}\n</head>\n<body>\n{body}\n</body>\n</html>'

    def _render_style(self) -> str:
        return """<style>
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif; max-width: 1400px; margin: 0 auto; padding: 20px; color: #24292f; background: #fff; }
h1 { border-bottom: 1px solid #d0d7de; padding-bottom: 8px; }
h2 { border-bottom: 1px solid #d0d7de; padding-bottom: 4px; margin-top: 32px; }
table { border-collapse: collapse; width: 100%; margin: 8px 0; }
th, td { border: 1px solid #d0d7de; padding: 6px 12px; text-align: left; }
th { background: #f6f8fa; }
details { margin: 8px 0; }
details.nb-section { margin-top: 32px; }
details.nb-section > summary { font-size: 15px; padding: 10px 12px; background: #eaeef2; border-radius: 6px; }
details.nb-section > summary:hover { background: #d8dee4; }
summary { cursor: pointer; font-weight: 600; padding: 8px; background: transparent; border-radius: 6px; }
summary:hover { background: #f6f8fa; }
.cell-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 8px; margin: 8px 0; }
.cell-box { border: 1px solid #d0d7de; border-radius: 6px; padding: 8px; max-height: 400px; overflow-y: auto; font-size: 13px; }
.cell-box pre { margin: 0; white-space: pre-wrap; word-break: break-word; }
.cell-label { font-weight: 600; font-size: 12px; margin-bottom: 4px; }
.status-added { background: #dafbe1; border-color: #82e596; }
.status-removed { background: #ffebe9; border-color: #ff8182; }
.status-modified { background: #fff8c5; border-color: #d4a72c; }
.status-renamed { background: #ddf4ff; border-color: #54aeff; }
.diff-line { font-family: monospace; font-size: 12px; }
.diff-removed { background: #ffebe9; color: #82071e; }
.diff-added { background: #dafbe1; color: #116329; }
.diff-hunk { color: #656d76; }
.warning { color: #9a6700; font-weight: 600; }
.badge { display: inline-block; padding: 2px 8px; border-radius: 12px; font-size: 12px; font-weight: 600; }
.badge-added { background: #dafbe1; color: #116329; }
.badge-removed { background: #ffebe9; color: #82071e; }
.badge-modified { background: #fff8c5; color: #9a6700; }
.badge-renamed { background: #ddf4ff; color: #0969da; }
.badge-unchanged { background: #f6f8fa; color: #656d76; }
.embed-table { overflow-x: auto; }
.embed-table table { font-size: 12px; }
.section-tag { background: #eaeef2; color: #656d76; padding: 1px 6px; border-radius: 8px; font-size: 11px; font-weight: normal; }
.timing-annotation { color: #656d76; font-weight: normal; font-size: 13px; margin-left: 6px; }
</style>"""

    def _esc(self, text: str) -> str:
        return html_mod.escape(text)

    def _badge(self, status: CellMatchStatus) -> str:
        return f'<span class="badge badge-{status.value}">{status.value}</span>'

    def render_title(self) -> str:
        return (
            f"<h1>Drift Report: {self._esc(self.drift.old_meta.sha)} \u2192 {self._esc(self.drift.new_meta.sha)}</h1>"
        )

    def render_summary(self) -> str:
        d = self.drift
        old_nb = len(d.old_manifest.notebooks)
        new_nb = len(d.new_manifest.notebooks)
        return (
            f"<h2>Summary</h2>\n<ul>\n"
            f"<li><strong>Old:</strong> {self._esc(d.old_meta.label)}</li>\n"
            f"<li><strong>New:</strong> {self._esc(d.new_meta.label)}</li>\n"
            f"<li>Notebooks captured: {old_nb} (old), {new_nb} (new)</li>\n</ul>"
        )

    def _status_html(self, status: str) -> str:
        cls = {
            "OK": "badge-added",
            "FAILED": "badge-removed",
            "ERROR": "badge-modified",
            "SKIPPED": "badge-unchanged",
        }.get(status, "badge-unchanged")
        return f'<span class="badge {cls}">{self._esc(status)}</span>'

    def render_section_map_changes(self) -> str:
        rows = []
        for nb, ((oc, oo), (nc, no)) in self.section_map_parsed().items():
            old_st, new_st = self.notebook_run_status(nb)
            cd = nc - oc if oc or nc else 0
            od = no - oo if oc or nc else 0
            cd_cls = ' class="diff-added"' if cd > 0 else (' class="diff-removed"' if cd < 0 else "")
            od_cls = ' class="diff-added"' if od > 0 else (' class="diff-removed"' if od < 0 else "")
            rows.append(
                f"<tr><td>{self._esc(nb)}</td><td>{self._status_html(old_st)}</td><td>{self._status_html(new_st)}</td>"
                f"<td>{oc} ({oo})</td><td>{nc} ({no})</td>"
                f"<td{cd_cls}>{cd:+d}</td><td{od_cls}>{od:+d}</td></tr>"
            )
        return (
            "<h2>Section Map Changes</h2>\n<table>\n"
            "<tr><th>Notebook</th><th>Old status</th><th>New status</th><th>Old cells (outputs)</th><th>New cells (outputs)</th>"
            "<th>Cell delta</th><th>Output delta</th></tr>\n" + "\n".join(rows) + "\n</table>"
        )

    def render_model_metrics(self) -> str:
        data = self.metric_comparison_data()
        if not data:
            return "<h2>Model Metrics (NB08)</h2>\n<p><em>No metric tables found in either run.</em></p>"
        parts = ["<h2>Model Metrics (NB08)</h2>"]
        for entry in data:
            if entry["merged_df"] is not None:
                parts.append(f'<div class="embed-table">{entry["merged_df"].to_html(index=False)}</div>')
                for w in entry["warnings"]:
                    parts.append(f'<p class="warning">\u26a0 {self._esc(w)}</p>')
            elif entry["old_df"] is not None:
                parts.append(f"<p><strong>Old only ({self._esc(str(entry['old_name']))}):</strong></p>")
                parts.append(f'<div class="embed-table">{entry["old_df"].to_html(index=False)}</div>')
            elif entry["new_df"] is not None:
                parts.append(f"<p><strong>New only ({self._esc(str(entry['new_name']))}):</strong></p>")
                parts.append(f'<div class="embed-table">{entry["new_df"].to_html(index=False)}</div>')
        return "\n".join(parts)

    def _render_yaml_table(
        self, title: str, old_d: Optional[dict], new_d: Optional[dict], keys: list[str], extra: str = ""
    ) -> str:
        if old_d is None and new_d is None:
            return f"<h2>{self._esc(title)}</h2>\n<p><em>Not available.</em></p>"
        rows = []
        for key, ov, nv, changed in self.yaml_diff_rows(old_d, new_d, keys):
            cls = ' class="status-modified"' if changed else ""
            rows.append(f"<tr{cls}><td>{self._esc(key)}</td><td>{self._esc(ov)}</td><td>{self._esc(nv)}</td></tr>")
        table = (
            f"<h2>{self._esc(title)}</h2>\n<table>\n"
            "<tr><th>Parameter</th><th>Old</th><th>New</th></tr>\n" + "\n".join(rows) + "\n</table>"
        )
        if extra:
            table += f"\n<p>{self._esc(extra)}</p>"
        return table

    def render_snapshot_grid(self) -> str:
        return self._render_yaml_table(
            "Snapshot Grid",
            self.drift.snapshot_grid_old,
            self.drift.snapshot_grid_new,
            self.SNAPSHOT_GRID_KEYS,
            self.grid_dates_summary() or "",
        )

    def render_project_context(self) -> str:
        return self._render_yaml_table(
            "Project Context / Intent",
            self.drift.project_context_old,
            self.drift.project_context_new,
            self.PROJECT_CONTEXT_KEYS,
        )

    def render_feature_profile(self) -> str:
        added, removed, common = self.feature_profile_sets()
        if not added and not removed and not common:
            return "<h2>Feature Profile</h2>\n<p><em>Not found in either run.</em></p>"
        parts = [
            "<h2>Feature Profile</h2>",
            f"<p>Old: {len(removed) + len(common)} features | New: {len(added) + len(common)} features | "
            f"Added: {len(added)} | Removed: {len(removed)} | Common: {len(common)}</p>",
        ]
        if added:
            parts.append(
                f'<p><span class="badge badge-added">Added</span> {self._esc(", ".join(sorted(added)[:20]))}</p>'
            )
        if removed:
            parts.append(
                f'<p><span class="badge badge-removed">Removed</span> {self._esc(", ".join(sorted(removed)[:20]))}</p>'
            )
        return "\n".join(parts)

    def render_timing(self) -> str:
        ot, nt = self.drift.old_timings, self.drift.new_timings
        if not ot and not nt:
            return "<h2>Per-Notebook Timing</h2>\n<p><em>No timing logs found.</em></p>"
        rows = []
        for nb in dict.fromkeys(list(ot) + list(nt)):
            ov, nv = ot.get(nb, "N/A"), nt.get(nb, "N/A")
            try:
                delta = f"{float(nv) - float(ov):+.1f}s"
            except (ValueError, TypeError):
                delta = "N/A"
            rows.append(f"<tr><td>{self._esc(nb)}</td><td>{ov}</td><td>{nv}</td><td>{delta}</td></tr>")
        table = (
            "<table>\n<tr><th>Notebook</th><th>Old</th><th>New</th><th>Delta</th></tr>\n"
            + "\n".join(rows)
            + "\n</table>"
        )
        return f"<h2>Per-Notebook Timing</h2>\n<details><summary>Show timing table</summary>\n{table}\n</details>"

    def render_cell_profiling(self) -> str:
        op, np_ = self.drift.old_cell_profiles, self.drift.new_cell_profiles
        if op is None and np_ is None:
            return "<h2>Per-Cell Profiling</h2>\n<p><em>No cell profiles found.</em></p>"
        parts = ["<h2>Per-Cell Profiling</h2>"]
        has_spark = False
        has_mem = False
        for profiles in [op, np_]:
            if profiles is None:
                continue
            for nb_prof in profiles.notebooks.values():
                for c in nb_prof.cells:
                    if c.spark_jobs is not None:
                        has_spark = True
                    if c.peak_memory_mb is not None:
                        has_mem = True
        if op and np_ and op.environment != np_.environment:
            parts.append(
                f'<p class="note"><em>Cross-environment comparison ({self._esc(op.environment)} vs '
                f"{self._esc(np_.environment)}) &mdash; elapsed times reflect different hardware.</em></p>"
            )

        all_nbs = sorted(set(list(op.notebooks if op else {}) + list(np_.notebooks if np_ else {})))
        for nb_name in all_nbs:
            nb_ann = self._nb_timing_annotation(nb_name)
            header_extra = f" ({self._esc(nb_ann)})" if nb_ann else ""
            summary_text = f"{self._esc(nb_name)}{header_extra}"

            cols = ["Cell Name", "Cell ID", "Old", "New", "Delta"]
            if has_spark:
                cols.append("Spark Jobs")
            if has_mem:
                cols.append("Mem MB")
            header_row = "".join(f"<th>{c}</th>" for c in cols)

            old_idx = self._cell_profile_index(op, nb_name)
            new_idx = self._cell_profile_index(np_, nb_name)
            old_name_idx = self._cell_profile_name_index(op, nb_name)
            new_name_idx = self._cell_profile_name_index(np_, nb_name)

            seen_ids: set[str] = set()
            ordered_cells: list[tuple[str, str]] = []
            for profiles in [np_, op]:
                if profiles is None:
                    continue
                nb_prof = profiles.notebooks.get(nb_name)
                if nb_prof is None:
                    continue
                for c in nb_prof.cells:
                    if c.cell_id not in seen_ids:
                        seen_ids.add(c.cell_id)
                        ordered_cells.append((c.cell_name, c.cell_id))

            rows = []
            for cell_name, cell_id in ordered_cells:
                old_e = old_idx.get(cell_id) or old_name_idx.get(cell_name)
                new_e = new_idx.get(cell_id) or new_name_idx.get(cell_name)
                old_sec = old_e.elapsed_sec if old_e else None
                new_sec = new_e.elapsed_sec if new_e else None
                old_str = self._fmt_sec(old_sec) if old_sec is not None else "&mdash;"
                new_str = self._fmt_sec(new_sec) if new_sec is not None else "&mdash;"
                if old_sec is not None and new_sec is not None:
                    delta = new_sec - old_sec
                    delta_str = self._fmt_delta(delta)
                    delta_cls = (
                        ' class="diff-added"' if delta < -0.5 else (' class="diff-removed"' if delta > 0.5 else "")
                    )
                else:
                    delta_str = "&mdash;"
                    delta_cls = ""
                row_cells = (
                    f"<td>{self._esc(cell_name)}</td><td><code>{self._esc(cell_id[:8])}</code></td>"
                    f"<td>{old_str}</td><td>{new_str}</td><td{delta_cls}>{delta_str}</td>"
                )
                if has_spark:
                    old_sj = str(old_e.spark_jobs) if old_e and old_e.spark_jobs is not None else "&mdash;"
                    new_sj = str(new_e.spark_jobs) if new_e and new_e.spark_jobs is not None else "&mdash;"
                    row_cells += f"<td>{old_sj}/{new_sj}</td>"
                if has_mem:
                    old_mm = f"{old_e.peak_memory_mb:.1f}" if old_e and old_e.peak_memory_mb is not None else "&mdash;"
                    new_mm = f"{new_e.peak_memory_mb:.1f}" if new_e and new_e.peak_memory_mb is not None else "&mdash;"
                    row_cells += f"<td>{old_mm}/{new_mm}</td>"
                rows.append(f"<tr>{row_cells}</tr>")
            table = f"<table>\n<tr>{header_row}</tr>\n" + "\n".join(rows) + "\n</table>"
            parts.append(f"<details><summary>{summary_text}</summary>\n{table}\n</details>")
        return "\n".join(parts)

    def _render_diff_block(self, diff_text: str) -> str:
        lines = []
        for line in diff_text.split("\n"):
            if line.startswith("---") or line.startswith("+++"):
                continue
            if line.startswith("@@"):
                lines.append(f'<div class="diff-line diff-hunk">{self._esc(line)}</div>')
            elif line.startswith("-"):
                lines.append(f'<div class="diff-line diff-removed">{self._esc(line)}</div>')
            elif line.startswith("+"):
                lines.append(f'<div class="diff-line diff-added">{self._esc(line)}</div>')
            else:
                lines.append(f'<div class="diff-line">{self._esc(line)}</div>')
        return "\n".join(lines)

    def _render_cell_content(self, content: str) -> str:
        if "<table" in content.lower():
            return f'<div class="embed-table">{content}</div>'
        return f"<pre>{self._esc(content[:_CELL_CONTENT_MAX])}</pre>"

    def _render_cell_pair(self, m: CellMatch, nb_name: str = "") -> str:
        old_label = f"Old: {self._esc(m.old_cell.name)}" if m.old_cell else ""
        new_label = f"New: {self._esc(m.new_cell.name)}" if m.new_cell else ""
        title = new_label or old_label
        # Add inline timing annotation
        cell = m.new_cell or m.old_cell
        timing_ann = self._cell_timing_annotation(cell.name, cell.cell_id, nb_name) if cell and nb_name else ""
        timing_html = f' <span class="timing-annotation">{self._esc(timing_ann)}</span>' if timing_ann else ""
        lines = [f"<h4>{title} {self._badge(m.status)}{timing_html}</h4>"]
        if m.old_cell and m.new_cell:
            sec_old = f' <span class="section-tag">{self._esc(m.old_cell.section)}</span>' if m.old_cell.section else ""
            sec_new = f' <span class="section-tag">{self._esc(m.new_cell.section)}</span>' if m.new_cell.section else ""
            lines.append('<div class="cell-grid">')
            lines.append(
                f'<div class="cell-box"><div class="cell-label">Old: {self._esc(m.old_cell.name)}{sec_old}</div>{self._render_cell_content(m.old_cell.content)}</div>'
            )
            lines.append(
                f'<div class="cell-box"><div class="cell-label">New: {self._esc(m.new_cell.name)}{sec_new}</div>{self._render_cell_content(m.new_cell.content)}</div>'
            )
            lines.append("</div>")
            if m.diff_text:
                lines.append(
                    f'<details><summary>Show unified diff</summary><div class="cell-box">{self._render_diff_block(m.diff_text)}</div></details>'
                )
        elif m.status == CellMatchStatus.ADDED:
            lines.append('<div class="cell-grid">')
            lines.append('<div class="cell-box" style="opacity:0.3"><em>no old cell</em></div>')
            lines.append(f'<div class="cell-box status-added">{self._render_cell_content(m.new_cell.content)}</div>')
            lines.append("</div>")
        elif m.status == CellMatchStatus.REMOVED:
            lines.append('<div class="cell-grid">')
            lines.append(f'<div class="cell-box status-removed">{self._render_cell_content(m.old_cell.content)}</div>')
            lines.append('<div class="cell-box" style="opacity:0.3"><em>no new cell</em></div>')
            lines.append("</div>")
        return "\n".join(lines)

    def render_cell_by_cell_diff(self) -> str:
        parts = ["<h2>Cell-by-Cell Diff</h2>"]
        any_diff = False
        for nd in self.drift.notebook_drifts:
            changed = nd.changed_matches
            old_st, new_st = self.notebook_run_status(nd.notebook_name)
            change_counts: dict[str, int] = {}
            for m in changed:
                change_counts[m.status.value] = change_counts.get(m.status.value, 0) + 1
            count_str = ", ".join(f"{v} {k}" for k, v in change_counts.items()) if changed else "no changes"
            old_time, new_time = self.notebook_timing(nd.notebook_name)
            old_pill = f"{self._status_html(old_st)} {old_time}" if old_time else self._status_html(old_st)
            new_pill = f"{self._status_html(new_st)} {new_time}" if new_time else self._status_html(new_st)
            status_pills = f" &mdash; Old: {old_pill} | New: {new_pill}"
            nb_timing = self._nb_timing_annotation(nd.notebook_name)
            timing_extra = f' <span class="timing-annotation">{self._esc(nb_timing)}</span>' if nb_timing else ""
            nb_parts = [
                f'<details class="nb-section"><summary>{self._esc(nd.notebook_name)} ({count_str}){timing_extra}{status_pills}</summary>'
            ]
            if not changed:
                nb_parts.append("<p><em>No differences.</em></p>")
            for m in changed:
                nb_parts.append(self._render_cell_pair(m, nb_name=nd.notebook_name))
            nb_parts.append("</details>")
            parts.append("\n".join(nb_parts))
            if changed:
                any_diff = True
        if not any_diff:
            parts.append("<p><em>No cell-level differences detected.</em></p>")
        return "\n".join(parts)


# ---------------------------------------------------------------------------
# Backward-compatible wrapper
# ---------------------------------------------------------------------------


def generate_drift_report(
    old_manifest_path: Path,
    new_manifest_path: Path,
    old_run_dir: Optional[Path],
    new_run_dir: Optional[Path],
) -> str:
    drift = compute_drift_result(old_manifest_path, new_manifest_path, old_run_dir, new_run_dir)
    return MarkdownDriftRenderer(drift).render()


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------


def main() -> None:
    parser = argparse.ArgumentParser(description="Generate exploration drift report.")
    parser.add_argument("--old-manifest", type=Path, required=True)
    parser.add_argument("--new-manifest", type=Path, required=True)
    parser.add_argument("--old-run-dir", type=Path, default=None)
    parser.add_argument("--new-run-dir", type=Path, default=None)
    parser.add_argument("--output", type=Path, default=None, help="Output file (default: stdout)")
    parser.add_argument("--format", choices=["md", "html", "both"], default="both", dest="fmt")
    args = parser.parse_args()

    drift = compute_drift_result(args.old_manifest, args.new_manifest, args.old_run_dir, args.new_run_dir)
    renderers = {"md": MarkdownDriftRenderer, "html": HtmlDriftRenderer}
    formats = ["md", "html"] if args.fmt == "both" else [args.fmt]

    for fmt in formats:
        report = renderers[fmt](drift).render()
        if args.output:
            suffix = ".html" if fmt == "html" else ".md"
            out_path = args.output.with_suffix(suffix)
            out_path.parent.mkdir(parents=True, exist_ok=True)
            out_path.write_text(report, encoding="utf-8")
            print(f"Drift report written to {out_path}", file=sys.stderr)
        else:
            print(report)


if __name__ == "__main__":
    main()
