# %% [markdown]
# # Qualitative Developer Analytics — LLM-Powered Report Generator
# Uses gpt-4o-mini for deep qualitative analysis supplemented by heuristics.
# Generates a self-contained HTML report per developer with:
# 1. Session arc narratives (what was the developer trying to do?)
# 2. Prompt effectiveness analysis (which prompting styles work best?)
# 3. Tacit knowledge extraction (what does the developer keep re-teaching the AI?)
# 4. Self-correction root cause analysis (why does the AI undo its own work?)
# 5. Cross-developer comparison table

# %%
import sys, os, re, json, base64, io, textwrap, html as html_mod, hashlib
from datetime import datetime
from collections import Counter, defaultdict
from itertools import combinations

sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))

import pandas as pd
import numpy as np
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import seaborn as sns
from openai import OpenAI
from utils.connection import init, query_df
from IPython.display import display, Markdown

pd.set_option("display.max_columns", 30)
pd.set_option("display.max_rows", 200)
pd.set_option("future.no_silent_downcasting", True)

# %% [markdown]
# ## Connection & OpenAI setup

# %%
def _connect_with_retry(max_retries=3):
    import time
    for attempt in range(max_retries):
        try:
            c = init()
            with c.cursor() as cur:
                cur.execute("SELECT 1")
            return c
        except Exception as e:
            if attempt < max_retries - 1:
                time.sleep(3)
            else:
                raise

conn = _connect_with_retry()
oai = OpenAI()  # uses OPENAI_API_KEY env var
display(Markdown("**Connected to DB + OpenAI.**"))

def _query(sql, params=()):
    """query_df with auto-reconnect on SSL errors."""
    global conn
    import time
    for attempt in range(3):
        try:
            return query_df(sql, params, conn)
        except Exception:
            if attempt < 2:
                time.sleep(3)
                try: conn.close()
                except Exception: pass
                conn = _connect_with_retry()
            else:
                raise

# %% [markdown]
# ## LLM helpers

# %%
# Simple disk cache to avoid re-calling gpt-4o-mini for same inputs
CACHE_DIR = os.path.join(os.path.dirname(__file__), ".llm_cache")
os.makedirs(CACHE_DIR, exist_ok=True)

def _cache_key(prompt: str, system: str = "") -> str:
    return hashlib.sha256((system + "|||" + prompt).encode()).hexdigest()

def _cache_get(key: str):
    path = os.path.join(CACHE_DIR, f"{key}.json")
    if os.path.exists(path):
        with open(path) as f:
            return json.load(f)["result"]
    return None

def _cache_set(key: str, result: str):
    path = os.path.join(CACHE_DIR, f"{key}.json")
    with open(path, "w") as f:
        json.dump({"result": result}, f)

def llm_call(prompt: str, system: str = "", max_tokens: int = 1000) -> str:
    """Call gpt-4o-mini with disk caching."""
    key = _cache_key(prompt, system)
    cached = _cache_get(key)
    if cached is not None:
        return cached

    messages = []
    if system:
        messages.append({"role": "system", "content": system})
    messages.append({"role": "user", "content": prompt})

    resp = oai.chat.completions.create(
        model="gpt-4o-mini",
        messages=messages,
        max_tokens=max_tokens,
        temperature=0.3,
    )
    result = resp.choices[0].message.content.strip()
    _cache_set(key, result)
    return result

def llm_call_json(prompt: str, system: str = "", max_tokens: int = 1000):
    """Call gpt-4o-mini and parse JSON response."""
    result = llm_call(prompt, system, max_tokens)
    # Try to extract JSON from markdown code blocks
    json_match = re.search(r"```(?:json)?\s*([\s\S]*?)```", result)
    if json_match:
        result = json_match.group(1).strip()
    return json.loads(result)


# %% [markdown]
# ## Filters & data loading

# %%
ORG_FILTER  = None
REPO_PREFIX = None
USER_FILTER = None
OUTPUT_DIR  = os.path.join(os.path.dirname(__file__), "reports")
os.makedirs(OUTPUT_DIR, exist_ok=True)

# %% [markdown]
# ## Seaborn theme

# %%
PALETTE = sns.color_palette("muted")
sns.set_theme(style="whitegrid", palette=PALETTE, font_scale=0.95,
              rc={"figure.figsize": (8, 3.8), "axes.titlesize": 13,
                  "axes.titleweight": "bold", "axes.labelsize": 11})

C_PRIMARY   = PALETTE[0]
C_SECONDARY = PALETTE[1]
C_SUCCESS   = PALETTE[2]
C_DANGER    = PALETTE[3]
C_PURPLE    = PALETTE[4]

# %% [markdown]
# ## Data loading

# %%
clauses, params = [], []
if ORG_FILTER:
    clauses.append("s.org = %s"); params.append(ORG_FILTER)
if REPO_PREFIX:
    clauses.append("s.repo_name LIKE %s"); params.append(f"{REPO_PREFIX}%")
if USER_FILTER:
    clauses.append("s.user_email = %s"); params.append(USER_FILTER)
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""

sess = _query(f"""
    SELECT s.*,
           DATE_TRUNC('week', COALESCE(
               (SELECT MIN(m.timestamp) FROM messages m WHERE m.session_id = s.id),
               s.first_seen
           ))::date AS week
    FROM sessions s {where}
    ORDER BY s.first_seen
""", tuple(params))

if sess.empty:
    raise SystemExit("No sessions match filters.")

sids = sess["id"].tolist()
display(Markdown(f"**{len(sess)} sessions loaded.**"))

msgs = _query("""
    SELECT m.id, m.session_id, m.msg_type, m.content, m.timestamp,
           s.user_email, s.source, s.repo_name, s.cwd
    FROM messages m JOIN sessions s ON s.id = m.session_id
    WHERE m.session_id = ANY(%s)
    ORDER BY m.timestamp
""", (sids,))

tc = _query("""
    SELECT tc.tool_name, tc.tool_input, tc.tool_id,
           m.id AS message_id, m.session_id, m.timestamp,
           s.user_email, s.cwd, s.repo_name
    FROM tool_calls tc
    JOIN messages m ON m.id = tc.message_id
    JOIN sessions s ON s.id = m.session_id
    WHERE m.session_id = ANY(%s)
    ORDER BY m.timestamp
""", (sids,))

tr = _query("""
    SELECT tr.status, tr.call_id, m.session_id, s.user_email
    FROM tool_results tr
    JOIN messages m ON m.id = tr.message_id
    JOIN sessions s ON s.id = m.session_id
    WHERE m.session_id = ANY(%s)
""", (sids,))

# %% [markdown]
# ## Helpers (shared with 002)

# %%
SHELL   = {"Bash", "shell_command"}
EDIT    = {"Edit", "Write", "MultiEdit", "MultiEditTool"}
EXPLORE = {"Read", "Grep", "Glob", "View", "ReadFile", "read_file",
           "grep_search", "codebase_search", "list_dir", "file_search"}

_EXEC_EXPLORE_RE = re.compile(
    r"^\s*(?:cat|nl|sed\s+-n|head|tail|less|bat)\s|"
    r"^\s*(?:rg|grep|ag|find|fd|ls|tree|wc|du)\s|"
    r"^\s*(?:nl\s+-ba)\s", re.IGNORECASE)
_EXEC_EDIT_RE = re.compile(
    r"cat\s*>|cat\s*<<|tee\s|sed\s+-i|"
    r">\s*\S+|>>\s*\S+|"
    r"patch\s|mv\s|cp\s", re.IGNORECASE)

def classify_exec_command(tool_input):
    if not isinstance(tool_input, dict):
        return "shell"
    cmd = (tool_input.get("cmd") or "").strip()
    if not cmd:
        return "shell"
    if cmd.lower().startswith("echo ") and ">" not in cmd:
        return "shell"
    if _EXEC_EDIT_RE.search(cmd):
        return "edit"
    if _EXEC_EXPLORE_RE.search(cmd):
        return "explore"
    return "shell"

def extract_path(ti):
    if isinstance(ti, dict):
        return ti.get("file_path") or ti.get("path") or ti.get("target_file")
    return None

def make_rel(fp, cwd):
    if fp and cwd and isinstance(fp, str) and isinstance(cwd, str) and fp.startswith(cwd):
        return fp[len(cwd):].lstrip("/") or fp
    return fp

def strip_system_prompts(text):
    if not text:
        return ""
    text = re.sub(
        r"^#\s*AGENTS\.md\s+instructions\s+for\s+.+?\n\n<INSTRUCTIONS?>[\s\S]*?</INSTRUCTIONS?>\s*",
        "", text, flags=re.IGNORECASE)
    text = re.sub(r"<INSTRUCTIONS?>[\s\S]*?</INSTRUCTIONS?>", "", text, flags=re.IGNORECASE)
    text = text.encode("ascii", errors="ignore").decode("ascii")
    return text.strip()

def fig_to_b64(fig):
    buf = io.BytesIO()
    fig.savefig(buf, format="png", dpi=300, bbox_inches="tight", facecolor="white")
    plt.close(fig)
    buf.seek(0)
    return base64.b64encode(buf.read()).decode()


# %% [markdown]
# ## 1. Session Arc Narratives (LLM-powered)

# %%
SESSION_ARC_SYSTEM = """You are an expert at analyzing developer-AI interaction sessions.
Given a sequence of user prompts from a coding session, reconstruct:
1. What the developer was trying to accomplish (the goal)
2. Key pivots or direction changes during the session
3. Whether the session appears successful, partially successful, or abandoned
4. A one-sentence narrative summary

Respond in JSON format:
{
  "goal": "brief description of what they were trying to do",
  "pivots": ["list of direction changes if any"],
  "outcome": "successful|partial|abandoned|exploration_only",
  "narrative": "One sentence story of this session",
  "difficulty": "low|medium|high",
  "key_files": ["most important files mentioned or worked on"]
}"""

def build_session_arc(session_id, u_msgs, u_tc):
    """Build a session arc narrative using LLM + heuristics."""
    s_msgs = u_msgs[u_msgs["session_id"] == session_id].sort_values("timestamp")
    user_prompts = s_msgs[s_msgs["msg_type"] == "user"]["content"].apply(strip_system_prompts)
    user_prompts = [p for p in user_prompts if p and len(p) > 5]

    if not user_prompts:
        return None

    # Heuristic enrichment: what files were touched?
    s_tc = u_tc[u_tc["session_id"] == session_id]
    edited_files = set()
    read_files = set()
    for _, r in s_tc.iterrows():
        fp = extract_path(r["tool_input"])
        if fp:
            short = os.path.basename(fp)
            if r["tool_name"] in EDIT:
                edited_files.add(short)
            elif r["tool_name"] in EXPLORE:
                read_files.add(short)

    # Heuristic: detect self-corrections in this session
    edit_calls = s_tc[s_tc["tool_name"].isin(EDIT)].sort_values("timestamp")
    n_self_corrections = 0
    prev_new = {}  # file -> last new_string
    for _, r in edit_calls.iterrows():
        ti = r["tool_input"]
        if not isinstance(ti, dict):
            continue
        fp = extract_path(ti)
        old_str = ti.get("old_string", "") or ""
        new_str = ti.get("new_string", "") or ti.get("content", "") or ""
        if fp and fp in prev_new and old_str.strip() and prev_new[fp].strip():
            if (prev_new[fp].strip() in old_str.strip() or
                old_str.strip() in prev_new[fp].strip()) and len(prev_new[fp].strip()) > 10:
                n_self_corrections += 1
        if fp and new_str:
            prev_new[fp] = new_str

    # Heuristic: detect git commit
    has_commit = False
    for _, r in s_tc[s_tc["tool_name"].isin(SHELL | {"exec_command"})].iterrows():
        ti = r["tool_input"]
        if isinstance(ti, dict):
            ti = ti.get("command") or ti.get("cmd") or json.dumps(ti)
        if ti and isinstance(ti, str) and ("git commit" in ti.lower() or "git push" in ti.lower()):
            has_commit = True
            break

    # Build prompt for LLM (truncate to keep costs low)
    prompt_text = "\n---\n".join(
        p[:300] + ("..." if len(p) > 300 else "") for p in user_prompts[:15]
    )

    heuristic_context = (
        f"\nFiles edited: {', '.join(sorted(edited_files)[:10]) or 'none'}"
        f"\nFiles read: {', '.join(sorted(read_files)[:10]) or 'none'}"
        f"\nTotal tool calls: {len(s_tc)}"
        f"\nSelf-corrections detected: {n_self_corrections}"
        f"\nGit commit made: {has_commit}"
        f"\nTotal user prompts: {len(user_prompts)}"
    )

    try:
        result = llm_call_json(
            f"Session prompts:\n{prompt_text}\n\nHeuristic context:{heuristic_context}",
            system=SESSION_ARC_SYSTEM,
            max_tokens=500,
        )
    except Exception as e:
        # Fallback to heuristic-only arc
        result = {
            "goal": f"Working on {', '.join(sorted(edited_files)[:3]) or 'unknown files'}",
            "pivots": [],
            "outcome": "successful" if has_commit else "partial" if edited_files else "exploration_only",
            "narrative": f"Session with {len(user_prompts)} prompts touching {len(edited_files)} files.",
            "difficulty": "high" if n_self_corrections > 3 else "medium" if n_self_corrections > 0 else "low",
            "key_files": sorted(edited_files)[:5],
        }

    result["session_id"] = session_id
    result["n_prompts"] = len(user_prompts)
    result["n_tool_calls"] = len(s_tc)
    result["n_self_corrections"] = n_self_corrections
    result["has_commit"] = has_commit
    result["edited_files"] = sorted(edited_files)
    result["n_files_edited"] = len(edited_files)
    result["n_files_read"] = len(read_files)
    return result


# %% [markdown]
# ## 2. Prompt Effectiveness Analysis (LLM + heuristic correlation)

# %%
PROMPT_SCORE_SYSTEM = """You are an expert at evaluating developer prompts to AI coding assistants.
Score this opening prompt on a 1-5 scale for each dimension:

1. specificity: Does it name specific files, functions, or line numbers?
2. clarity: Is the desired outcome clearly stated?
3. context: Does it provide relevant background or constraints?
4. actionability: Can the AI immediately act on it without clarification?

Respond in JSON: {"specificity": N, "clarity": N, "context": N, "actionability": N, "brief_assessment": "one line"}"""

def score_opening_prompt(prompt_text: str) -> dict:
    """Score an opening prompt using LLM + heuristics."""
    if not prompt_text or len(prompt_text.strip()) < 10:
        return {"specificity": 1, "clarity": 1, "context": 1, "actionability": 1,
                "brief_assessment": "Too short to evaluate"}

    # Heuristic pre-scoring (supplements LLM)
    h_specificity = 1
    if re.search(r"[/\\]\w+\.\w+", prompt_text):  # file paths
        h_specificity += 2
    if re.search(r"line\s+\d+|:\d+", prompt_text):  # line numbers
        h_specificity += 1
    if re.search(r"function\s+\w+|class\s+\w+|def\s+\w+|method\s+\w+", prompt_text):
        h_specificity += 1
    h_specificity = min(h_specificity, 5)

    h_context = min(5, 1 + len(prompt_text.split()) // 50)  # longer = more context

    truncated = prompt_text[:800] + ("..." if len(prompt_text) > 800 else "")
    try:
        result = llm_call_json(
            f"Opening prompt:\n{truncated}",
            system=PROMPT_SCORE_SYSTEM,
            max_tokens=200,
        )
        # Blend heuristic and LLM scores (average)
        result["specificity"] = round((result.get("specificity", 3) + h_specificity) / 2, 1)
        result["context"] = round((result.get("context", 3) + h_context) / 2, 1)
    except Exception:
        result = {
            "specificity": h_specificity,
            "clarity": 3,
            "context": h_context,
            "actionability": 3,
            "brief_assessment": "LLM scoring failed, heuristic only",
        }
    return result


# %% [markdown]
# ## 3. Tacit Knowledge Extraction (LLM-powered)

# %%
TACIT_KNOWLEDGE_SYSTEM = """You are an expert at extracting implicit codebase knowledge from developer-AI conversations.
Given a batch of user messages from coding sessions, extract facts the developer implicitly knows about their codebase that the AI has to rediscover each session.

Categories of tacit knowledge:
- Architecture: project structure, module relationships, data flow
- Conventions: naming patterns, coding style, preferred libraries
- Gotchas: known bugs, workarounds, things that break easily
- Domain: business logic, feature requirements, user-facing behavior
- Tooling: build system, test commands, deployment process

Respond in JSON:
{
  "facts": [
    {"category": "architecture|conventions|gotchas|domain|tooling",
     "fact": "the actual knowledge statement",
     "evidence": "brief quote or reference from the messages",
     "sessions_seen": 1}
  ],
  "claude_md_suggestions": ["lines that should go in CLAUDE.md or system prompt"]
}"""

def extract_tacit_knowledge(user_messages: list[str], batch_size: int = 30) -> dict:
    """Extract tacit knowledge from user messages across sessions."""
    all_facts = []
    all_suggestions = []

    # Process in batches to stay within token limits
    for i in range(0, len(user_messages), batch_size):
        batch = user_messages[i:i + batch_size]
        combined = "\n---\n".join(
            m[:400] + ("..." if len(m) > 400 else "") for m in batch
        )

        try:
            result = llm_call_json(
                f"Developer messages (batch {i // batch_size + 1}):\n{combined}",
                system=TACIT_KNOWLEDGE_SYSTEM,
                max_tokens=1500,
            )
            all_facts.extend(result.get("facts", []))
            all_suggestions.extend(result.get("claude_md_suggestions", []))
        except Exception:
            continue

    # Deduplicate facts by similarity (simple: same category + >50% word overlap)
    deduped = []
    for fact in all_facts:
        is_dup = False
        fact_words = set(fact.get("fact", "").lower().split())
        for existing in deduped:
            existing_words = set(existing.get("fact", "").lower().split())
            if existing.get("category") == fact.get("category"):
                overlap = len(fact_words & existing_words) / max(len(fact_words | existing_words), 1)
                if overlap > 0.5:
                    existing["sessions_seen"] = existing.get("sessions_seen", 1) + 1
                    is_dup = True
                    break
        if not is_dup:
            deduped.append(fact)

    return {
        "facts": sorted(deduped, key=lambda f: -f.get("sessions_seen", 1)),
        "claude_md_suggestions": list(dict.fromkeys(all_suggestions)),  # dedupe preserving order
    }


# %% [markdown]
# ## 4. Self-Correction Root Cause Analysis (heuristic + LLM)

# %%
REVERT_ANALYSIS_SYSTEM = """You are analyzing why an AI coding assistant kept correcting its own code edits.
Given the sequence of edits (old_string -> new_string) for a file in one session, along with the user prompts,
determine the root cause of the self-corrections.

Common causes:
- unclear_requirements: The user's request was ambiguous
- wrong_approach: The AI chose a wrong architecture/pattern
- incremental_refinement: The AI was iteratively improving (not really a problem)
- missing_context: The AI didn't have enough codebase context
- api_mismatch: The AI used wrong API/library signatures
- syntax_errors: The AI made syntax mistakes it had to fix

Respond in JSON:
{"root_cause": "one of the causes above", "explanation": "1-2 sentences", "severity": "low|medium|high"}"""

def analyze_self_corrections(session_id, file_path, edit_sequence, user_prompts):
    """Analyze why self-corrections happened in a specific file+session."""
    # Build edit chain description
    edits_desc = []
    for i, edit in enumerate(edit_sequence[:8]):  # cap at 8 edits
        old = (edit.get("old_string") or "")[:150]
        new = (edit.get("new_string") or edit.get("content") or "")[:150]
        edits_desc.append(f"Edit {i+1}: '{old}' -> '{new}'")

    prompts_desc = "\n".join(p[:200] for p in user_prompts[:5])

    # Heuristic pre-classification
    all_new_strings = " ".join(
        (e.get("new_string") or e.get("content") or "") for e in edit_sequence
    )
    h_cause = "unclear_requirements"
    if re.search(r"syntax|SyntaxError|IndentationError|TypeError|NameError", all_new_strings):
        h_cause = "syntax_errors"
    elif re.search(r"import|from\s+\w+\s+import|require\(", all_new_strings) and len(edit_sequence) > 4:
        h_cause = "api_mismatch"

    try:
        result = llm_call_json(
            f"File: {os.path.basename(file_path)}\n\nEdit sequence:\n"
            + "\n".join(edits_desc)
            + f"\n\nUser prompts in this session:\n{prompts_desc}",
            system=REVERT_ANALYSIS_SYSTEM,
            max_tokens=300,
        )
    except Exception:
        result = {
            "root_cause": h_cause,
            "explanation": f"Heuristic classification based on edit patterns ({len(edit_sequence)} edits).",
            "severity": "high" if len(edit_sequence) > 6 else "medium",
        }

    return result


# %% [markdown]
# ## 5. Session Goal Clustering (heuristic + LLM)

# %%
CLUSTER_LABEL_SYSTEM = """Given a list of session goals from a developer's AI coding sessions, group them into
3-7 high-level themes/projects. For each theme, provide a short label and list which goal indices belong to it.

Respond in JSON:
{"themes": [{"label": "short name", "goal_indices": [0, 3, 7], "description": "what this work stream is about"}]}"""

def cluster_session_goals(arcs: list[dict]) -> list[dict]:
    """Cluster session arcs into themes/projects."""
    if len(arcs) < 5:
        return []

    goals = [a.get("goal", "unknown") for a in arcs]

    # Heuristic pre-clustering by file overlap
    file_clusters = defaultdict(list)
    for i, arc in enumerate(arcs):
        key_files = frozenset(arc.get("key_files", [])[:3])
        if key_files:
            file_clusters[key_files].append(i)

    # LLM clustering for narrative themes
    goals_text = "\n".join(f"{i}. {g}" for i, g in enumerate(goals[:50]))
    try:
        result = llm_call_json(
            f"Session goals:\n{goals_text}",
            system=CLUSTER_LABEL_SYSTEM,
            max_tokens=800,
        )
        return result.get("themes", [])
    except Exception:
        # Fallback: cluster by shared files
        themes = []
        for files, indices in sorted(file_clusters.items(), key=lambda x: -len(x[1])):
            if len(indices) >= 2:
                themes.append({
                    "label": ", ".join(sorted(files)[:3]),
                    "goal_indices": indices,
                    "description": f"Sessions working on {', '.join(sorted(files)[:3])}",
                })
            if len(themes) >= 5:
                break
        return themes


# %% [markdown]
# ## Compute qualitative report per developer

# %%
def compute_qualitative_report(email):
    """Compute the qualitative analysis report for a single developer."""
    report = {"email": email, "sections": [], "charts": [], "tables": [], "raw_html": []}

    u_sess = sess[sess["user_email"] == email].copy()
    u_msgs = msgs[msgs["user_email"] == email].copy()
    u_tc   = tc[tc["user_email"] == email].copy()
    u_tr   = tr[tr["user_email"] == email].copy()

    n_sess = len(u_sess)
    if n_sess == 0:
        return report

    # Tool groups
    def _tool_group(row):
        t = row["tool_name"]
        if t == "exec_command":
            return classify_exec_command(row["tool_input"])
        if t in SHELL:   return "shell"
        if t in EDIT:    return "edit"
        if t in EXPLORE: return "explore"
        return "other"
    u_tc["group"] = u_tc.apply(_tool_group, axis=1)

    # Active duration
    IDLE_THRESHOLD_SEC = 30 * 60
    def _active_minutes(ts_series):
        ts = ts_series.sort_values()
        if len(ts) < 2:
            return 0.0
        gaps = ts.diff().dropna().dt.total_seconds()
        return gaps[gaps < IDLE_THRESHOLD_SEC].sum() / 60.0

    msg_duration = (
        u_msgs.groupby("session_id")["timestamp"]
        .apply(_active_minutes)
        .rename("duration_min")
    )

    # Per-session aggregates
    s_msg_types = u_msgs.groupby(["session_id", "msg_type"]).size().unstack(fill_value=0)
    for col in ["user", "assistant", "tool_call", "tool_result"]:
        if col not in s_msg_types.columns:
            s_msg_types[col] = 0
    s_msg_types = s_msg_types.rename(columns={"user": "user_msgs", "assistant": "assistant_msgs"})

    s_tc_groups = u_tc.groupby(["session_id", "group"]).size().unstack(fill_value=0)
    for col in ["explore", "edit", "shell", "other"]:
        if col not in s_tc_groups.columns:
            s_tc_groups[col] = 0
    s_tc_groups["tool_calls"] = s_tc_groups.sum(axis=1)

    s_df = (u_sess.set_index("id")
            .join(s_msg_types)
            .join(s_tc_groups, rsuffix="_tc")
            .join(msg_duration)
            .fillna({"duration_min": 0})
            .infer_objects(copy=False)
            .fillna(0))

    # Git commit detection
    commit_sids = set()
    for _, r in u_tc[u_tc["tool_name"].isin(SHELL | {"exec_command"})].iterrows():
        ti = r["tool_input"]
        if isinstance(ti, dict):
            ti = ti.get("command") or ti.get("cmd") or json.dumps(ti)
        if ti and isinstance(ti, str) and ("git commit" in ti.lower() or "git push" in ti.lower()):
            commit_sids.add(r["session_id"])
    s_df["has_commit"] = s_df.index.isin(commit_sids)

    display(Markdown(f"  Computing session arcs for {email}..."))

    # ═══════════════════════════════════════════════════════
    # SECTION 1: Session Arc Narratives
    # ═══════════════════════════════════════════════════════
    session_ids = u_sess["id"].tolist()
    arcs = []
    for sid in session_ids:
        arc = build_session_arc(sid, u_msgs, u_tc)
        if arc:
            arcs.append(arc)

    display(Markdown(f"  {len(arcs)} session arcs computed."))

    if arcs:
        # Summary statistics from arcs
        outcomes = Counter(a.get("outcome", "unknown") for a in arcs)
        difficulties = Counter(a.get("difficulty", "unknown") for a in arcs)

        arc_section = [
            ("Sessions analyzed", f"{len(arcs)} sessions had enough data for narrative reconstruction"),
            ("Outcome distribution",
                ", ".join(f"{k}: {v}" for k, v in outcomes.most_common())),
            ("Difficulty distribution",
                ", ".join(f"{k}: {v}" for k, v in difficulties.most_common())),
        ]

        # Find the most complex sessions (high difficulty + many self-corrections)
        hard_arcs = sorted(arcs, key=lambda a: (-a.get("n_self_corrections", 0), -a.get("n_tool_calls", 0)))[:5]
        for i, arc in enumerate(hard_arcs[:3]):
            if arc.get("n_self_corrections", 0) > 0 or arc.get("n_tool_calls", 0) > 20:
                arc_section.append((
                    f"Notable session {i+1}",
                    f"{arc.get('narrative', 'N/A')} "
                    f"({arc['n_prompts']} prompts, {arc['n_tool_calls']} tool calls, "
                    f"{arc['n_self_corrections']} self-corrections, "
                    f"{'committed' if arc['has_commit'] else 'no commit'})"
                ))

        report["sections"].append(("Session Arc Narratives", arc_section))

        # Top 15 session narrative table
        arc_table_data = []
        for arc in sorted(arcs, key=lambda a: -a.get("n_tool_calls", 0))[:15]:
            arc_table_data.append({
                "Session": arc["session_id"][:12] + "...",
                "Goal": (arc.get("goal", "")[:80] + "...") if len(arc.get("goal", "")) > 80 else arc.get("goal", ""),
                "Outcome": arc.get("outcome", ""),
                "Difficulty": arc.get("difficulty", ""),
                "Prompts": arc["n_prompts"],
                "Tools": arc["n_tool_calls"],
                "Self-fix": arc["n_self_corrections"],
                "Commit": "Y" if arc["has_commit"] else "",
            })
        if arc_table_data:
            report["tables"].append(("Session narratives (most active sessions)", pd.DataFrame(arc_table_data)))

        # Outcome chart
        if outcomes:
            fig, ax = plt.subplots(figsize=(5, 4))
            labels = list(outcomes.keys())
            sizes = list(outcomes.values())
            colors = [C_SUCCESS if l == "successful" else C_SECONDARY if l == "partial"
                      else C_DANGER if l == "abandoned" else C_PRIMARY for l in labels]
            ax.pie(sizes, labels=labels, colors=colors, autopct="%1.0f%%",
                   startangle=140, wedgeprops={"edgecolor": "white", "linewidth": 1.5})
            ax.set_title("Session Outcomes", fontsize=13, fontweight="bold")
            report["charts"].append(("Session outcome distribution", fig_to_b64(fig)))

    # ═══════════════════════════════════════════════════════
    # SECTION 2: Prompt Effectiveness Analysis
    # ═══════════════════════════════════════════════════════
    display(Markdown(f"  Scoring opening prompts..."))

    user_text = u_msgs[u_msgs["msg_type"] == "user"].copy()
    user_text["clean"] = user_text["content"].apply(strip_system_prompts)
    user_text = user_text[user_text["clean"].str.len() > 0]

    first_prompts = user_text.sort_values("timestamp").groupby("session_id").first().reset_index()

    prompt_scores = []
    for _, row in first_prompts.iterrows():
        score = score_opening_prompt(row["clean"])
        score["session_id"] = row["session_id"]
        score["word_count"] = len(row["clean"].split())
        score["prompt_preview"] = row["clean"][:120] + ("..." if len(row["clean"]) > 120 else "")
        prompt_scores.append(score)

    display(Markdown(f"  {len(prompt_scores)} prompts scored."))

    if prompt_scores:
        ps_df = pd.DataFrame(prompt_scores)

        # Correlate prompt quality with session efficiency
        # Merge with session data
        ps_df = ps_df.merge(
            s_df[["user_msgs", "tool_calls", "explore", "edit", "duration_min", "has_commit"]].reset_index().rename(
                columns={"id": "session_id"}),
            on="session_id", how="left"
        )

        # Compute composite prompt quality score
        for col in ["specificity", "clarity", "context", "actionability"]:
            if col not in ps_df.columns:
                ps_df[col] = 3
        ps_df["quality_score"] = (
            ps_df["specificity"] + ps_df["clarity"] + ps_df["context"] + ps_df["actionability"]
        ) / 4

        # Split into high/low quality
        median_quality = ps_df["quality_score"].median()
        high_q = ps_df[ps_df["quality_score"] >= median_quality]
        low_q = ps_df[ps_df["quality_score"] < median_quality]

        prompt_section = [
            ("Prompts analyzed", f"{len(ps_df)} opening prompts scored for quality"),
            ("Average quality score", f"{ps_df['quality_score'].mean():.1f}/5"),
        ]

        if len(high_q) > 2 and len(low_q) > 2:
            hq_explore = high_q["explore"].median()
            lq_explore = low_q["explore"].median()
            hq_duration = high_q["duration_min"].median()
            lq_duration = low_q["duration_min"].median()

            if lq_explore > 0 and hq_explore < lq_explore:
                reduction = (1 - hq_explore / lq_explore) * 100
                prompt_section.append(("Exploration savings with better prompts",
                    f"High-quality prompts lead to {reduction:.0f}% fewer explore calls "
                    f"(median {hq_explore:.0f} vs {lq_explore:.0f})"))

            if lq_duration > 0 and hq_duration < lq_duration:
                time_saving = (1 - hq_duration / lq_duration) * 100
                prompt_section.append(("Time savings with better prompts",
                    f"High-quality prompts correlate with {time_saving:.0f}% shorter sessions "
                    f"(median {hq_duration:.0f} min vs {lq_duration:.0f} min)"))

        # Best and worst prompts
        best = ps_df.nlargest(3, "quality_score")
        worst = ps_df.nsmallest(3, "quality_score")

        prompt_section.append(("Best-rated opening prompts",
            " | ".join(f"[{r['quality_score']:.1f}/5] {r['prompt_preview']}" for _, r in best.iterrows())))

        if len(ps_df) > 5:
            prompt_section.append(("Lowest-rated opening prompts",
                " | ".join(f"[{r['quality_score']:.1f}/5] {r['prompt_preview']}" for _, r in worst.iterrows())))

        report["sections"].append(("Prompt Effectiveness Analysis", prompt_section))

        # Quality vs efficiency scatter
        if len(ps_df) > 5 and "explore" in ps_df.columns:
            fig, axes = plt.subplots(1, 2, figsize=(12, 4.5))

            ax = axes[0]
            ax.scatter(ps_df["quality_score"], ps_df["explore"], alpha=0.5, color=C_PRIMARY, s=30)
            z = np.polyfit(ps_df["quality_score"].fillna(3), ps_df["explore"].fillna(0), 1)
            p = np.poly1d(z)
            x_line = np.linspace(ps_df["quality_score"].min(), ps_df["quality_score"].max(), 50)
            ax.plot(x_line, p(x_line), "--", color=C_DANGER, linewidth=1.5, label=f"Trend")
            ax.set_xlabel("Prompt Quality Score"); ax.set_ylabel("Explore Calls")
            ax.set_title("Better Prompts → Less Exploration?"); ax.legend(fontsize=9)

            ax = axes[1]
            ax.scatter(ps_df["quality_score"], ps_df["duration_min"], alpha=0.5, color=C_SUCCESS, s=30)
            z2 = np.polyfit(ps_df["quality_score"].fillna(3), ps_df["duration_min"].fillna(0), 1)
            p2 = np.poly1d(z2)
            ax.plot(x_line, p2(x_line), "--", color=C_DANGER, linewidth=1.5, label=f"Trend")
            ax.set_xlabel("Prompt Quality Score"); ax.set_ylabel("Session Duration (min)")
            ax.set_title("Better Prompts → Shorter Sessions?"); ax.legend(fontsize=9)

            plt.tight_layout()
            report["charts"].append(("Prompt quality vs session efficiency", fig_to_b64(fig)))

        # Prompt quality dimension breakdown chart
        dim_means = {d: ps_df[d].mean() for d in ["specificity", "clarity", "context", "actionability"]}
        fig, ax = plt.subplots(figsize=(6, 3.5))
        bars = ax.barh(list(dim_means.keys()), list(dim_means.values()), color=C_PRIMARY)
        ax.set_xlim(0, 5)
        ax.set_title("Average Prompt Quality by Dimension")
        ax.set_xlabel("Score (1-5)")
        for bar, v in zip(bars, dim_means.values()):
            ax.text(v + 0.1, bar.get_y() + bar.get_height()/2, f"{v:.1f}", va="center", fontsize=10)
        plt.tight_layout()
        report["charts"].append(("Prompt quality dimensions", fig_to_b64(fig)))

    # ═══════════════════════════════════════════════════════
    # SECTION 3: Tacit Knowledge Extraction
    # ═══════════════════════════════════════════════════════
    display(Markdown(f"  Extracting tacit knowledge..."))

    all_user_messages = [
        strip_system_prompts(row["content"])
        for _, row in u_msgs[u_msgs["msg_type"] == "user"].iterrows()
        if strip_system_prompts(row.get("content") or "")
    ]

    if all_user_messages:
        tacit = extract_tacit_knowledge(all_user_messages)
        facts = tacit.get("facts", [])
        suggestions = tacit.get("claude_md_suggestions", [])

        if facts:
            tacit_section = [
                ("Knowledge facts extracted", f"{len(facts)} implicit codebase facts found across sessions"),
            ]

            # Group by category
            by_cat = defaultdict(list)
            for f in facts:
                by_cat[f.get("category", "other")].append(f)

            for cat in ["architecture", "conventions", "gotchas", "domain", "tooling"]:
                cat_facts = by_cat.get(cat, [])
                if cat_facts:
                    top_facts = sorted(cat_facts, key=lambda x: -x.get("sessions_seen", 1))[:3]
                    tacit_section.append((
                        f"{cat.title()} knowledge ({len(cat_facts)} facts)",
                        " | ".join(f.get("fact", "") for f in top_facts),
                    ))

            if suggestions:
                tacit_section.append((
                    "Recommended CLAUDE.md entries",
                    f"{len(suggestions)} suggestions. Adding these to your system prompt would "
                    f"eliminate repeated context discovery."
                ))

            report["sections"].append(("Tacit Knowledge — what you keep re-teaching the AI", tacit_section))

            # Table of all facts
            facts_table = pd.DataFrame([
                {"Category": f.get("category", ""), "Knowledge": f.get("fact", ""),
                 "Evidence": f.get("evidence", "")[:80], "Seen": f.get("sessions_seen", 1)}
                for f in facts[:25]
            ])
            report["tables"].append(("Extracted tacit knowledge", facts_table))

            # CLAUDE.md suggestions table
            if suggestions:
                sugg_table = pd.DataFrame([
                    {"#": i+1, "Suggested CLAUDE.md line": s}
                    for i, s in enumerate(suggestions[:15])
                ])
                report["tables"].append(("Recommended CLAUDE.md / system prompt entries", sugg_table))

    # ═══════════════════════════════════════════════════════
    # SECTION 4: Self-Correction Root Cause Analysis
    # ═══════════════════════════════════════════════════════
    display(Markdown(f"  Analyzing self-corrections..."))

    edit_calls = u_tc[u_tc["group"] == "edit"].sort_values(["session_id", "timestamp"]).copy()
    edit_calls["fp"] = edit_calls["tool_input"].apply(extract_path)

    revert_sessions = []
    for sid, grp in edit_calls.groupby("session_id"):
        file_edits_by_file = grp.groupby("fp")
        for fp, fgrp in file_edits_by_file:
            if fp is None or len(fgrp) < 3:
                continue
            # Detect self-corrections
            prev_new = None
            n_reverts = 0
            for _, row in fgrp.iterrows():
                ti = row["tool_input"]
                if not isinstance(ti, dict):
                    prev_new = None
                    continue
                old_str = ti.get("old_string", "") or ""
                new_str = ti.get("new_string", "") or ti.get("content", "") or ""
                if prev_new and old_str.strip() and prev_new.strip():
                    if (prev_new.strip() in old_str.strip() or
                        old_str.strip() in prev_new.strip()) and len(prev_new.strip()) > 10:
                        n_reverts += 1
                prev_new = new_str

            if n_reverts >= 2:
                edit_sequence = [r["tool_input"] for _, r in fgrp.iterrows() if isinstance(r["tool_input"], dict)]
                s_user_prompts = [
                    strip_system_prompts(r["content"])
                    for _, r in u_msgs[(u_msgs["session_id"] == sid) & (u_msgs["msg_type"] == "user")].iterrows()
                    if strip_system_prompts(r.get("content") or "")
                ]
                analysis = analyze_self_corrections(sid, fp, edit_sequence, s_user_prompts)
                revert_sessions.append({
                    "session_id": sid,
                    "file": fp,
                    "total_edits": len(fgrp),
                    "self_corrections": n_reverts,
                    "root_cause": analysis.get("root_cause", "unknown"),
                    "explanation": analysis.get("explanation", ""),
                    "severity": analysis.get("severity", "medium"),
                })

    if revert_sessions:
        rev_df = pd.DataFrame(revert_sessions)
        cause_counts = Counter(rev_df["root_cause"])

        revert_section = [
            ("Files with significant self-corrections",
                f"{len(rev_df)} file×session pairs had 2+ self-corrections"),
            ("Root cause breakdown",
                ", ".join(f"{k}: {v}" for k, v in cause_counts.most_common())),
        ]

        # Most common root cause
        top_cause = cause_counts.most_common(1)[0] if cause_counts else ("unknown", 0)
        revert_section.append(("Primary root cause",
            f"'{top_cause[0]}' accounts for {top_cause[1]} of {len(rev_df)} cases. "
            f"This suggests {'the developer should provide more specific requirements upfront' if top_cause[0] == 'unclear_requirements' else 'the AI needs better codebase context pre-loaded' if top_cause[0] == 'missing_context' else 'the AI struggles with API/library usage in this codebase' if top_cause[0] == 'api_mismatch' else 'iterative refinement is part of the workflow'}."))

        report["sections"].append(("Self-Correction Root Cause Analysis", revert_section))

        # Root cause chart
        if cause_counts:
            fig, ax = plt.subplots(figsize=(6, 4))
            labels = [k for k, _ in cause_counts.most_common()]
            sizes = [v for _, v in cause_counts.most_common()]
            colors = sns.color_palette("muted", len(labels))
            ax.pie(sizes, labels=labels, colors=colors, autopct="%1.0f%%",
                   startangle=140, wedgeprops={"edgecolor": "white", "linewidth": 1.5})
            ax.set_title("Why does the AI undo its own work?", fontsize=13, fontweight="bold")
            report["charts"].append(("Self-correction root causes", fig_to_b64(fig)))

        # Detailed table
        rev_table = rev_df.sort_values("self_corrections", ascending=False).head(15).copy()
        rev_table["file"] = rev_table["file"].apply(lambda x: os.path.basename(str(x)) if x else "")
        rev_table["session_id"] = rev_table["session_id"].str[:12] + "..."
        rev_table = rev_table.rename(columns={
            "session_id": "Session", "file": "File", "total_edits": "Edits",
            "self_corrections": "Self-fixes", "root_cause": "Root Cause",
            "explanation": "Explanation", "severity": "Severity",
        })
        report["tables"].append(("Self-correction root cause details", rev_table))

    # ═══════════════════════════════════════════════════════
    # SECTION 5: Session Goal Clustering / Project Themes
    # ═══════════════════════════════════════════════════════
    if arcs and len(arcs) >= 5:
        display(Markdown(f"  Clustering session goals into themes..."))
        themes = cluster_session_goals(arcs)

        if themes:
            theme_section = [
                ("Work themes detected", f"{len(themes)} distinct themes/projects identified"),
            ]
            for i, t in enumerate(themes[:7], 1):
                n_sessions = len(t.get("goal_indices", []))
                theme_section.append((
                    f"Theme {i}: {t.get('label', 'Unknown')}",
                    f"{n_sessions} sessions — {t.get('description', '')}",
                ))

            report["sections"].append(("Project Themes — what is this developer working on?", theme_section))

            # Theme size chart
            fig, ax = plt.subplots(figsize=(7, max(3, len(themes) * 0.5)))
            labels = [t.get("label", "?")[:30] for t in themes]
            sizes = [len(t.get("goal_indices", [])) for t in themes]
            sns.barplot(x=sizes, y=labels, color=C_PRIMARY, edgecolor="white", ax=ax, orient="h")
            ax.set_title("Session Distribution by Project Theme")
            ax.set_xlabel("Number of sessions")
            plt.tight_layout()
            report["charts"].append(("Sessions by project theme", fig_to_b64(fig)))

    # ═══════════════════════════════════════════════════════
    # Summary stats for cross-developer comparison
    # ═══════════════════════════════════════════════════════
    n_explore = int((u_tc["group"] == "explore").sum())
    n_edit = int((u_tc["group"] == "edit").sum())
    explore_edit = n_explore / max(n_edit, 1)
    commit_rate = s_df["has_commit"].mean() * 100

    # Warmup
    warmup_records = []
    for sid, grp in u_msgs.groupby("session_id"):
        sid_tc = u_tc[u_tc["session_id"] == sid].sort_values("timestamp")
        edit_tc_first = sid_tc[sid_tc["group"] == "edit"]
        if edit_tc_first.empty:
            continue
        first_edit_ts = edit_tc_first["timestamp"].iloc[0]
        explore_before = sid_tc[(sid_tc["group"] == "explore") & (sid_tc["timestamp"] < first_edit_ts)]
        warmup_records.append(len(explore_before))

    report["_comparison_stats"] = {
        "email": email,
        "sessions": n_sess,
        "discovery_overhead": round(explore_edit, 1),
        "commit_rate": round(commit_rate, 0),
        "median_warmup_explores": round(np.median(warmup_records), 0) if warmup_records else 0,
        "median_duration": round(s_df["duration_min"].median(), 0),
        "median_prompts": round(s_df["user_msgs"].median(), 0),
        "sessions_with_edits_pct": round(
            len(s_df[s_df.get("edit", 0) > 0]) / max(n_sess, 1) * 100, 0) if "edit" in s_df.columns else 0,
        "outcome_success_pct": round(
            sum(1 for a in arcs if a.get("outcome") == "successful") / max(len(arcs), 1) * 100, 0) if arcs else 0,
        "avg_prompt_quality": round(
            ps_df["quality_score"].mean(), 1) if prompt_scores and "quality_score" in ps_df.columns else 0,
        "n_tacit_facts": len(tacit.get("facts", [])) if 'tacit' in dir() else 0,
        "top_root_cause": cause_counts.most_common(1)[0][0] if 'cause_counts' in dir() and cause_counts else "N/A",
    }

    return report


# %% [markdown]
# ## Cross-Developer Comparison

# %%
def build_comparison_section(reports):
    """Build a cross-developer comparison table from all reports."""
    stats = [r.get("_comparison_stats", {}) for r in reports if r.get("_comparison_stats")]
    if len(stats) < 2:
        return None

    df = pd.DataFrame(stats)
    # Make email shorter
    df["Developer"] = df["email"].apply(lambda e: e.split("@")[0][:20])
    df = df.rename(columns={
        "sessions": "Sessions",
        "discovery_overhead": "Discovery Overhead (x)",
        "commit_rate": "Commit Rate %",
        "median_warmup_explores": "Warmup Explores (med)",
        "median_duration": "Duration min (med)",
        "median_prompts": "Prompts/session (med)",
        "outcome_success_pct": "Success Rate %",
        "avg_prompt_quality": "Prompt Quality (avg)",
        "top_root_cause": "Top Self-fix Cause",
    })
    cols = ["Developer", "Sessions", "Discovery Overhead (x)", "Commit Rate %",
            "Warmup Explores (med)", "Duration min (med)", "Prompts/session (med)",
            "Success Rate %", "Prompt Quality (avg)", "Top Self-fix Cause"]
    cols = [c for c in cols if c in df.columns]
    return df[cols]


# %% [markdown]
# ## HTML template

# %%
HTML_TEMPLATE = textwrap.dedent("""\
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Qualitative Report — {email}</title>
<style>
  * {{ margin: 0; padding: 0; box-sizing: border-box; }}
  body {{ font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
         background: #F7F8FA; color: #2C3E50; padding: 24px; max-width: 960px; margin: 0 auto;
         line-height: 1.5; }}
  h1 {{ font-size: 1.5em; margin: 16px 0 8px; border-bottom: 3px solid #8E44AD; padding-bottom: 8px; }}
  h2 {{ font-size: 1.15em; margin: 16px 0 10px; color: #34495E; }}
  .subtitle {{ color: #7F8C8D; font-size: 0.85em; margin-bottom: 20px; }}
  .card {{ background: white; border-radius: 10px; padding: 18px 22px; margin: 14px 0;
           box-shadow: 0 1px 4px rgba(0,0,0,0.08); }}
  .kv {{ display: grid; grid-template-columns: 1fr; gap: 10px; }}
  .kv-row {{ display: flex; gap: 8px; }}
  .kv-row .label {{ font-weight: 600; color: #555; min-width: 220px; flex-shrink: 0; }}
  .kv-row .value {{ color: #222; }}
  .chart {{ margin: 10px 0; text-align: center; }}
  .chart img {{ max-width: 100%; border-radius: 6px; }}
  .table-wrap {{ overflow-x: auto; -webkit-overflow-scrolling: touch; max-width: 100%; }}
  table {{ border-collapse: separate; border-spacing: 0; width: 100%; margin: 10px 0;
           font-size: 0.88em; border-radius: 8px; overflow: hidden;
           border: 1px solid #E0E4E8; }}
  thead th {{ background: #8E44AD; color: white; text-align: left; padding: 10px 14px;
              font-weight: 600; font-size: 0.85em; text-transform: uppercase; letter-spacing: 0.3px; }}
  tbody td {{ padding: 9px 14px; border-bottom: 1px solid #EEF0F2; color: #333;
              overflow-wrap: break-word; word-wrap: break-word; max-width: 400px; }}
  tbody tr:nth-child(even) td {{ background: #F8F9FB; }}
  tbody tr:hover td {{ background: #EBF0F5; }}
  tbody tr:last-child td {{ border-bottom: none; }}
  .footer {{ color: #AAA; font-size: 0.75em; margin-top: 40px; text-align: center;
             padding-top: 16px; border-top: 1px solid #E0E4E8; }}
</style>
</head>
<body>
<h1>Qualitative Analysis: {email}</h1>
<p class="subtitle">Generated {generated_at} &bull; LLM-powered analysis using gpt-4o-mini + heuristics</p>

{sections_html}

{charts_html}

{tables_html}

{raw_html}

<p class="footer">Generated by qc_trace qualitative analytics (003)</p>
</body>
</html>
""")


def render_qualitative_report(report):
    email = report["email"]

    sections_html = ""
    for title, rows in report.get("sections", []):
        inner = ""
        for label, value in rows:
            inner += f'<div class="kv-row"><span class="label">{html_mod.escape(str(label))}</span><span class="value">{html_mod.escape(str(value))}</span></div>\n'
        sections_html += f'<div class="card"><h2>{html_mod.escape(title)}</h2><div class="kv">{inner}</div></div>\n'

    charts_html = ""
    for title, b64 in report.get("charts", []):
        charts_html += (f'<div class="card"><h2>{html_mod.escape(title)}</h2>'
                        f'<div class="chart"><img src="data:image/png;base64,{b64}" alt="{html_mod.escape(title)}"></div></div>\n')

    tables_html = ""
    for title, df in report.get("tables", []):
        table_html = df.to_html(index=False, classes="", border=0)
        tables_html += f'<div class="card"><h2>{html_mod.escape(title)}</h2><div class="table-wrap">{table_html}</div></div>\n'

    raw_html = "\n".join(report.get("raw_html", []))

    return HTML_TEMPLATE.format(
        email=email,
        generated_at=datetime.now().strftime("%Y-%m-%d %H:%M"),
        sections_html=sections_html,
        charts_html=charts_html,
        tables_html=tables_html,
        raw_html=raw_html,
    )


# %% [markdown]
# ## Run

# %%
TARGET_EMAILS = [
    "sagarsarkale.work@gmail.com",
    "54902986+zzjjaayy@users.noreply.github.com",
    "ajaysingh07032003@gmail.com",
    "patil.vaibhav2147.vp@gmail.com",
]

if USER_FILTER:
    target_emails = [USER_FILTER]
elif TARGET_EMAILS:
    target_emails = [e for e in TARGET_EMAILS if e in sess["user_email"].values]
else:
    target_emails = sorted(e for e in sess["user_email"].unique() if e)

all_reports = []
for email in target_emails:
    display(Markdown(f"### Generating qualitative report for `{email}`..."))
    report = compute_qualitative_report(email)
    all_reports.append(report)
    html = render_qualitative_report(report)
    safe_name = re.sub(r"[^a-zA-Z0-9_.-]", "_", email)
    out_path = os.path.join(OUTPUT_DIR, f"qual_{safe_name}.html")
    with open(out_path, "w") as f:
        f.write(html)
    display(Markdown(f"Saved: `{out_path}`"))

# %% [markdown]
# ## Cross-Developer Comparison Report

# %%
comparison_df = build_comparison_section(all_reports)
if comparison_df is not None:
    # Generate a standalone comparison HTML
    comparison_html = textwrap.dedent(f"""\
    <!DOCTYPE html>
    <html lang="en">
    <head>
    <meta charset="UTF-8">
    <title>Cross-Developer Comparison</title>
    <style>
      body {{ font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
             background: #F7F8FA; color: #2C3E50; padding: 24px; max-width: 1100px; margin: 0 auto; }}
      h1 {{ font-size: 1.5em; border-bottom: 3px solid #8E44AD; padding-bottom: 8px; }}
      .subtitle {{ color: #7F8C8D; font-size: 0.85em; margin-bottom: 20px; }}
      .card {{ background: white; border-radius: 10px; padding: 18px 22px; margin: 14px 0;
               box-shadow: 0 1px 4px rgba(0,0,0,0.08); }}
      table {{ border-collapse: separate; border-spacing: 0; width: 100%; font-size: 0.9em;
               border-radius: 8px; overflow: hidden; border: 1px solid #E0E4E8; }}
      thead th {{ background: #8E44AD; color: white; padding: 12px 14px; text-align: left;
                  font-weight: 600; font-size: 0.85em; text-transform: uppercase; }}
      tbody td {{ padding: 10px 14px; border-bottom: 1px solid #EEF0F2; }}
      tbody tr:nth-child(even) td {{ background: #F8F9FB; }}
      tbody tr:hover td {{ background: #EBF0F5; }}
      .footer {{ color: #AAA; font-size: 0.75em; margin-top: 40px; text-align: center; }}
    </style>
    </head>
    <body>
    <h1>Cross-Developer Comparison</h1>
    <p class="subtitle">Generated {datetime.now().strftime('%Y-%m-%d %H:%M')} &bull; {len(all_reports)} developers</p>
    <div class="card">
    <h2>Developer Metrics Comparison</h2>
    {comparison_df.to_html(index=False, border=0)}
    </div>
    <p class="footer">Generated by qc_trace qualitative analytics (003)</p>
    </body></html>
    """)
    comp_path = os.path.join(OUTPUT_DIR, "cross_developer_comparison.html")
    with open(comp_path, "w") as f:
        f.write(comparison_html)
    display(Markdown(f"**Cross-developer comparison saved: `{comp_path}`**"))
    display(comparison_df)

display(Markdown(f"**Done. {len(all_reports)} qualitative report(s) + comparison in `{OUTPUT_DIR}/`**"))

# %% [markdown]
# ---
# *End of notebook.*
