# %% [markdown]
# # Notebook 8: Synthesis — Developer Profiles & Actionable Recommendations
# Combines insights from all prior notebooks into per-developer profiles
# and context pre-loading recommendations.

# %%
import sys, os, re, json
from itertools import combinations
from collections import Counter

sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))

import pandas as pd
import numpy as np
from utils.connection import init, query_df
from utils import sessions, messages, tokens, tools
from IPython.display import display, Markdown

pd.set_option("display.max_columns", 30)
pd.set_option("display.max_rows", 100)
pd.set_option("display.max_colwidth", 120)

conn = init()
display(Markdown("**Connected.**"))

# %% [markdown]
# ## 1 — Load all foundational data

# %%
all_sessions = sessions.list_all(conn, limit=500)
all_sessions["duration_min"] = (
    (all_sessions["last_updated"] - all_sessions["first_seen"]).dt.total_seconds() / 60
)
users_list = all_sessions["user_email"].unique()

# Message counts per session
msg_counts = query_df(conn, """
    SELECT m.session_id,
           COUNT(*) AS total_msgs,
           COUNT(*) FILTER (WHERE m.msg_type = 'user') AS user_msgs,
           COUNT(*) FILTER (WHERE m.msg_type = 'tool_call') AS tool_calls
    FROM messages m GROUP BY m.session_id
""")

# User messages for intent/specificity
user_msgs = query_df(conn, """
    SELECT m.id, m.session_id, m.content, m.timestamp, s.user_email
    FROM messages m JOIN sessions s ON s.id = m.session_id
    WHERE m.msg_type = 'user' ORDER BY m.timestamp
""")

def strip_tags(text):
    """Remove <INSTRUCTION(S)>...</INSTRUCTION(S)> blocks — Codex system prompts, not user content."""
    if not text: return ""
    text = re.sub(r"<INSTRUCTIONS?>.*?</INSTRUCTIONS?>", "", text, flags=re.DOTALL | re.IGNORECASE)
    return text.strip()

user_msgs["clean"] = user_msgs["content"].apply(strip_tags)
user_msgs["word_count"] = user_msgs["clean"].str.split().str.len().fillna(0).astype(int)

# File edits
edit_calls = query_df(conn, """
    SELECT tc.tool_input, m.session_id, s.user_email, s.repo_name, s.cwd
    FROM tool_calls tc
    JOIN messages m ON m.id = tc.message_id
    JOIN sessions s ON s.id = m.session_id
    WHERE tc.tool_name IN ('Edit', 'Write', 'MultiEdit', 'MultiEditTool')
""")

def extract_path(ti):
    if isinstance(ti, str):
        try: ti = json.loads(ti)
        except: return None
    if isinstance(ti, dict):
        return ti.get("file_path") or ti.get("path") or ti.get("target_file")
    return None

def make_rel(fp, cwd):
    if fp and cwd and fp.startswith(cwd):
        return fp[len(cwd):].lstrip("/") or fp
    return fp

edit_calls["file_path"] = edit_calls["tool_input"].apply(extract_path)
edit_calls["rel_path"] = edit_calls.apply(lambda r: make_rel(r["file_path"], r["cwd"]), axis=1)

# %% [markdown]
# ## 2 — Build per-developer profiles

# %%
for email in users_list:
    display(Markdown(f"---\n# Developer Profile: `{email}`"))

    # --- Session stats ---
    user_sess = all_sessions[all_sessions["user_email"] == email]
    n_sess = len(user_sess)
    user_msg_counts = msg_counts[msg_counts["session_id"].isin(user_sess["id"])]

    display(Markdown("## Session Overview"))
    display(Markdown(
        f"- **Sessions:** {n_sess}\n"
        f"- **CLIs used:** {', '.join(user_sess['source'].unique())}\n"
        f"- **Repos:** {', '.join(user_sess['repo_name'].dropna().unique())}\n"
        f"- **Avg duration:** {user_sess['duration_min'].mean():.0f} min\n"
        f"- **Median duration:** {user_sess['duration_min'].median():.0f} min"
    ))

    display(Markdown("## Message Style"))
    user_um = user_msgs[user_msgs["user_email"] == email]
    avg_wc = user_um["word_count"].mean()
    median_wc = user_um["word_count"].median()

    # Specificity
    def spec_score(text):
        if not text: return 0
        s = 0
        if re.search(r"[\w/]+\.\w{1,5}\b", text): s += 1
        if re.search(r"\b[a-z]+[A-Z]\w+\b|\b[a-z]+_[a-z]+\w*\b", text): s += 1
        if re.search(r"(Error|Exception|Traceback)", text, re.IGNORECASE): s += 1
        if "```" in text: s += 1
        return s

    user_um_specs = user_um["clean"].apply(spec_score)
    avg_spec = user_um_specs.mean()
    vague_pct = ((user_um["word_count"] < 10) & (user_um_specs == 0)).mean() * 100

    display(Markdown(
        f"- **Avg words/message:** {avg_wc:.0f} (median: {median_wc:.0f})\n"
        f"- **Avg specificity:** {avg_spec:.2f}/4\n"
        f"- **Vague prompt rate:** {vague_pct:.0f}%\n"
        f"- **Total user messages:** {len(user_um)}"
    ))

    # --- Tool preferences ---
    display(Markdown("## Tool Preferences"))
    tf = tools.call_frequency(conn, email=email)
    if not tf.empty:
        total_tools = tf["call_count"].sum()
        explore = tf[tf["tool_name"].isin(["Read", "Grep", "Glob", "View", "ReadFile", "read_file"])]["call_count"].sum()
        edit = tf[tf["tool_name"].isin(["Edit", "Write", "MultiEdit"])]["call_count"].sum()
        shell = tf[tf["tool_name"].isin(["Bash", "shell_command", "exec_command"])]["call_count"].sum()
        display(Markdown(
            f"- **Total tool calls:** {total_tools}\n"
            f"- **Explore (Read/Grep/Glob):** {explore} ({explore/total_tools*100:.0f}%)\n"
            f"- **Edit (Edit/Write):** {edit} ({edit/total_tools*100:.0f}%)\n"
            f"- **Shell (Bash/shell_command):** {shell} ({shell/total_tools*100:.0f}%)"
        ))
        display(Markdown("**Top 10 tools:**"))
        display(tf.head(10))

    # --- File hotspots ---
    display(Markdown("## File Hotspots"))
    user_edits = edit_calls[edit_calls["user_email"] == email]
    if not user_edits.empty:
        top_files = (
            user_edits.groupby("rel_path")["session_id"].nunique()
            .sort_values(ascending=False).head(10)
        )
        display(top_files.to_frame("sessions_edited"))

    # --- Archetype ---
    display(Markdown("## Archetype"))
    if not user_msg_counts.empty:
        avg_um = user_msg_counts["user_msgs"].mean()
        avg_tc = user_msg_counts["tool_calls"].mean()
        archetype = []
        if avg_um <= 3 and avg_tc > 10:
            archetype.append("One-shotter")
        if avg_um > 10:
            archetype.append("Iterator")
        if explore > edit * 2 if not tf.empty else False:
            archetype.append("Explorer")
        if not archetype:
            archetype.append("Balanced")
        display(Markdown(f"**{', '.join(archetype)}** "
                         f"(avg {avg_um:.1f} user msgs, {avg_tc:.1f} tool calls/session)"))

# %% [markdown]
# ## 3 — Context pre-loading recommendations

# %%
display(Markdown("---\n# Context Pre-loading Recommendations"))
display(Markdown(
    "For each user+repo, these are the files that should be pre-loaded "
    "based on edit frequency and co-occurrence patterns."
))

for email in users_list:
    user_edits = edit_calls[edit_calls["user_email"] == email]
    for repo in user_edits["repo_name"].dropna().unique():
        repo_edits = user_edits[user_edits["repo_name"] == repo]
        total_sess = repo_edits["session_id"].nunique()

        # Top files by session frequency
        file_freq = (
            repo_edits.groupby("rel_path")["session_id"].nunique()
            .sort_values(ascending=False)
        )
        core_files = file_freq[file_freq >= max(2, total_sess * 0.2)].head(10)

        if not core_files.empty:
            display(Markdown(f"### {email} / {repo}"))
            display(Markdown(f"*{total_sess} sessions in this repo*"))
            display(Markdown("**Always pre-load these files:**"))
            for f, count in core_files.items():
                pct = count / total_sess * 100
                print(f"  {f}  ({count} sessions, {pct:.0f}%)")

# %% [markdown]
# ## 4 — First-message clustering

# %%
display(Markdown("---\n# First-Message Patterns"))

first_msgs = user_msgs.sort_values("timestamp").groupby("session_id").first().reset_index()

for email in users_list:
    user_first = first_msgs[first_msgs["user_email"] == email]
    display(Markdown(f"### {email} — typical opening prompts"))

    # Intent keywords
    INTENTS = {
        "fix/debug": r"\b(fix|bug|error|debug|broken|fail)\b",
        "create/add": r"\b(create|add|implement|build|new|make)\b",
        "refactor": r"\b(refactor|clean|rename|move|extract)\b",
        "explain": r"\b(explain|understand|what|how|why)\b",
    }
    for intent, pattern in INTENTS.items():
        count = user_first["clean"].str.contains(pattern, case=False, na=False).sum()
        if count > 0:
            pct = count / len(user_first) * 100
            print(f"  {intent}: {count} sessions ({pct:.0f}%)")

    # Sample first messages
    display(Markdown("**Sample first messages:**"))
    samples = user_first.nlargest(5, "word_count")[["clean", "word_count"]]
    for _, r in samples.iterrows():
        text = r["clean"][:200] + ("..." if len(r["clean"]) > 200 else "")
        print(f"  [{r['word_count']}w] {text}")

# %% [markdown]
# ## 5 — Common tool sequences

# %%
display(Markdown("---\n# Common Tool Sequences"))

tool_seq = query_df(conn, """
    SELECT tc.tool_name, m.session_id, m.timestamp, s.user_email
    FROM tool_calls tc
    JOIN messages m ON m.id = tc.message_id
    JOIN sessions s ON s.id = m.session_id
    ORDER BY m.session_id, m.timestamp
""")

# Build bigrams
for email in users_list:
    user_tools = tool_seq[tool_seq["user_email"] == email]
    bigrams = Counter()
    for _, group in user_tools.groupby("session_id"):
        names = group["tool_name"].tolist()
        for a, b in zip(names[:-1], names[1:]):
            bigrams[(a, b)] += 1

    display(Markdown(f"### {email} — top tool transitions"))
    top_bigrams = bigrams.most_common(15)
    for (a, b), count in top_bigrams:
        print(f"  {a} → {b}: {count}")

# %% [markdown]
# ## 6 — Key actionable insights

# %%
display(Markdown("---\n# Key Actionable Insights"))
display(Markdown("""
## Summary of findings:

### 1. Context pre-loading
Each developer has a consistent set of files they touch across sessions.
Pre-loading these files eliminates 2-3 rounds of Read/Grep exploration
that currently happen at the start of every session.

### 2. Prompt specificity matters
Sessions started with specific prompts (file names, function names, error messages)
tend to be shorter and require fewer tool calls. Coaching developers to include
more context in their first message has the highest ROI.

### 3. Exploration overhead is real
The Read→Grep→Read→Edit pattern dominates tool sequences. A significant portion
of every session is spent re-discovering the codebase. Pre-loaded context
directly reduces this overhead.

### 4. Developer-specific strategies needed
Different developers have different styles (one-shot vs iterative, explorer vs editor).
Context loading strategies should match the developer's style:
- **One-shotters**: maximum upfront context, file contents pre-loaded
- **Iterators**: session memory, remember corrections from previous turns
- **Explorers**: codebase map, dependency graphs pre-loaded

### 5. Tool normalization across CLIs
shell_command (Codex), Bash (Claude Code), and exec_command (Gemini) are the same
concept. Any cross-CLI analytics must normalize these.
"""))

# %% [markdown]
# ---
# *End of Notebook 8.*
