# %% [markdown]
# # Developer Analytics
# Filterable, minimal, strong metrics — org-level, dev-level, week-on-week.

# %%
import sys, os, re, json
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))

import pandas as pd
import numpy as np
from utils.connection import init, query_df
from IPython.display import display, Markdown

pd.set_option("display.max_columns", 30)
pd.set_option("display.max_rows", 200)
pd.set_option("display.max_colwidth", 100)
pd.set_option("display.float_format", lambda x: f"{x:.1f}")

conn = init()

# %% [markdown]
# ## Filters
# Set these to scope all analysis below.

# %%
# --- CONFIGURE FILTERS ---
ORG_FILTER = None              # e.g. "pratilipi-platform" or None for all
REPO_PREFIX = "Pratilipi/"     # only repos matching this prefix, or None for all
USER_FILTER = None             # set to an email to see single-dev deep dive, None for org overview
# --------------------------

# %% [markdown]
# ## Load filtered data

# %%
# -- Sessions --
sess_clauses, sess_params = [], []
if ORG_FILTER:
    sess_clauses.append("s.org = %s")
    sess_params.append(ORG_FILTER)
if REPO_PREFIX:
    sess_clauses.append("s.repo_name LIKE %s")
    sess_params.append(f"{REPO_PREFIX}%")
if USER_FILTER:
    sess_clauses.append("s.user_email = %s")
    sess_params.append(USER_FILTER)

where = f"WHERE {' AND '.join(sess_clauses)}" if sess_clauses else ""

sess = query_df(conn, f"""
    SELECT s.*,
           EXTRACT(EPOCH FROM (s.last_updated - s.first_seen)) / 60.0 AS duration_min,
           DATE_TRUNC('week', s.first_seen)::date AS week
    FROM sessions s
    {where}
    ORDER BY s.first_seen DESC
""", tuple(sess_params))

session_ids = sess["id"].tolist()
display(Markdown(f"**{len(sess)} sessions** loaded "
    f"(org={ORG_FILTER or 'all'}, repo_prefix={REPO_PREFIX or 'all'}, user={USER_FILTER or 'all'})"))

if sess.empty:
    raise SystemExit("No sessions match filters.")

# -- Messages (counts per session) --
msg_stats = query_df(conn, """
    SELECT m.session_id,
           COUNT(*) AS total_msgs,
           COUNT(*) FILTER (WHERE m.msg_type = 'user')        AS user_msgs,
           COUNT(*) FILTER (WHERE m.msg_type = 'assistant')   AS assistant_msgs,
           COUNT(*) FILTER (WHERE m.msg_type = 'tool_call')   AS tool_calls,
           COUNT(*) FILTER (WHERE m.msg_type = 'tool_result') AS tool_results
    FROM messages m
    WHERE m.session_id = ANY(%s)
    GROUP BY m.session_id
""", (session_ids,))

# -- Tokens per session --
tok_stats = query_df(conn, """
    SELECT m.session_id,
           SUM(t.input_tokens)                    AS input_tok,
           SUM(t.output_tokens)                   AS output_tok,
           SUM(t.cached_tokens)                   AS cached_tok,
           SUM(t.input_tokens + t.output_tokens)  AS billed_tok
    FROM token_usage t
    JOIN messages m ON m.id = t.message_id
    WHERE m.session_id = ANY(%s)
    GROUP BY m.session_id
""", (session_ids,))

# -- Tool calls (normalized name) --
tool_calls = query_df(conn, """
    SELECT tc.tool_name, tc.tool_input, m.session_id, m.timestamp,
           s.user_email, s.repo_name, s.cwd
    FROM tool_calls tc
    JOIN messages m ON m.id = tc.message_id
    JOIN sessions s ON s.id = m.session_id
    WHERE m.session_id = ANY(%s)
    ORDER BY m.timestamp
""", (session_ids,))

SHELL_NAMES = {"Bash", "shell_command", "exec_command"}
EDIT_NAMES  = {"Edit", "Write", "MultiEdit", "MultiEditTool"}
EXPLORE_NAMES = {"Read", "Grep", "Glob", "View", "ReadFile", "read_file",
                 "grep_search", "codebase_search", "list_dir", "file_search"}

tool_calls["tool_group"] = tool_calls["tool_name"].apply(
    lambda t: "shell" if t in SHELL_NAMES
    else "edit" if t in EDIT_NAMES
    else "explore" if t in EXPLORE_NAMES
    else "other"
)

# -- Git commit detection --
git_sessions = set()
shell_calls = tool_calls[tool_calls["tool_group"] == "shell"]
for _, r in shell_calls.iterrows():
    ti = r["tool_input"]
    if isinstance(ti, dict):
        ti = json.dumps(ti)
    elif ti is None:
        continue
    if isinstance(ti, str) and ("git commit" in ti.lower() or "git push" in ti.lower()):
        git_sessions.add(r["session_id"])

# -- Merge everything into one session-level DataFrame --
df = sess.merge(msg_stats, left_on="id", right_on="session_id", how="left")
df = df.merge(tok_stats, left_on="id", right_on="session_id", how="left", suffixes=("", "_tok"))
df["has_commit"] = df["id"].isin(git_sessions)
df["tool_calls"] = df["tool_calls"].fillna(0).astype(int)
df["user_msgs"]  = df["user_msgs"].fillna(0).astype(int)
df["billed_tok"] = df["billed_tok"].fillna(0).astype(int)

# Tool group counts per session
tool_group_counts = (
    tool_calls.groupby(["session_id", "tool_group"]).size()
    .unstack(fill_value=0)
    .rename(columns=lambda c: f"tc_{c}")
)
df = df.merge(tool_group_counts, left_on="id", right_index=True, how="left")
for c in ["tc_explore", "tc_edit", "tc_shell", "tc_other"]:
    if c not in df.columns:
        df[c] = 0
    df[c] = df[c].fillna(0).astype(int)

# %% [markdown]
# ---
# # Part 1 — Org-Level Overview

# %% [markdown]
# ## Org summary

# %%
display(Markdown("### Org summary"))
org_summary = df.groupby("org").agg(
    devs=("user_email", "nunique"),
    sessions=("id", "count"),
    repos=("repo_name", "nunique"),
    total_billed_tokens=("billed_tok", "sum"),
    sessions_with_commit=("has_commit", "sum"),
).reset_index()
org_summary["commit_rate_%"] = (org_summary["sessions_with_commit"] / org_summary["sessions"] * 100).round(1)
display(org_summary)

# %% [markdown]
# ## Per-dev scoreboard

# %%
display(Markdown("### Per-dev scoreboard"))
dev_board = df.groupby("user_email").agg(
    sessions=("id", "count"),
    repos=("repo_name", "nunique"),
    median_duration_min=("duration_min", "median"),
    median_user_msgs=("user_msgs", "median"),
    median_tool_calls=("tool_calls", "median"),
    total_billed_tok=("billed_tok", "sum"),
    commits=("has_commit", "sum"),
    explore_calls=("tc_explore", "sum"),
    edit_calls=("tc_edit", "sum"),
    shell_calls=("tc_shell", "sum"),
).reset_index()
dev_board["commit_rate_%"] = (dev_board["commits"] / dev_board["sessions"] * 100).round(1)
dev_board["explore_edit_ratio"] = (dev_board["explore_calls"] / dev_board["edit_calls"].replace(0, np.nan)).round(1)
display(dev_board)

# %% [markdown]
# ---
# # Part 2 — Week-on-Week Trends

# %%
display(Markdown("### Week-on-week — all devs"))
wow = df.groupby("week").agg(
    sessions=("id", "count"),
    active_devs=("user_email", "nunique"),
    median_duration_min=("duration_min", "median"),
    median_tool_calls=("tool_calls", "median"),
    total_billed_tok=("billed_tok", "sum"),
    commits=("has_commit", "sum"),
).reset_index().sort_values("week")
wow["commit_rate_%"] = (wow["commits"] / wow["sessions"] * 100).round(1)
display(wow)

# %%
display(Markdown("### Week-on-week — per dev"))
wow_dev = df.groupby(["week", "user_email"]).agg(
    sessions=("id", "count"),
    median_duration_min=("duration_min", "median"),
    median_user_msgs=("user_msgs", "median"),
    total_billed_tok=("billed_tok", "sum"),
    commits=("has_commit", "sum"),
    explore=("tc_explore", "sum"),
    edits=("tc_edit", "sum"),
).reset_index().sort_values(["user_email", "week"])
wow_dev["commit_rate_%"] = (wow_dev["commits"] / wow_dev["sessions"] * 100).round(1)
display(wow_dev)

# %% [markdown]
# ---
# # Part 3 — Per-Repo Breakdown

# %%
display(Markdown("### Per-repo activity"))
repo_board = df.groupby("repo_name").agg(
    sessions=("id", "count"),
    devs=("user_email", "nunique"),
    median_duration_min=("duration_min", "median"),
    total_billed_tok=("billed_tok", "sum"),
    commits=("has_commit", "sum"),
).reset_index().sort_values("sessions", ascending=False)
repo_board["commit_rate_%"] = (repo_board["commits"] / repo_board["sessions"] * 100).round(1)
display(repo_board)

# %% [markdown]
# ---
# # Part 4 — Single-Dev Deep Dive
# Set `USER_FILTER` at the top to a specific email to activate this section.

# %%
if USER_FILTER:
    display(Markdown(f"---\n# Deep Dive: `{USER_FILTER}`"))
    udf = df[df["user_email"] == USER_FILTER]
    utc = tool_calls[tool_calls["user_email"] == USER_FILTER]

    # --- Overview ---
    display(Markdown("## Overview"))
    display(Markdown(
        f"- **Sessions:** {len(udf)}\n"
        f"- **Repos:** {', '.join(udf['repo_name'].dropna().unique())}\n"
        f"- **CLIs:** {', '.join(udf['source'].unique())}\n"
        f"- **Date range:** {udf['first_seen'].min():%Y-%m-%d} to {udf['first_seen'].max():%Y-%m-%d}\n"
        f"- **Median duration:** {udf['duration_min'].median():.0f} min\n"
        f"- **Median user msgs/session:** {udf['user_msgs'].median():.0f}\n"
        f"- **Commit rate:** {udf['has_commit'].mean()*100:.0f}%\n"
        f"- **Total billed tokens:** {udf['billed_tok'].sum():,}"
    ))

    # --- Session distribution ---
    display(Markdown("## Session profile"))
    display(udf[["duration_min", "user_msgs", "tool_calls", "tc_explore", "tc_edit", "tc_shell", "billed_tok"]].describe().round(1))

    # --- Week-on-week for this dev ---
    display(Markdown("## Week-on-week"))
    u_wow = udf.groupby("week").agg(
        sessions=("id", "count"),
        median_duration_min=("duration_min", "median"),
        median_user_msgs=("user_msgs", "median"),
        total_billed_tok=("billed_tok", "sum"),
        commits=("has_commit", "sum"),
        explore=("tc_explore", "sum"),
        edits=("tc_edit", "sum"),
    ).reset_index().sort_values("week")
    u_wow["commit_rate_%"] = (u_wow["commits"] / u_wow["sessions"] * 100).round(1)
    u_wow["explore_edit_ratio"] = (u_wow["explore"] / u_wow["edits"].replace(0, np.nan)).round(1)
    display(u_wow)

    # --- Per-repo for this dev ---
    display(Markdown("## Per-repo"))
    u_repo = udf.groupby("repo_name").agg(
        sessions=("id", "count"),
        median_duration_min=("duration_min", "median"),
        total_billed_tok=("billed_tok", "sum"),
        commits=("has_commit", "sum"),
    ).reset_index().sort_values("sessions", ascending=False)
    u_repo["commit_rate_%"] = (u_repo["commits"] / u_repo["sessions"] * 100).round(1)
    display(u_repo)

    # --- Top tools ---
    display(Markdown("## Top tools"))
    display(
        utc.groupby("tool_name").size()
        .sort_values(ascending=False).head(15)
        .to_frame("calls")
    )

    # --- Top edited files ---
    display(Markdown("## Top edited files"))
    edit_tc = utc[utc["tool_group"] == "edit"].copy()
    def extract_fp(ti):
        if isinstance(ti, dict):
            return ti.get("file_path") or ti.get("path") or ti.get("target_file")
        return None
    def make_rel(fp, cwd):
        if fp and cwd and fp.startswith(cwd):
            return fp[len(cwd):].lstrip("/") or fp
        return fp

    edit_tc["file_path"] = edit_tc["tool_input"].apply(extract_fp)
    edit_tc["rel_path"] = edit_tc.apply(lambda r: make_rel(r["file_path"], r["cwd"]), axis=1)
    if not edit_tc["rel_path"].isna().all():
        display(
            edit_tc.groupby("rel_path").agg(
                edits=("session_id", "count"),
                sessions=("session_id", "nunique"),
            ).sort_values("sessions", ascending=False).head(20)
        )

    # --- Struggle signal: files edited 5+ times in one session ---
    display(Markdown("## Struggle signals (5+ edits to same file in one session)"))
    if not edit_tc["rel_path"].isna().all():
        struggle = (
            edit_tc.groupby(["session_id", "rel_path"]).size()
            .reset_index(name="edit_count")
            .query("edit_count >= 5")
            .sort_values("edit_count", ascending=False)
        )
        if struggle.empty:
            display(Markdown("*None detected.*"))
        else:
            display(struggle.head(20))

    # --- First message patterns ---
    display(Markdown("## First message patterns"))
    user_msgs_raw = query_df(conn, """
        SELECT m.session_id, m.content, m.timestamp
        FROM messages m
        WHERE m.session_id = ANY(%s) AND m.msg_type = 'user'
        ORDER BY m.timestamp
    """, (udf["id"].tolist(),))

    if not user_msgs_raw.empty:
        user_msgs_raw["clean"] = user_msgs_raw["content"].apply(
            lambda t: re.sub(r"<INSTRUCTIONS?>.*?</INSTRUCTIONS?>", "", t or "", flags=re.DOTALL|re.IGNORECASE).strip()
        )
        first = user_msgs_raw.sort_values("timestamp").groupby("session_id").first().reset_index()
        first["word_count"] = first["clean"].str.split().str.len().fillna(0).astype(int)
        display(Markdown(f"Median first-message length: **{first['word_count'].median():.0f} words**"))
        display(Markdown("**Sample opening prompts (longest):**"))
        for _, r in first.nlargest(5, "word_count").iterrows():
            text = r["clean"][:300] + ("..." if len(r["clean"]) > 300 else "")
            print(f"  [{r['word_count']}w] {text}\n")

else:
    display(Markdown("*Set `USER_FILTER` to a specific email at the top to see the single-dev deep dive.*"))

# %% [markdown]
# ---
# *End of notebook.*
