# %% [markdown]
# # Notebook 3: File Edit Patterns
# Extract file paths from Edit/Write/MultiEdit tool calls.
# Map hotspots, exploration-to-edit ratios, and repeated-edit signals.

# %%
import sys, os, re, json
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))

import pandas as pd
import numpy as np
from utils.connection import init, query_df
from utils import sessions, tools
from IPython.display import display, Markdown

pd.set_option("display.max_columns", 30)
pd.set_option("display.max_rows", 100)
pd.set_option("display.max_colwidth", 120)

conn = init()
display(Markdown("**Connected.**"))

# %% [markdown]
# ## 1 — Extract file paths from edit tool calls

# %%
# Edit tools across CLIs
EDIT_TOOLS = ("Edit", "Write", "MultiEdit", "MultiEditTool")
EXPLORE_TOOLS = ("Read", "Grep", "Glob", "View", "ReadFile", "ListFiles",
                 "read_file", "list_dir", "grep_search", "codebase_search",
                 "file_search")

edit_calls = query_df(conn, """
    SELECT tc.message_id, tc.tool_name, tc.tool_input,
           m.session_id, m.timestamp,
           s.user_email, s.org, s.repo_name, s.cwd
    FROM   tool_calls tc
    JOIN   messages m ON m.id = tc.message_id
    JOIN   sessions s ON s.id = m.session_id
    WHERE  tc.tool_name = ANY(%s)
    ORDER  BY m.timestamp ASC
""", (list(EDIT_TOOLS),))

display(Markdown(f"**Total edit tool calls:** {len(edit_calls)}"))
display(edit_calls.groupby(["user_email", "tool_name"]).size().to_frame("count").reset_index())

# %%
def extract_file_path(tool_input, tool_name):
    """Extract file_path from tool_input JSONB."""
    if tool_input is None:
        return None
    if isinstance(tool_input, str):
        try:
            tool_input = json.loads(tool_input)
        except (json.JSONDecodeError, TypeError):
            return None
    if isinstance(tool_input, dict):
        return tool_input.get("file_path") or tool_input.get("path") or tool_input.get("target_file")
    return None

edit_calls["file_path"] = edit_calls.apply(
    lambda r: extract_file_path(r["tool_input"], r["tool_name"]), axis=1
)

display(Markdown(f"**Paths extracted:** {edit_calls['file_path'].notna().sum()} / {len(edit_calls)}"))

# %%
# Make relative paths
def make_relative(file_path, cwd):
    if not file_path or not cwd:
        return file_path
    if file_path.startswith(cwd):
        rel = file_path[len(cwd):].lstrip("/")
        return rel if rel else file_path
    return file_path

edit_calls["rel_path"] = edit_calls.apply(
    lambda r: make_relative(r["file_path"], r["cwd"]), axis=1
)

# File extension
edit_calls["ext"] = edit_calls["rel_path"].apply(
    lambda p: os.path.splitext(p)[1].lower() if p else None
)

# %% [markdown]
# ## 2 — Top edited files

# %%
display(Markdown("### Top edited files — overall"))
top_files = (
    edit_calls.groupby("rel_path")
    .agg(edit_count=("message_id", "count"), sessions=("session_id", "nunique"))
    .sort_values("edit_count", ascending=False)
    .head(30)
)
display(top_files)

display(Markdown("### Top edited files — per user"))
for email in edit_calls["user_email"].unique():
    mask = edit_calls["user_email"] == email
    display(Markdown(f"#### {email}"))
    display(
        edit_calls[mask].groupby("rel_path")
        .agg(edit_count=("message_id", "count"), sessions=("session_id", "nunique"))
        .sort_values("edit_count", ascending=False)
        .head(20)
    )

# %% [markdown]
# ## 3 — File type breakdown

# %%
display(Markdown("### Edits by file extension"))
display(
    edit_calls.groupby("ext")
    .agg(edit_count=("message_id", "count"), unique_files=("rel_path", "nunique"))
    .sort_values("edit_count", ascending=False)
    .head(15)
)

# %% [markdown]
# ## 4 — Edit count per file per session (struggle signal)

# %%
edits_per_file_session = (
    edit_calls.groupby(["session_id", "rel_path", "user_email"])
    .agg(edit_count=("message_id", "count"))
    .reset_index()
    .sort_values("edit_count", ascending=False)
)

display(Markdown("### Files edited many times in a single session (struggle signal)"))
display(edits_per_file_session[edits_per_file_session["edit_count"] >= 3].head(30))

display(Markdown("### Distribution of edits per file per session"))
display(edits_per_file_session["edit_count"].describe().round(1).to_frame())

# %% [markdown]
# ## 5 — Config churn detection

# %%
CONFIG_PATTERNS = r"(\.env|config\.|\.toml|\.yaml|\.yml|\.json|Dockerfile|docker-compose|Makefile|\.ini|\.cfg)"

config_edits = edit_calls[
    edit_calls["rel_path"].str.contains(CONFIG_PATTERNS, case=False, na=False)
]

display(Markdown(f"### Config file edits: {len(config_edits)}"))
if not config_edits.empty:
    display(
        config_edits.groupby("rel_path")
        .agg(edit_count=("message_id", "count"), sessions=("session_id", "nunique"))
        .sort_values("sessions", ascending=False)
    )

# %% [markdown]
# ## 6 — Test file ratio

# %%
TEST_PATTERNS = r"(test_|_test\.|\.test\.|\.spec\.|tests/|__tests__|spec/)"

edit_calls["is_test"] = edit_calls["rel_path"].str.contains(TEST_PATTERNS, case=False, na=False)
test_ratio = edit_calls.groupby("user_email")["is_test"].mean().round(3) * 100
display(Markdown("### Test file edit ratio (% of all edits)"))
display(test_ratio.to_frame("test_edit_pct"))

# %% [markdown]
# ## 7 — Exploration-to-edit ratio

# %%
explore_calls = query_df(conn, """
    SELECT tc.message_id, tc.tool_name, m.session_id, m.timestamp,
           s.user_email
    FROM   tool_calls tc
    JOIN   messages m ON m.id = tc.message_id
    JOIN   sessions s ON s.id = m.session_id
    WHERE  tc.tool_name = ANY(%s)
    ORDER  BY m.timestamp ASC
""", (list(EXPLORE_TOOLS),))

display(Markdown(f"**Total explore calls:** {len(explore_calls)}"))

# Per-session: explore count vs edit count
explore_per_session = explore_calls.groupby("session_id").size().to_frame("explore_count")
edit_per_session = edit_calls.groupby("session_id").size().to_frame("edit_count")

ratio_df = explore_per_session.join(edit_per_session, how="outer").fillna(0)
ratio_df["explore_to_edit"] = (ratio_df["explore_count"] / ratio_df["edit_count"].replace(0, np.nan)).round(2)

display(Markdown("### Exploration-to-edit ratio per session"))
display(ratio_df.describe().round(2))

# By user
all_sess = sessions.list_all(conn, limit=500)
ratio_df = ratio_df.join(all_sess.set_index("id")[["user_email"]])
display(Markdown("### Avg exploration-to-edit ratio per user"))
display(ratio_df.groupby("user_email")["explore_to_edit"].describe().round(2))

# %% [markdown]
# ## 8 — Time to first edit per session

# %%
first_edit_per_session = edit_calls.sort_values("timestamp").groupby("session_id")["timestamp"].first()
session_starts = all_sess.set_index("id")["first_seen"]

time_to_first_edit = (first_edit_per_session - session_starts).dropna().dt.total_seconds() / 60

display(Markdown("### Time to first edit (minutes)"))
display(time_to_first_edit.describe().round(1).to_frame("minutes"))

# %% [markdown]
# ## 9 — Core modules (edited in >30% of sessions per repo)

# %%
display(Markdown("### Core modules — files edited in many sessions"))
for repo in edit_calls["repo_name"].dropna().unique():
    repo_edits = edit_calls[edit_calls["repo_name"] == repo]
    total_sessions = repo_edits["session_id"].nunique()
    file_session_pct = (
        repo_edits.groupby("rel_path")["session_id"].nunique() / total_sessions * 100
    ).sort_values(ascending=False)
    core = file_session_pct[file_session_pct >= 30]
    if not core.empty:
        display(Markdown(f"#### {repo} ({total_sessions} sessions)"))
        display(core.round(1).to_frame("pct_sessions"))

# %% [markdown]
# ---
# *End of Notebook 3.*
