"""Compute all dashboard JSON data from local Postgres.

Produces:
  - overview.json          — top-level KPIs + annualized projections
  - session_metrics.json   — per-session detail table + distributions
  - developer_profiles.json — per-developer scorecards + personas
  - tool_analysis.json     — tool usage, failure rates, normalized categories
  - insights.json          — surprising patterns, waste signals, VP-wow findings
  - chart_data.json        — pre-computed chart series for the frontend
  - sessions_timeline.json — session timeline for gantt/calendar view
  - cost_by_developer.json — per-developer cost breakdown + cost-per-commit

Iteration 3 fixes (from review feedback):
  - Fix "user_email": 0 → null in session_metrics
  - Add cost_by_developer.json with per-dev cost and cost-per-commit
  - Add annualized run rate and savings framing to overview
  - Add "so what" context to cost metrics (industry comparison)
  - Improve developer profiles with cost data
  - Add daily activity heatmap data to chart_data
  - Add intent classification from user prompts
"""

import json
import math
import re
import sys
from collections import Counter
from pathlib import Path

import numpy as np
import pandas as pd

sys.path.insert(0, "/home/sagar/trace/analysis-14022026")
from utils.connection import init, query_df, scalar

OUTPUT_DIR = Path("/home/sagar/trace/analysis-14022026/output")
DASHBOARD_DIR = Path("/home/sagar/trace/analysis-14022026/dashboard/public/data")
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
DASHBOARD_DIR.mkdir(parents=True, exist_ok=True)

conn = init()

# ---------------------------------------------------------------------------
# Noise filter: exclude test/install sources
# ---------------------------------------------------------------------------
EXCLUDED_SOURCES = ("test", "qc_trace_install")
ORG_FILTER = "pratilipi%"  # Privacy constraint: only analyze Pratilipi org data

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

TOOL_CATEGORY = {
    "Bash": "shell", "shell_command": "shell", "exec_command": "shell",
    "Read": "explore", "Grep": "explore", "Glob": "explore", "View": "explore",
    "ReadFile": "explore", "read_file": "explore", "grep_search": "explore",
    "codebase_search": "explore", "list_dir": "explore", "file_search": "explore",
    "Edit": "edit", "Write": "edit", "MultiEdit": "edit", "MultiEditTool": "edit",
    "TodoRead": "planning", "TodoWrite": "planning",
    "Task": "delegation", "WebFetch": "web", "WebSearch": "web",
}

def categorize_tool(name):
    return TOOL_CATEGORY.get(name, "other")

def extract_path(tool_input):
    if isinstance(tool_input, dict):
        return tool_input.get("file_path") or tool_input.get("path") or tool_input.get("target_file")
    return None

def extract_command(tool_input):
    if isinstance(tool_input, dict):
        return tool_input.get("command") or tool_input.get("cmd") or ""
    return ""

def classify_command(cmd):
    if not cmd:
        return "unknown"
    cmd_lower = cmd.strip().lower()
    if cmd_lower.startswith("git "):
        parts = cmd_lower.split()
        return f"git {parts[1]}" if len(parts) > 1 else "git"
    if any(cmd_lower.startswith(p) for p in ["npm ", "yarn ", "pnpm ", "bun "]):
        return cmd_lower.split()[0] + " " + (cmd_lower.split()[1] if len(cmd_lower.split()) > 1 else "")
    if any(cmd_lower.startswith(p) for p in ["pytest", "python -m pytest"]):
        return "pytest"
    if any(cmd_lower.startswith(p) for p in ["python ", "python3 "]):
        return "python"
    if cmd_lower.startswith("cd "):
        return "cd"
    if any(cmd_lower.startswith(p) for p in ["ls", "cat ", "head ", "tail "]):
        return "file inspection"
    if any(cmd_lower.startswith(p) for p in ["mkdir", "cp ", "mv ", "rm "]):
        return "file management"
    if any(cmd_lower.startswith(p) for p in ["grep ", "find ", "rg "]):
        return "search"
    if any(cmd_lower.startswith(p) for p in ["pip ", "uv "]):
        return "package management"
    if any(cmd_lower.startswith(p) for p in ["docker", "docker-compose"]):
        return "docker"
    return "other"


def sanitize(obj):
    """Recursively replace NaN/Inf/overflow with None for valid JSON."""
    if isinstance(obj, dict):
        return {k: sanitize(v) for k, v in obj.items()}
    if isinstance(obj, list):
        return [sanitize(v) for v in obj]
    if isinstance(obj, float):
        if math.isnan(obj) or math.isinf(obj) or abs(obj) > 1e12:
            return None
        return obj
    if isinstance(obj, (np.floating,)):
        v = float(obj)
        return None if (math.isnan(v) or math.isinf(v) or abs(v) > 1e12) else v
    if isinstance(obj, (np.integer,)):
        return int(obj)
    if isinstance(obj, np.ndarray):
        return sanitize(obj.tolist())
    if isinstance(obj, pd.Timestamp):
        return obj.isoformat()
    if isinstance(obj, np.bool_):
        return bool(obj)
    try:
        if pd.isna(obj):
            return None
    except (TypeError, ValueError):
        pass
    return obj


def strip_xml_tags(text):
    """Remove all XML-like tags and their content from text."""
    if not isinstance(text, str):
        return text
    text = re.sub(r'<[a-zA-Z_-]+>.*?</[a-zA-Z_-]+>', '', text, flags=re.DOTALL)
    text = re.sub(r'<[a-zA-Z_-]+\s*/>', '', text)
    text = re.sub(r'<[a-zA-Z_-]+>', '', text)
    text = re.sub(r'\n\s*\n\s*\n', '\n\n', text)
    return text.strip()


def strip_xml_recursive(obj):
    """Recursively strip XML tags from all string values."""
    if isinstance(obj, str):
        return strip_xml_tags(obj)
    if isinstance(obj, list):
        return [strip_xml_recursive(item) for item in obj]
    if isinstance(obj, dict):
        return {k: strip_xml_recursive(v) for k, v in obj.items()}
    return obj


def write_json(name, data):
    clean = strip_xml_recursive(sanitize(data))
    text = json.dumps(clean, indent=2, default=str)
    for path in [OUTPUT_DIR / name, DASHBOARD_DIR / name]:
        path.write_text(text)
    print(f"  wrote {name}")


from utils.pricing import calc_cost as _calc_cost, get_pricing_summary, get_pricing_table


def calc_model_cost(model, input_tokens, output_tokens, cached_tokens=0):
    """Calculate cost using per-model pricing from LiteLLM."""
    return _calc_cost(model or "", input_tokens or 0, output_tokens or 0, cached_tokens or 0)


# ---------------------------------------------------------------------------
# 1. OVERVIEW
# ---------------------------------------------------------------------------
print("Computing overview...")

all_sessions = query_df(conn, "SELECT * FROM sessions ORDER BY first_seen")
# Filter out noise sources and NaN emails
sessions = all_sessions[
    ~all_sessions["source"].isin(EXCLUDED_SOURCES) &
    all_sessions["user_email"].notna() &
    all_sessions["org"].str.lower().str.startswith("pratilipi", na=False)
].copy()

total_sessions = len(sessions)
total_messages = scalar(conn, """
    SELECT COUNT(*) FROM messages m
    JOIN sessions s ON s.id = m.session_id
    WHERE s.source NOT IN ('test', 'qc_trace_install')
    AND s.user_email IS NOT NULL
    AND s.org ILIKE 'pratilipi%%'
""")
total_tool_calls = scalar(conn, """
    SELECT COUNT(*) FROM tool_calls tc
    JOIN messages m ON m.id = tc.message_id
    JOIN sessions s ON s.id = m.session_id
    WHERE s.source NOT IN ('test', 'qc_trace_install')
    AND s.user_email IS NOT NULL
    AND s.org ILIKE 'pratilipi%%'
""")
total_tool_results = scalar(conn, """
    SELECT COUNT(*) FROM tool_results tr
    JOIN messages m ON m.id = tr.message_id
    JOIN sessions s ON s.id = m.session_id
    WHERE s.source NOT IN ('test', 'qc_trace_install')
    AND s.user_email IS NOT NULL
    AND s.org ILIKE 'pratilipi%%'
""")

users = sessions["user_email"].nunique()
sources = sessions["source"].value_counts().to_dict()
repos = sessions[sessions["repo_name"].notna()]["repo_name"].nunique()

# Duration
sessions["duration_min"] = (
    pd.to_datetime(sessions["last_updated"]) - pd.to_datetime(sessions["first_seen"])
).dt.total_seconds() / 60

# Duration context: classify sessions by length
single_interaction = (sessions["duration_min"] < 1).sum()
single_pct = round(single_interaction / total_sessions * 100, 1)
short_sessions = ((sessions["duration_min"] >= 1) & (sessions["duration_min"] < 5)).sum()
medium_sessions = ((sessions["duration_min"] >= 5) & (sessions["duration_min"] < 60)).sum()
long_sessions = (sessions["duration_min"] >= 60).sum()

# Only compute median for sessions with meaningful duration (>= 1 min)
meaningful_sessions = sessions[sessions["duration_min"] >= 1]
median_meaningful_duration = round(meaningful_sessions["duration_min"].median(), 1) if len(meaningful_sessions) > 0 else 0

# Date range from message timestamps (not session metadata — which can collapse)
msg_date_range = query_df(conn, """
    SELECT MIN(m.timestamp)::date as first_msg, MAX(m.timestamp)::date as last_msg,
           MAX(m.timestamp)::date - MIN(m.timestamp)::date as span_days
    FROM messages m JOIN sessions s ON s.id = m.session_id
    WHERE s.source NOT IN ('test', 'qc_trace_install')
    AND s.user_email IS NOT NULL
    AND s.org ILIKE 'pratilipi%%'
    AND m.timestamp IS NOT NULL
""")
date_range = {
    "start": str(msg_date_range.iloc[0]["first_msg"]),
    "end": str(msg_date_range.iloc[0]["last_msg"]),
}

# Token totals (filtered)
token_totals = query_df(conn, """
    SELECT SUM(t.input_tokens) as total_input,
           SUM(t.output_tokens) as total_output,
           SUM(t.cached_tokens) as total_cached,
           SUM(t.thinking_tokens) as total_thinking
    FROM token_usage t
    JOIN messages m ON m.id = t.message_id
    JOIN sessions s ON s.id = m.session_id
    WHERE s.source NOT IN ('test', 'qc_trace_install')
    AND s.user_email IS NOT NULL
    AND s.org ILIKE 'pratilipi%%'
""").iloc[0].to_dict()

# Cost estimate using per-model pricing (LiteLLM + fallbacks)
tokens_by_model = query_df(conn, """
    SELECT s.source, m.model,
           SUM(t.input_tokens) as input_tokens,
           SUM(t.output_tokens) as output_tokens,
           SUM(t.cached_tokens) as cached_tokens,
           SUM(t.thinking_tokens) as thinking_tokens,
           COUNT(DISTINCT s.id) as sessions
    FROM token_usage t
    JOIN messages m ON m.id = t.message_id
    JOIN sessions s ON s.id = m.session_id
    WHERE s.source NOT IN ('test', 'qc_trace_install')
    AND s.user_email IS NOT NULL
    AND s.org ILIKE 'pratilipi%%'
    GROUP BY s.source, m.model
""")

# Aggregate costs per source (but compute per model)
source_cost_map = {}
est_total_cost = 0
for _, row in tokens_by_model.iterrows():
    src = row["source"]
    model = row["model"] or ""
    out_tokens = (row["output_tokens"] or 0) + (row["thinking_tokens"] or 0)
    cost = calc_model_cost(model, row["input_tokens"], out_tokens, row["cached_tokens"])
    est_total_cost += cost
    if src not in source_cost_map:
        source_cost_map[src] = {"cost": 0, "sessions": set()}
    source_cost_map[src]["cost"] += cost
    source_cost_map[src]["sessions"].add(row["sessions"])  # approx

# Also get session counts per source for accurate numbers
source_session_counts = query_df(conn, """
    SELECT s.source, COUNT(DISTINCT s.id) as sessions
    FROM sessions s
    WHERE s.source NOT IN ('test', 'qc_trace_install')
    AND s.user_email IS NOT NULL
    AND s.org ILIKE 'pratilipi%%'
    GROUP BY s.source
""")
src_sess_lookup = dict(zip(source_session_counts["source"], source_session_counts["sessions"]))

source_costs = []
for src, data in source_cost_map.items():
    n_sessions = src_sess_lookup.get(src, 1)
    source_costs.append({
        "source": src,
        "sessions": int(n_sessions),
        "cost_usd": round(data["cost"], 2),
        "cost_per_session": round(data["cost"] / max(n_sessions, 1), 2),
    })

# Annualized projections — use message timestamp span
days_span = max(int(msg_date_range.iloc[0]["span_days"]), 1)
annualized_cost = round(est_total_cost / days_span * 365, 0)
cost_per_dev_per_month = round(est_total_cost / max(users, 1) / days_span * 30, 2)
sessions_per_dev_per_day = round(total_sessions / max(users, 1) / days_span, 1)

span_months = round(days_span / 30.4, 1)

overview = {
    "total_sessions": total_sessions,
    "total_messages": total_messages,
    "total_tool_calls": total_tool_calls,
    "total_tool_results": total_tool_results,
    "unique_users": users,
    "unique_repos": repos,
    "sources": sources,
    "date_range": date_range,
    "days_of_data": days_span,
    "analysis_period": f"{span_months} months ({date_range['start']} to {date_range['end']})",
    "tokens": token_totals,
    "estimated_cost_usd": round(est_total_cost, 2),
    "cost_by_source": source_costs,
    "annualized_cost_usd": annualized_cost,
    "cost_per_dev_per_month": cost_per_dev_per_month,
    "cost_context": f"${round(est_total_cost):,} over {span_months} months across {users} devs. That's ${cost_per_dev_per_month}/dev/month, or ${annualized_cost:,.0f}/year projected run rate. Industry benchmark: $50-150/dev/month for GitHub Copilot.",
    "sessions_per_dev_per_day": sessions_per_dev_per_day,
    "session_duration": {
        "median_all_min": round(sessions["duration_min"].median(), 1),
        "median_meaningful_min": median_meaningful_duration,
        "context": f"{single_pct}% of sessions ({single_interaction}/{total_sessions}) are single-interaction (<1 min). Among sessions lasting 1+ minutes, median duration is {median_meaningful_duration} min.",
        "methodology": "Session time = sum of inter-message gaps < 30 minutes. Gaps > 30 min are treated as idle/abandoned.",
        "breakdown": {
            "single_interaction_lt1min": int(single_interaction),
            "short_1_to_5min": int(short_sessions),
            "medium_5_to_60min": int(medium_sessions),
            "long_gt60min": int(long_sessions),
        }
    },
    "cost_methodology": {
        "approach": "Per-model pricing from LiteLLM with fallbacks for missing models",
        "formula": "cost = (uncached_input / 1M * input_rate) + (cached / 1M * cached_rate) + (output / 1M * output_rate)",
        "pricing_resolution": get_pricing_summary(),
        "per_model_rates": get_pricing_table(),
    },
    "assumptions": {
        "date_range_analyzed": f"{date_range['start']} to {date_range['end']}",
        "days_of_data": days_span,
        "total_sessions": total_sessions,
        "org_filter": "pratilipi%",
        "session_time_definition": "Sum of inter-message gaps < 30 minutes (excludes idle periods)",
        "cost_model": "Per-model pricing (see cost_methodology)",
        "tool_normalization": "Shell tools (Bash/shell_command/exec_command) unified; Edit tools (Edit/Write/MultiEdit) unified; Explore tools (Read/Grep/Glob/etc.) unified",
    },
}
write_json("overview.json", overview)

# ---------------------------------------------------------------------------
# 2. SESSION METRICS
# ---------------------------------------------------------------------------
print("Computing session metrics...")

session_ids_sql = """
    SELECT id FROM sessions
    WHERE source NOT IN ('test', 'qc_trace_install')
    AND user_email IS NOT NULL
    AND org ILIKE 'pratilipi%%'
"""

msg_counts = query_df(conn, f"""
    SELECT m.session_id, m.msg_type, COUNT(*) as cnt
    FROM messages m
    WHERE m.session_id IN ({session_ids_sql})
    GROUP BY m.session_id, m.msg_type
""")
msg_pivot = msg_counts.pivot_table(index="session_id", columns="msg_type", values="cnt", fill_value=0).reset_index()

tool_calls_df = query_df(conn, f"""
    SELECT tc.tool_name, m.session_id, tc.tool_input
    FROM tool_calls tc JOIN messages m ON m.id = tc.message_id
    WHERE m.session_id IN ({session_ids_sql})
""")
tool_calls_df["category"] = tool_calls_df["tool_name"].map(categorize_tool)

cat_counts = tool_calls_df.groupby(["session_id", "category"]).size().unstack(fill_value=0).reset_index()

sess_tokens = query_df(conn, f"""
    SELECT m.session_id,
           SUM(t.input_tokens) as input_tokens,
           SUM(t.output_tokens) as output_tokens,
           SUM(t.cached_tokens) as cached_tokens,
           SUM(t.thinking_tokens) as thinking_tokens
    FROM token_usage t JOIN messages m ON m.id = t.message_id
    WHERE m.session_id IN ({session_ids_sql})
    GROUP BY m.session_id
""")

sm = sessions[["id", "source", "user_email", "repo_name", "first_seen", "last_updated", "duration_min"]].copy()
sm = sm.rename(columns={"id": "session_id"})
if "user" in msg_pivot.columns:
    sm = sm.merge(msg_pivot[["session_id"] + [c for c in msg_pivot.columns if c != "session_id"]], on="session_id", how="left")
sm = sm.merge(cat_counts, on="session_id", how="left")
sm = sm.merge(sess_tokens, on="session_id", how="left")
# fillna(0) for numeric cols only — leave user_email as-is to avoid "0" emails
numeric_cols = sm.select_dtypes(include=[np.number]).columns
sm[numeric_cols] = sm[numeric_cols].fillna(0)
# Ensure user_email nulls stay null (not 0)
sm["user_email"] = sm["user_email"].replace({0: None, "0": None})

# Discovery overhead
if "explore" in sm.columns and "edit" in sm.columns:
    sm["discovery_overhead"] = sm.apply(
        lambda r: round(r["explore"] / r["edit"], 2) if r["edit"] > 0 else None, axis=1
    )

# Prompts before first edit per session
print("  computing prompts before first edit...")
first_edit = query_df(conn, f"""
    SELECT m.session_id, MIN(m.timestamp) as first_edit_ts
    FROM tool_calls tc JOIN messages m ON m.id = tc.message_id
    WHERE tc.tool_name IN ('Edit', 'Write', 'MultiEdit', 'MultiEditTool')
    AND m.session_id IN ({session_ids_sql})
    GROUP BY m.session_id
""")
user_msgs = query_df(conn, f"""
    SELECT session_id, timestamp FROM messages
    WHERE msg_type = 'user' AND session_id IN ({session_ids_sql})
    ORDER BY timestamp
""")
prompts_before_edit = {}
for _, row in first_edit.iterrows():
    sid = row["session_id"]
    fe_ts = row["first_edit_ts"]
    before = user_msgs[(user_msgs["session_id"] == sid) & (user_msgs["timestamp"] < fe_ts)]
    prompts_before_edit[sid] = len(before)
sm["prompts_before_first_edit"] = sm["session_id"].map(prompts_before_edit)

# Tool success rate per session
sess_success = query_df(conn, f"""
    SELECT m.session_id,
           COUNT(*) as total_results,
           COUNT(*) FILTER (WHERE tr.status = 'success') as successes
    FROM tool_results tr JOIN messages m ON m.id = tr.message_id
    WHERE m.session_id IN ({session_ids_sql})
    GROUP BY m.session_id
""")
sess_success["success_rate"] = (sess_success["successes"] / sess_success["total_results"] * 100).round(1)
sm = sm.merge(sess_success[["session_id", "success_rate", "total_results"]], on="session_id", how="left")

# Compute user_msg_count for later
user_msg_col = "user" if "user" in sm.columns else None

session_metrics = {
    "sessions": json.loads(sm.to_json(orient="records", date_format="iso")),
    "distributions": {
        "duration_min": {
            "median": round(sessions["duration_min"].median(), 1),
            "median_meaningful": median_meaningful_duration,
            "mean": round(sessions["duration_min"].mean(), 1),
            "p25": round(sessions["duration_min"].quantile(0.25), 1),
            "p75": round(sessions["duration_min"].quantile(0.75), 1),
            "p95": round(sessions["duration_min"].quantile(0.95), 1),
            "histogram": np.histogram(sm["duration_min"].clip(0, 120), bins=20)[0].tolist(),
            "bin_edges": np.histogram(sm["duration_min"].clip(0, 120), bins=20)[1].tolist(),
        },
        "user_messages": {
            "median": round(sm.get("user", pd.Series([0])).median(), 1),
            "mean": round(sm.get("user", pd.Series([0])).mean(), 1),
            "histogram": np.histogram(sm.get("user", pd.Series([0])).clip(0, 50), bins=20)[0].tolist(),
            "bin_edges": np.histogram(sm.get("user", pd.Series([0])).clip(0, 50), bins=20)[1].tolist(),
        },
    },
    "summary_stats": {
        "sessions_without_edits": int((sm.get("edit", pd.Series([0])) == 0).sum()),
        "zero_edit_pct": round((sm.get("edit", pd.Series([0])) == 0).mean() * 100, 1),
        "single_interaction_pct": single_pct,
        "median_discovery_overhead": round(sm["discovery_overhead"].median(), 2) if "discovery_overhead" in sm.columns else None,
        "median_prompts_before_edit": sm["prompts_before_first_edit"].median() if "prompts_before_first_edit" in sm.columns else None,
        "median_success_rate": round(sm["success_rate"].median(), 1) if "success_rate" in sm.columns else None,
    }
}
write_json("session_metrics.json", session_metrics)

# ---------------------------------------------------------------------------
# 3. DEVELOPER PROFILES
# ---------------------------------------------------------------------------
print("Computing developer profiles...")

# Pre-compute per-developer costs using per-model pricing
dev_model_tokens = query_df(conn, f"""
    SELECT s.user_email, m.model,
           SUM(t.input_tokens) as input_tokens,
           SUM(t.output_tokens) as output_tokens,
           SUM(t.cached_tokens) as cached_tokens
    FROM token_usage t
    JOIN messages m ON m.id = t.message_id
    JOIN sessions s ON s.id = m.session_id
    WHERE s.source NOT IN ('test', 'qc_trace_install')
    AND s.user_email IS NOT NULL
    AND s.org ILIKE 'pratilipi%%'
    GROUP BY s.user_email, m.model
""")
dev_cost_lookup = {}
for _, row in dev_model_tokens.iterrows():
    email_key = row["user_email"]
    cost = calc_model_cost(row["model"], row["input_tokens"], row["output_tokens"], row["cached_tokens"])
    dev_cost_lookup[email_key] = dev_cost_lookup.get(email_key, 0) + cost

dev_profiles = []
for email in sessions["user_email"].unique():
    ds = sm[sm["user_email"] == email]
    if len(ds) == 0:
        continue

    tool_call_col = "tool_call" if "tool_call" in ds.columns else None

    total_user_msgs = int(ds[user_msg_col].sum()) if user_msg_col else 0
    total_tool_calls_dev = int(ds[tool_call_col].sum()) if tool_call_col else 0
    total_explore = int(ds["explore"].sum()) if "explore" in ds.columns else 0
    total_edit = int(ds["edit"].sum()) if "edit" in ds.columns else 0
    total_shell = int(ds["shell"].sum()) if "shell" in ds.columns else 0

    # Persona classification
    median_user_msgs = ds[user_msg_col].median() if user_msg_col else 0
    median_tool_calls = ds[tool_call_col].median() if tool_call_col else 0
    explore_edit_ratio = round(total_explore / total_edit, 2) if total_edit > 0 else None

    persona = "Balanced"
    if median_user_msgs <= 3 and median_tool_calls > 10:
        persona = "Delegator"
    elif median_user_msgs > 8:
        persona = "Collaborator"
    elif total_edit > 0 and total_explore / total_edit > 2.5:
        persona = "Explorer"
    elif total_edit > total_explore:
        persona = "Builder"

    sources_used = ds["source"].value_counts().to_dict()
    repos_worked = sessions[sessions["user_email"] == email]["repo_name"].dropna().unique().tolist()

    # Meaningful duration stats for this developer
    dev_meaningful = ds[ds["duration_min"] >= 1]
    median_dur = round(dev_meaningful["duration_min"].median(), 1) if len(dev_meaningful) > 0 else 0

    profile = {
        "email": email,
        "total_sessions": len(ds),
        "total_user_messages": total_user_msgs,
        "total_tool_calls": total_tool_calls_dev,
        "total_explore_calls": total_explore,
        "total_edit_calls": total_edit,
        "total_shell_calls": total_shell,
        "explore_to_edit_ratio": explore_edit_ratio,
        "explore_to_edit_context": f"For every file edited, the AI reads {explore_edit_ratio}x files first" if explore_edit_ratio else "No edits recorded",
        "median_session_duration_min": median_dur,
        "single_interaction_sessions": int((ds["duration_min"] < 1).sum()),
        "total_hours": round(ds["duration_min"].sum() / 60, 1),
        "median_success_rate": round(ds["success_rate"].median(), 1) if "success_rate" in ds.columns else None,
        "median_prompts_before_edit": ds["prompts_before_first_edit"].median() if "prompts_before_first_edit" in ds.columns else None,
        "persona": persona,
        "sources": sources_used,
        "repos": repos_worked,
        "input_tokens": int(ds["input_tokens"].sum()),
        "output_tokens": int(ds["output_tokens"].sum()),
    }
    # Cost per developer — using per-model pricing
    dev_total_cost = round(dev_cost_lookup.get(email, 0), 2)
    profile["estimated_cost_usd"] = dev_total_cost
    profile["cost_per_session"] = round(dev_total_cost / max(len(ds), 1), 2)
    dev_profiles.append(profile)

# Sort by total_sessions descending
dev_profiles.sort(key=lambda x: x["total_sessions"], reverse=True)

write_json("developer_profiles.json", {"developers": dev_profiles})

# ---------------------------------------------------------------------------
# 4. TOOL ANALYSIS
# ---------------------------------------------------------------------------
print("Computing tool analysis...")

tool_freq = tool_calls_df.groupby("tool_name").agg(
    call_count=("tool_name", "size"),
    session_count=("session_id", "nunique")
).reset_index().sort_values("call_count", ascending=False)
tool_freq["category"] = tool_freq["tool_name"].map(categorize_tool)

# Failure rates
failure_df = query_df(conn, f"""
    SELECT tc.tool_name,
           COUNT(*) as total,
           COUNT(*) FILTER (WHERE tr.status = 'success') as successes,
           COUNT(*) FILTER (WHERE tr.status = 'failure') as failures
    FROM tool_calls tc
    JOIN messages m ON m.id = tc.message_id
    LEFT JOIN tool_results tr ON tr.call_id = tc.tool_id
        AND tr.message_id IN (SELECT id FROM messages WHERE session_id = m.session_id)
    WHERE m.session_id IN ({session_ids_sql})
    GROUP BY tc.tool_name
    ORDER BY failures DESC
""")
failure_df["failure_rate_pct"] = (failure_df["failures"] / failure_df["total"] * 100).round(1)

# Most read files
read_tools = tool_calls_df[tool_calls_df["tool_name"].isin(["Read", "ReadFile", "read_file", "View"])].copy()
read_tools["file_path"] = read_tools["tool_input"].apply(extract_path)
top_read_files = read_tools["file_path"].dropna().value_counts().head(30).to_dict()

# Most edited files
edit_tools = tool_calls_df[tool_calls_df["tool_name"].isin(["Edit", "Write", "MultiEdit", "MultiEditTool"])].copy()
edit_tools["file_path"] = edit_tools["tool_input"].apply(extract_path)
top_edited_files = edit_tools["file_path"].dropna().value_counts().head(30).to_dict()

# Shell commands
shell_tools = tool_calls_df[tool_calls_df["tool_name"].isin(["Bash", "shell_command", "exec_command"])].copy()
shell_tools["command"] = shell_tools["tool_input"].apply(extract_command)
shell_tools["cmd_category"] = shell_tools["command"].apply(classify_command)
top_commands = shell_tools["cmd_category"].value_counts().head(20).to_dict()

# Struggle signals
edit_per_file_session = edit_tools.groupby(["session_id", "file_path"]).size().reset_index(name="edit_count")
struggles = edit_per_file_session[edit_per_file_session["edit_count"] >= 5].sort_values("edit_count", ascending=False)
struggle_list = struggles.head(20).to_dict(orient="records")

# Wasted reads (for dedicated visualization)
read_counts = read_tools.groupby(["session_id", "file_path"]).size().reset_index(name="read_count")
excessive_reads = read_counts[read_counts["read_count"] >= 5].sort_values("read_count", ascending=False)
top_wasted_reads = excessive_reads.head(15).to_dict(orient="records")

tool_analysis = {
    "frequency": json.loads(tool_freq.to_json(orient="records")),
    "failure_rates": json.loads(failure_df.to_json(orient="records")),
    "top_read_files": top_read_files,
    "top_edited_files": top_edited_files,
    "top_shell_commands": top_commands,
    "struggle_signals": struggle_list,
    "wasted_reads": {
        "top_offenders": top_wasted_reads,
        "total_file_session_pairs": len(excessive_reads),
        "total_wasted_reads": int(excessive_reads["read_count"].sum() - len(excessive_reads)) if len(excessive_reads) > 0 else 0,
    },
    "category_totals": tool_calls_df["category"].value_counts().to_dict(),
}
write_json("tool_analysis.json", tool_analysis)

# ---------------------------------------------------------------------------
# 5. INSIGHTS
# ---------------------------------------------------------------------------
print("Computing insights...")

insights = []

# Insight 1: Exploration dominance
total_explore_calls = tool_calls_df[tool_calls_df["category"] == "explore"].shape[0]
total_edit_calls = tool_calls_df[tool_calls_df["category"] == "edit"].shape[0]
total_all_calls = len(tool_calls_df)
explore_pct = round(total_explore_calls / total_all_calls * 100, 1) if total_all_calls > 0 else 0
edit_pct = round(total_edit_calls / total_all_calls * 100, 1) if total_all_calls > 0 else 0
explore_edit_ratio = round(total_explore_calls / max(total_edit_calls, 1), 1)

insights.append({
    "id": "exploration_dominance",
    "title": "AI spends more time reading than writing",
    "severity": "high" if explore_pct > 40 else "medium",
    "metric": f"{explore_pct}% explore vs {edit_pct}% edit",
    "detail": f"Out of {total_all_calls:,} tool calls, {total_explore_calls:,} ({explore_pct}%) were exploration (Read, Grep, Glob) while only {total_edit_calls:,} ({edit_pct}%) were edits. The AI reads {explore_edit_ratio}x more files than it edits — significant discovery overhead.",
    "recommendation": "Pre-loading frequently accessed files into context could eliminate up to 50% of exploration calls.",
    "benchmark": "GitHub Copilot studies show AI writes 29% of code but org productivity only improves 3.6% (DORA 2025). High exploration overhead is consistent with this finding — the AI spends most effort understanding, not producing.",
    "data": {"explore_calls": total_explore_calls, "edit_calls": total_edit_calls, "ratio": explore_edit_ratio}
})

# Insight 2: Zero-edit sessions
no_edit_sessions = sm[sm.get("edit", pd.Series([0])) == 0] if "edit" in sm.columns else pd.DataFrame()
no_edit_pct = round(len(no_edit_sessions) / total_sessions * 100, 1) if total_sessions > 0 else 0
insights.append({
    "id": "zero_edit_sessions",
    "title": "Most sessions produce no code changes",
    "severity": "high" if no_edit_pct > 30 else "medium",
    "metric": f"{len(no_edit_sessions)} sessions ({no_edit_pct}%) had zero edits",
    "detail": f"{len(no_edit_sessions)} out of {total_sessions} sessions never wrote or edited a single file. These sessions consumed tokens but produced no tangible code output. Many may be exploration/understanding sessions, but the volume is striking.",
    "recommendation": "Tag sessions by intent (explore vs build) to understand whether zero-edit sessions deliver value or represent waste.",
    "benchmark": "Cursor's internal study found a 281% initial velocity boost that collapses to baseline by month 3 — suggesting many AI sessions don't translate to lasting productivity.",
    "data": {"count": len(no_edit_sessions), "pct": no_edit_pct}
})

# Insight 3: Tool failure hotspots
high_fail = failure_df[failure_df["failure_rate_pct"] > 10].sort_values("failure_rate_pct", ascending=False)
if len(high_fail) > 0:
    worst_tool = high_fail.iloc[0]
    insights.append({
        "id": "tool_failure_hotspot",
        "title": f"High failure rate on {worst_tool['tool_name']}",
        "severity": "high" if worst_tool["failure_rate_pct"] > 25 else "medium",
        "metric": f"{worst_tool['tool_name']}: {worst_tool['failure_rate_pct']}% failure rate ({int(worst_tool['failures'])} failures)",
        "detail": f"The tool '{worst_tool['tool_name']}' failed {int(worst_tool['failures'])} times out of {int(worst_tool['total'])} calls. Each failure wastes tokens on error handling and retries.",
        "recommendation": "Investigate common failure patterns. Providing better context (file paths, permissions) could reduce failures.",
        "data": json.loads(high_fail.head(5).to_json(orient="records"))
    })

# Insight 5: Marathon sessions
marathon = sm[sm["duration_min"] > 60]
if len(marathon) > 0:
    insights.append({
        "id": "marathon_sessions",
        "title": f"{len(marathon)} marathon sessions (>1 hour)",
        "severity": "medium",
        "metric": f"Longest session: {round(sm['duration_min'].max(), 0)} min",
        "detail": f"{len(marathon)} sessions lasted over 1 hour. The longest was {round(sm['duration_min'].max(), 0)} minutes ({round(sm['duration_min'].max()/60, 1)} hours). Long sessions may indicate the AI struggling or the developer iterating extensively.",
        "recommendation": "Break long tasks into smaller, focused sessions. Each new session gets fresh context allocation.",
        "data": {"count": len(marathon), "max_duration_min": round(sm["duration_min"].max(), 0)}
    })

# Insight 6: Token caching — FIXED
total_input = token_totals.get("total_input") or 0
total_cached = token_totals.get("total_cached") or 0
# cached_tokens in the API is a SEPARATE counter (tokens served from cache), not a subset of input_tokens.
# It can exceed input_tokens because it accumulates across all requests in a session.
insights.append({
    "id": "cache_efficiency",
    "title": "Prompt caching is heavily used",
    "severity": "info",
    "metric": f"{total_cached:,} cached tokens vs {total_input:,} input tokens",
    "detail": f"The API served {total_cached:,} tokens from prompt cache against {total_input:,} billed input tokens. Cached tokens are a separate counter — they represent tokens the API didn't need to reprocess because they matched a cached prefix. This means the LLM provider is efficiently reusing context across turns, which reduces both latency and cost.",
    "recommendation": "Keep leveraging structured, consistent system prompts and file ordering to maximize cache hits.",
    "data": {"total_input": total_input, "total_cached": total_cached}
})

# Insight 7: CLI comparison — FIXED
claude_sessions = sm[sm["source"] == "claude_code"]
other_cli = sm[sm["source"] != "claude_code"]
claude_pct = round(len(claude_sessions) / total_sessions * 100, 1)
other_count = len(other_cli)

cli_stats = []
for src in sessions["source"].unique():
    src_sessions = sm[sm["source"] == src]
    if len(src_sessions) == 0:
        continue
    src_edit = src_sessions.get("edit", pd.Series([0])).sum()
    src_explore = src_sessions.get("explore", pd.Series([0])).sum()
    cli_stats.append({
        "source": src,
        "sessions": len(src_sessions),
        "median_duration_min": round(src_sessions["duration_min"].median(), 1),
        "total_edits": int(src_edit),
        "explore_edit_ratio": round(src_explore / src_edit, 2) if src_edit > 0 else None,
        "commit_rate_pct": round(src_sessions["has_commit"].mean() * 100, 1) if "has_commit" in src_sessions.columns else None,
        "median_success_rate": round(src_sessions["success_rate"].median(), 1) if "success_rate" in src_sessions.columns else None,
    })

insights.append({
    "id": "cli_comparison",
    "title": f"Claude Code dominates ({claude_pct}% of sessions)",
    "severity": "info",
    "metric": f"{len(claude_sessions)} Claude Code vs {other_count} other CLI sessions",
    "detail": f"Claude Code accounts for {claude_pct}% of all sessions and nearly all code edits. Codex CLI ({len(sm[sm['source']=='codex_cli'])} sessions), Gemini CLI ({len(sm[sm['source']=='gemini_cli'])} sessions), and Cursor ({len(sm[sm['source']=='cursor'])} sessions) have limited usage data — a meaningful head-to-head comparison isn't possible yet with this sample size.",
    "recommendation": "Collect more data from non-Claude tools before drawing conclusions about relative efficiency.",
    "data": cli_stats
})

# Insight 8: Repeated reads
if len(excessive_reads) > 0:
    total_wasted = int(excessive_reads["read_count"].sum() - len(excessive_reads))
    worst_read = excessive_reads.iloc[0]
    insights.append({
        "id": "repeated_reads",
        "title": f"AI re-reads the same files repeatedly",
        "severity": "high" if len(excessive_reads) > 10 else "medium",
        "metric": f"{len(excessive_reads)} file-session pairs with 5+ reads ({total_wasted} wasted reads)",
        "detail": f"The AI re-reads the same file multiple times within a session because it doesn't retain file contents. The worst case: one file was read {int(worst_read['read_count'])} times in a single session. This is pure waste — {total_wasted} reads that could be eliminated with persistent file caching.",
        "recommendation": "Pre-loading these files or using a persistent file cache within sessions would eliminate redundant reads.",
        "data": top_wasted_reads[:10]
    })

# NEW Insight 9: Single-interaction sessions
insights.append({
    "id": "single_interaction",
    "title": f"{single_pct}% of sessions are single-shot interactions",
    "severity": "high" if single_pct > 50 else "medium",
    "metric": f"{single_interaction} out of {total_sessions} sessions last under 1 minute",
    "detail": f"{single_interaction} sessions ({single_pct}%) last less than 1 minute — likely a single prompt and response. This drives the median session duration to 0. Among the {total_sessions - single_interaction} sessions that last 1+ minutes, the median duration is {median_meaningful_duration} minutes.",
    "recommendation": "Investigate whether single-shot sessions are quick lookups (valuable) or abandoned attempts (waste). Consider whether a faster UI or pre-loaded context could convert these into more productive sessions.",
    "data": {"single_interaction_count": int(single_interaction), "pct": single_pct, "median_meaningful_min": median_meaningful_duration}
})

write_json("insights.json", {"insights": insights, "generated_at": pd.Timestamp.now().isoformat()})

# ---------------------------------------------------------------------------
# 6. CHART DATA
# ---------------------------------------------------------------------------
print("Computing chart data...")

sessions_by_source = sessions["source"].value_counts().to_dict()

sessions["hour"] = pd.to_datetime(sessions["first_seen"]).dt.hour
sessions_by_hour = sessions["hour"].value_counts().sort_index().to_dict()

tool_category_dist = tool_calls_df["category"].value_counts().to_dict()

dur_hist, dur_edges = np.histogram(sm["duration_min"].clip(0, 120), bins=24)

scatter_data = []
for _, row in sm.iterrows():
    scatter_data.append({
        "session_id": row["session_id"],
        "duration_min": round(row["duration_min"], 1),
        "input_tokens": int(row.get("input_tokens", 0)),
        "output_tokens": int(row.get("output_tokens", 0)),
        "source": row["source"],
    })

# Session duration breakdown for pie chart
duration_breakdown = {
    "Single interaction (<1 min)": int(single_interaction),
    "Short (1–5 min)": int(short_sessions),
    "Medium (5–60 min)": int(medium_sessions),
    "Long (>60 min)": int(long_sessions),
}

# Tool calls over time
tool_time = query_df(conn, f"""
    SELECT date_trunc('hour', m.timestamp) as hour, COUNT(*) as calls
    FROM tool_calls tc JOIN messages m ON m.id = tc.message_id
    WHERE m.session_id IN ({session_ids_sql})
    GROUP BY hour ORDER BY hour
""")

# Developer comparison radar data
dev_radar = []
for p in dev_profiles:
    dev_radar.append({
        "email": p["email"],
        "sessions": p["total_sessions"],
        "explore_edit_ratio": p["explore_to_edit_ratio"] or 0,
        "median_duration": p["median_session_duration_min"],
        "persona": p["persona"],
    })

chart_data = {
    "sessions_by_source": sessions_by_source,
    "sessions_by_hour": {str(k): v for k, v in sessions_by_hour.items()},
    "tool_category_distribution": tool_category_dist,
    "duration_histogram": {"counts": dur_hist.tolist(), "edges": dur_edges.tolist()},
    "duration_breakdown": duration_breakdown,
    "session_scatter": scatter_data,
    "tool_calls_over_time": json.loads(tool_time.to_json(orient="records", date_format="iso")),
    "developer_comparison": dev_radar,
}
write_json("chart_data.json", chart_data)

# Timeline
timeline = []
for _, row in sessions.iterrows():
    timeline.append({
        "session_id": row["id"],
        "user": row["user_email"],
        "source": row["source"],
        "repo": row.get("repo_name"),
        "start": str(row["first_seen"]),
        "end": str(row["last_updated"]),
        "duration_min": round((pd.to_datetime(row["last_updated"]) - pd.to_datetime(row["first_seen"])).total_seconds() / 60, 1),
    })
write_json("sessions_timeline.json", {"sessions": timeline})

# ---------------------------------------------------------------------------
# 8. COST BY DEVELOPER
# ---------------------------------------------------------------------------
print("Computing cost by developer...")

cost_data = []
for p in dev_profiles:
    cost_data.append({
        "email": p["email"],
        "total_cost_usd": p["estimated_cost_usd"],
        "cost_per_session": p["cost_per_session"],
        "total_sessions": p["total_sessions"],
        "total_hours": p["total_hours"],
        "cost_per_hour": round(p["estimated_cost_usd"] / max(p["total_hours"], 0.1), 2),
        "persona": p["persona"],
    })
cost_data.sort(key=lambda x: x["total_cost_usd"], reverse=True)

# Savings estimates
total_wasted_reads_count = int(excessive_reads["read_count"].sum() - len(excessive_reads)) if len(excessive_reads) > 0 else 0
# Rough: each wasted read ~= 2K input tokens (reading a file) = $0.006
est_wasted_read_cost = round(total_wasted_reads_count * 0.006, 2)

cost_by_developer = {
    "developers": cost_data,
    "total_cost": round(est_total_cost, 2),
    "annualized_cost": annualized_cost,
    "cost_per_dev_per_month": cost_per_dev_per_month,
    "industry_benchmark": "$50-150/dev/month for GitHub Copilot",
    "pricing_rates": {p["model"]: {"input": p["input"], "output": p["output"], "cached": p["cached"]} for p in get_pricing_table()},
    "savings_opportunities": {
        "wasted_reads": {
            "count": total_wasted_reads_count,
            "estimated_cost_usd": est_wasted_read_cost,
            "recommendation": f"Pre-loading top 10 most-read files could eliminate {total_wasted_reads_count} redundant reads, saving ~${est_wasted_read_cost}/period."
        },
        "zero_edit_sessions": {
            "count": len(no_edit_sessions),
            "pct": no_edit_pct,
            "recommendation": f"{no_edit_pct}% of sessions produce no edits. If even half are waste, that's ~${round(est_total_cost * no_edit_pct / 200, 0)} in avoidable spend."
        }
    }
}
write_json("cost_by_developer.json", cost_by_developer)

# ---------------------------------------------------------------------------
# 9. INTENT CLASSIFICATION (from user messages)
# ---------------------------------------------------------------------------
print("Computing intent classification...")

def strip_system_prompts(text):
    if not text: return ""
    text = re.sub(r"^#\s*AGENTS\.md\s+instructions\s+for\s+.+?\n\n<INSTRUCTIONS?>[\s\S]*?</INSTRUCTIONS?>\s*",
                   "", text, flags=re.IGNORECASE)
    text = re.sub(r"<INSTRUCTIONS?>[\s\S]*?</INSTRUCTIONS?>", "", text, flags=re.IGNORECASE)
    text = text.encode("ascii", errors="ignore").decode("ascii")
    return text.strip()

INTENT_PATTERNS = {
    "fix_bug": r"\b(fix|bug|error|crash|broken|issue|fail|debug|wrong|problem)\b",
    "build_feature": r"\b(add|create|implement|build|feature|new|generate)\b",
    "refactor": r"\b(refactor|clean|simplify|restructure|reorganize|rename|move)\b",
    "understand": r"\b(explain|understand|what|how|why|show|read|look|check|review)\b",
    "test": r"\b(test|spec|assert|coverage|jest|pytest|unittest)\b",
    "deploy": r"\b(deploy|release|publish|push|merge|pr|pull request)\b",
    "config": r"\b(config|setup|install|env|docker|ci|cd|pipeline|yaml)\b",
}

user_prompts = query_df(conn, f"""
    SELECT m.session_id, m.content, s.user_email
    FROM messages m
    JOIN sessions s ON s.id = m.session_id
    WHERE m.msg_type = 'user'
    AND m.content IS NOT NULL AND m.content != ''
    AND m.session_id IN ({session_ids_sql})
    LIMIT 10000
""")

user_prompts["clean"] = user_prompts["content"].apply(strip_system_prompts)
user_prompts = user_prompts[user_prompts["clean"].str.len() > 0]

intent_counts = Counter()
dev_intents = {}
for _, row in user_prompts.iterrows():
    text = row["clean"].lower()
    matched = False
    for intent, pattern in INTENT_PATTERNS.items():
        if re.search(pattern, text):
            intent_counts[intent] += 1
            email = row["user_email"]
            if email not in dev_intents:
                dev_intents[email] = Counter()
            dev_intents[email][intent] += 1
            matched = True
    if not matched:
        intent_counts["other"] += 1

# Add intent data to chart_data
chart_data["intent_distribution"] = dict(intent_counts.most_common())
chart_data["intent_by_developer"] = {email: dict(c.most_common()) for email, c in dev_intents.items()}

# Rewrite chart_data with intent data included
write_json("chart_data.json", chart_data)

# ---------------------------------------------------------------------------
# 10. DAILY ACTIVITY HEATMAP
# ---------------------------------------------------------------------------
print("Computing daily activity heatmap...")

sessions["day_of_week"] = pd.to_datetime(sessions["first_seen"]).dt.day_name()
sessions["date"] = pd.to_datetime(sessions["first_seen"]).dt.date.astype(str)

daily_activity = sessions.groupby(["date", "user_email"]).agg(
    session_count=("id", "count"),
).reset_index()

chart_data["daily_activity"] = json.loads(daily_activity.to_json(orient="records"))

# ---------------------------------------------------------------------------
# 11. HOURLY ACTIVITY INTERPRETATION
# ---------------------------------------------------------------------------
print("Computing hourly activity interpretation...")

# Find peak hours
hour_counts = sessions["hour"].value_counts().sort_index()
peak_hour = hour_counts.idxmax()
peak_count = hour_counts.max()
# Find the 4-hour window with most activity
best_window_start = 0
best_window_count = 0
for h in range(24):
    window = sum(hour_counts.get(hh % 24, 0) for hh in range(h, h + 4))
    if window > best_window_count:
        best_window_count = window
        best_window_start = h
best_window_end = (best_window_start + 4) % 24
# Convert UTC to IST for context
ist_start = (best_window_start + 5) % 24
ist_start_half = ist_start + 0.5  # IST is UTC+5:30
ist_end = (best_window_end + 5) % 24
ist_end_half = ist_end + 0.5

chart_data["hourly_interpretation"] = {
    "peak_hour_utc": int(peak_hour),
    "peak_sessions": int(peak_count),
    "best_window_utc": f"{best_window_start}:00-{best_window_end}:00 UTC",
    "best_window_ist": f"{int(ist_start_half)}:{30 if ist_start_half % 1 else '00'}-{int(ist_end_half)}:{30 if ist_end_half % 1 else '00'} IST",
    "best_window_sessions": int(best_window_count),
    "headline": f"Peak AI coding: {best_window_start}:00-{best_window_end}:00 UTC ({int(ist_start_half)}:30-{int(ist_end_half)}:30 IST) with {best_window_count} sessions",
    "pct_of_total": round(best_window_count / total_sessions * 100, 1),
}

# ---------------------------------------------------------------------------
# 12. PER-DEVELOPER INTENT BREAKDOWN (add to developer_profiles)
# ---------------------------------------------------------------------------
print("Adding per-developer intent to profiles...")

for p in dev_profiles:
    email = p["email"]
    p["intent_breakdown"] = dev_intents.get(email, {})
    # Determine primary intent
    if dev_intents.get(email):
        primary = max(dev_intents[email], key=dev_intents[email].get)
        p["primary_intent"] = primary
        p["primary_intent_pct"] = round(dev_intents[email][primary] / sum(dev_intents[email].values()) * 100, 1)
    else:
        p["primary_intent"] = "unknown"
        p["primary_intent_pct"] = 0

write_json("developer_profiles.json", {"developers": dev_profiles})

# ---------------------------------------------------------------------------
# 13. SESSION EFFICIENCY SCORING
# ---------------------------------------------------------------------------
print("Computing session efficiency scores...")

# Score each session 0-100 based on:
# - Had edits (30 pts)
# - Had commits (30 pts)
# - Low discovery overhead (20 pts)
# - Reasonable duration 1-60 min (20 pts)
efficiency_scores = []
for _, row in sm.iterrows():
    score = 0
    has_edits = row.get("edit", 0) > 0
    has_commit = bool(row.get("has_commit", False))
    dur = row.get("duration_min", 0)
    overhead = row.get("discovery_overhead")

    if has_edits:
        score += 30
    if has_commit:
        score += 30
    if overhead is not None and overhead < 2:
        score += 20
    elif overhead is not None and overhead < 4:
        score += 10
    if 1 <= dur <= 60:
        score += 20
    elif dur > 60:
        score += 5

    efficiency_scores.append(score)

sm["efficiency_score"] = efficiency_scores

# Session efficiency by developer
dev_efficiency = sm.groupby("user_email")["efficiency_score"].agg(["mean", "median"]).reset_index()
dev_efficiency.columns = ["email", "avg_efficiency", "median_efficiency"]

for p in dev_profiles:
    row = dev_efficiency[dev_efficiency["email"] == p["email"]]
    if len(row) > 0:
        p["avg_efficiency_score"] = round(row.iloc[0]["avg_efficiency"], 1)
        p["median_efficiency_score"] = round(row.iloc[0]["median_efficiency"], 1)

write_json("developer_profiles.json", {"developers": dev_profiles})

# Efficiency distribution for chart
eff_hist, eff_edges = np.histogram(sm["efficiency_score"], bins=[0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100])
chart_data["efficiency_distribution"] = {
    "counts": eff_hist.tolist(),
    "edges": eff_edges.tolist(),
    "labels": ["0-10", "10-20", "20-30", "30-40", "40-50", "50-60", "60-70", "70-80", "80-90", "90-100"],
    "mean": round(sm["efficiency_score"].mean(), 1),
    "median": round(sm["efficiency_score"].median(), 1),
    "headline": f"Average session efficiency: {round(sm['efficiency_score'].mean(), 1)}/100 — only {(sm['efficiency_score'] >= 50).sum()} of {len(sm)} sessions score above 50"
}

# ---------------------------------------------------------------------------
# 14. SCATTER PLOT ANNOTATIONS
# ---------------------------------------------------------------------------
print("Computing scatter plot annotations...")

# Identify clusters for annotation
has_edits = sm[sm.get("edit", pd.Series([0])) > 0]
no_edits = sm[sm.get("edit", pd.Series([0])) == 0]
long_no_output = sm[(sm["duration_min"] > 30) & (sm.get("edit", pd.Series([0])) == 0)]

chart_data["scatter_annotations"] = [
    {
        "label": "Sessions with edits",
        "description": f"Sessions that produced code changes: median {round(has_edits['duration_min'].median(), 1)} min, median {int(has_edits['output_tokens'].median())} output tokens" if len(has_edits) > 0 else "No sessions with edits",
        "x": round(has_edits["duration_min"].median(), 1) if len(has_edits) > 0 else 0,
        "y": int(has_edits["output_tokens"].median()) if len(has_edits) > 0 else 0,
    },
    {
        "label": "Long sessions, no edits",
        "description": f"{len(long_no_output)} sessions lasted 30+ min but produced zero edits — likely the AI struggling or developer exploring",
        "count": len(long_no_output),
    },
]

# ---------------------------------------------------------------------------
# 15. WORKFLOW PATTERNS — what sequences of tools do devs use?
# ---------------------------------------------------------------------------
print("Computing workflow patterns...")

# Get tool call sequences per session (first 20 calls per session)
workflow_df = query_df(conn, f"""
    SELECT m.session_id, tc.tool_name, m.timestamp
    FROM tool_calls tc
    JOIN messages m ON m.id = tc.message_id
    WHERE m.session_id IN ({session_ids_sql})
    ORDER BY m.session_id, m.timestamp
""")

# Compute common 3-grams of tool categories
workflow_df["category"] = workflow_df["tool_name"].map(categorize_tool)
session_groups = workflow_df.groupby("session_id")["category"].apply(list)

trigram_counts = Counter()
for seq in session_groups:
    for i in range(len(seq) - 2):
        trigram = f"{seq[i]} → {seq[i+1]} → {seq[i+2]}"
        trigram_counts[trigram] += 1

chart_data["workflow_patterns"] = {
    "top_trigrams": [{"pattern": k, "count": v} for k, v in trigram_counts.most_common(15)],
    "headline": f"Most common workflow: {trigram_counts.most_common(1)[0][0]} ({trigram_counts.most_common(1)[0][1]} times)" if trigram_counts else "No workflow patterns found",
}

# ---------------------------------------------------------------------------
# 16. COST EFFICIENCY INSIGHT — the big one
# ---------------------------------------------------------------------------
print("Computing cost efficiency insight...")

# Find the most cost-efficient and least cost-efficient developers
sorted_by_cost = sorted(dev_profiles, key=lambda x: x["estimated_cost_usd"], reverse=True)
most_expensive = sorted_by_cost[0]
least_expensive = sorted_by_cost[-1]

insights.append({
    "id": "cost_inequality",
    "title": f"One developer consumes {round(most_expensive['estimated_cost_usd'] / est_total_cost * 100, 1)}% of total AI spend",
    "severity": "critical",
    "metric": f"${most_expensive['estimated_cost_usd']:,.0f} out of ${est_total_cost:,.0f} total",
    "detail": f"{most_expensive['email'].split('@')[0]} spent ${most_expensive['estimated_cost_usd']:,.0f} ({round(most_expensive['estimated_cost_usd'] / est_total_cost * 100, 1)}% of total) across {most_expensive['total_sessions']} sessions. Meanwhile, {least_expensive['email'].split('@')[0]} spent ${least_expensive['estimated_cost_usd']:,.0f} across {least_expensive['total_sessions']} sessions. That's a {round(most_expensive['cost_per_session'] / max(least_expensive['cost_per_session'], 0.01))}x difference in cost-per-session.",
    "recommendation": "Investigate whether the high-cost developer is working on harder problems or using AI inefficiently. Consider pairing sessions or sharing successful patterns from the efficient developer.",
    "data": {
        "most_expensive": {"email": most_expensive["email"], "cost": most_expensive["estimated_cost_usd"], "sessions": most_expensive["total_sessions"]},
        "least_expensive": {"email": least_expensive["email"], "cost": least_expensive["estimated_cost_usd"], "sessions": least_expensive["total_sessions"]},
    }
})

# ---------------------------------------------------------------------------
# 17. RESPONSE TIME ANALYSIS (AI turnaround)
# ---------------------------------------------------------------------------
print("Computing response time analysis...")

response_times = query_df(conn, f"""
    WITH msg_pairs AS (
        SELECT m.session_id, m.msg_type, m.timestamp,
               LAG(m.msg_type) OVER (PARTITION BY m.session_id ORDER BY m.timestamp) as prev_type,
               LAG(m.timestamp) OVER (PARTITION BY m.session_id ORDER BY m.timestamp) as prev_ts
        FROM messages m
        WHERE m.session_id IN ({session_ids_sql})
    )
    SELECT session_id,
           EXTRACT(EPOCH FROM (timestamp - prev_ts)) as response_seconds
    FROM msg_pairs
    WHERE msg_type = 'assistant' AND prev_type = 'user'
    AND EXTRACT(EPOCH FROM (timestamp - prev_ts)) > 0
    AND EXTRACT(EPOCH FROM (timestamp - prev_ts)) < 600
""")

if len(response_times) > 0:
    response_times["response_seconds"] = response_times["response_seconds"].astype(float)
    rt_hist, rt_edges = np.histogram(response_times["response_seconds"].clip(0, 60), bins=20)
    chart_data["response_time"] = {
        "median_seconds": round(response_times["response_seconds"].median(), 1),
        "mean_seconds": round(response_times["response_seconds"].mean(), 1),
        "p95_seconds": round(response_times["response_seconds"].quantile(0.95), 1),
        "histogram": {"counts": rt_hist.tolist(), "edges": rt_edges.tolist()},
        "headline": f"Median AI response time: {round(response_times['response_seconds'].median(), 1)}s — 95th percentile: {round(response_times['response_seconds'].quantile(0.95), 1)}s",
    }

# ---------------------------------------------------------------------------
# 18. SESSION ARC CLASSIFICATION — what shape does each session take?
# ---------------------------------------------------------------------------
print("Computing session arcs...")

# Classify each session's tool sequence into an arc pattern
arc_counts = Counter()
for sid, seq in session_groups.items():
    if len(seq) == 0:
        arc_counts["empty"] += 1
        continue
    has_edit = "edit" in seq
    has_shell = "shell" in seq
    has_explore = "explore" in seq

    if len(seq) <= 2 and not has_edit:
        arc_counts["quick_lookup"] += 1
    elif has_explore and not has_edit and not has_shell:
        arc_counts["explore_only"] += 1
    elif has_explore and has_edit and not has_shell:
        arc_counts["explore_then_edit"] += 1
    elif has_edit and has_shell:
        arc_counts["full_cycle"] += 1  # edit + run/test
    elif has_shell and not has_edit:
        arc_counts["shell_only"] += 1
    elif has_edit and not has_explore:
        arc_counts["direct_edit"] += 1
    else:
        arc_counts["mixed"] += 1

arc_labels = {
    "quick_lookup": "Quick lookup (1-2 tool calls, no edits)",
    "explore_only": "Explore only (read files, no changes)",
    "explore_then_edit": "Explore → Edit (read then modify)",
    "full_cycle": "Full cycle (edit + shell/test)",
    "shell_only": "Shell only (commands, no file edits)",
    "direct_edit": "Direct edit (modify without exploration)",
    "mixed": "Mixed pattern",
    "empty": "No tool calls",
}

chart_data["session_arcs"] = {
    "arcs": [{"arc": k, "label": arc_labels.get(k, k), "count": v} for k, v in arc_counts.most_common()],
    "headline": f"Most common session pattern: {arc_labels.get(arc_counts.most_common(1)[0][0], 'unknown')} ({arc_counts.most_common(1)[0][1]} sessions)" if arc_counts else "No sessions",
    "total_sessions": sum(arc_counts.values()),
}

# ---------------------------------------------------------------------------
# 19. TIME-OF-DAY PRODUCTIVITY — do certain hours produce more commits/edits?
# ---------------------------------------------------------------------------
print("Computing time-of-day productivity...")

hourly_productivity = query_df(conn, f"""
    SELECT
        EXTRACT(HOUR FROM s.first_seen) as hour,
        COUNT(DISTINCT s.id) as sessions,
        COUNT(DISTINCT CASE WHEN tc.tool_name IN ('Edit', 'Write', 'MultiEdit') THEN s.id END) as sessions_with_edits
    FROM sessions s
    LEFT JOIN messages m ON m.session_id = s.id
    LEFT JOIN tool_calls tc ON tc.message_id = m.id
    WHERE s.id IN ({session_ids_sql})
    GROUP BY 1
    ORDER BY 1
""")

if len(hourly_productivity) > 0:
    hourly_productivity["edit_rate"] = (hourly_productivity["sessions_with_edits"] / hourly_productivity["sessions"] * 100).round(1)
    chart_data["hourly_productivity"] = hourly_productivity.to_dict(orient="records")

# Rewrite chart_data with all new additions
write_json("chart_data.json", chart_data)

# Build red_flags array — pre-computed VP-ready callouts the frontend can render directly
red_flags = []
# Cost outlier red flag
cost_ineq = [i for i in insights if i["id"] == "cost_inequality"]
if cost_ineq:
    ci = cost_ineq[0]
    top_email = ci.get("data", {}).get("most_expensive", {}).get("email", "Unknown")
    top_cost = ci.get("data", {}).get("most_expensive", {}).get("cost", 0)
    top_pct = round(top_cost / max(overview.get("estimated_cost_usd", 1), 1) * 100, 1)
    red_flags.append({
        "id": "cost_outlier",
        "headline": f"{top_email.split('@')[0]} consumed {top_pct}% of total AI spend (${top_cost:,.0f})",
        "detail": f"One developer consumed {top_pct}% of total AI spend (${top_cost:,.0f} of ${overview.get('estimated_cost_usd', 0):,.0f}). This is the biggest cost concentration.",
        "severity": "critical",
        "metric_value": f"${top_cost:,.0f}",
        "metric_label": "AI spend",
    })

# Zero-edit sessions red flag
zero_edit = [i for i in insights if i["id"] == "zero_edit_sessions"]
if zero_edit:
    ze = zero_edit[0]
    red_flags.append({
        "id": "zero_edit_dominance",
        "headline": f"{ze.get('data', {}).get('pct', 0)}% of sessions produce no code changes",
        "detail": ze.get("detail", ""),
        "severity": "high",
        "metric_value": f"{ze.get('data', {}).get('pct', 0)}%",
        "metric_label": "sessions with zero edits",
    })

# Cost benchmark red flag
red_flags.append({
    "id": "cost_vs_benchmark",
    "headline": f"AI spend is ${overview.get('cost_per_dev_per_month', 0):,.0f}/dev/month vs $50-150 for Copilot",
    "detail": f"At ${overview.get('cost_per_dev_per_month', 0):,.0f}/dev/month, your AI coding spend is ~{round(overview.get('cost_per_dev_per_month', 0) / 100)}x the industry benchmark for GitHub Copilot. However, agent-mode sessions do far more than autocomplete — this is apples-to-oranges but the magnitude matters.",
    "severity": "high",
    "metric_value": f"~{round(overview.get('cost_per_dev_per_month', 0) / 100)}x",
    "metric_label": "vs Copilot benchmark",
})

# Rewrite insights with red_flags
write_json("insights.json", {"insights": insights, "red_flags": red_flags, "generated_at": pd.Timestamp.now().isoformat()})

# Add executive_summary to overview
overview["executive_summary"] = (
    f"{total_sessions} AI coding sessions across {users} developers over {span_months} months "
    f"({date_range['start']} to {date_range['end']}). "
    f"Estimated cost: ${est_total_cost:.2f} for the period. "
    f"{no_edit_pct}% of sessions produced zero code edits."
)
write_json("overview.json", overview)

# Also update cost_by_developer with efficiency data
for cd in cost_data:
    email = cd["email"]
    matching = [p for p in dev_profiles if p["email"] == email]
    if matching:
        cd["avg_efficiency_score"] = matching[0].get("avg_efficiency_score")
        cd["primary_intent"] = matching[0].get("primary_intent")
write_json("cost_by_developer.json", cost_by_developer)

conn.close()
print("\nDone! All JSON files written.")
