# Databricks notebook source
# MAGIC %md
# MAGIC # Multi-Peril Vine Copula — UK Home Insurance Demo
# MAGIC
# MAGIC This notebook demonstrates the full `insurance-copula` workflow on synthetic
# MAGIC UK home insurance loss data:
# MAGIC
# MAGIC 1. Generate synthetic correlated multi-peril losses
# MAGIC 2. Fit an exposure-weighted vine copula
# MAGIC 3. Inspect diagnostics and dependence structure
# MAGIC 4. Simulate loss scenarios
# MAGIC 5. Compute conditional expectations for pricing
# MAGIC 6. Estimate aggregate portfolio loss distribution (Solvency II SCR)
# MAGIC 7. Serialise and restore the fitted model

# COMMAND ----------

# MAGIC %md
# MAGIC ## Setup

# COMMAND ----------

# Install pyvinecopulib binary wheel (no source build on serverless)
import subprocess, sys
r = subprocess.run(
    [sys.executable, "-m", "pip", "install", "--only-binary=:all:", "--quiet",
     "pyvinecopulib>=0.7"],
    capture_output=True, text=True,
)
print("pyvinecopulib:", "OK" if r.returncode == 0 else f"FAILED: {r.stderr[-200:]}")

# COMMAND ----------

# %pip install insurance-copula[vine]
# For demo: install from local source instead
import sys
sys.path.insert(0, "/Workspace/insurance-copula/src")

# COMMAND ----------

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.stats import gamma, spearmanr

from insurance_copula import PerilVine, AggregateResult, PerilVineDiagnostics, HAS_PYVINECOPULIB

print(f"pyvinecopulib available: {HAS_PYVINECOPULIB}")

# COMMAND ----------

# MAGIC %md
# MAGIC ## 1. Synthetic UK Home Insurance Loss Data
# MAGIC
# MAGIC We generate 2,000 policy-years of losses across six perils. The correlation
# MAGIC structure is introduced via a shared latent factor (a proxy for "bad weather
# MAGIC year"). This is realistic: flood, storm, and escape of water all spike in
# MAGIC wet winters.

# COMMAND ----------

rng = np.random.default_rng(2024)
n_policies = 2_000

# Latent weather factor — drives correlated losses across perils
weather = rng.gamma(shape=2.0, scale=1.0, size=n_policies)

losses_df = pd.DataFrame({
    "flood":      rng.gamma(shape=0.5 + 0.4 * weather, scale=6_000.0, size=n_policies),
    "subsidence": rng.gamma(shape=0.4 + 0.05 * weather, scale=4_000.0, size=n_policies),
    "storm":      rng.gamma(shape=1.0 + 0.3 * weather, scale=900.0,   size=n_policies),
    "eow":        rng.gamma(shape=1.2 + 0.25 * weather, scale=700.0,  size=n_policies),
    "fire":       rng.gamma(shape=0.8 + 0.05 * weather, scale=2_500.0, size=n_policies),
    "theft":      rng.gamma(shape=1.8 + 0.05 * weather, scale=450.0,  size=n_policies),
})

# Exposure: fraction of year each policy was at risk (some short-period covers)
exposure = pd.Series(rng.uniform(0.25, 1.0, size=n_policies), name="exposure")

print(f"Dataset: {losses_df.shape[0]:,} observations, {losses_df.shape[1]} perils")
print("\nLoss summary (£):")
print(losses_df.describe().round(0).to_string())

# COMMAND ----------

# MAGIC %md
# MAGIC ## 2. Fit Exposure-Weighted Vine Copula

# COMMAND ----------

PERILS = ["flood", "subsidence", "storm", "eow", "fire", "theft"]

vine = PerilVine(
    perils=PERILS,
    family_set="parametric",   # Gaussian, t, Clayton, Gumbel, Frank, Joe, BB*
    selection_criterion="bic",  # BIC penalises complexity — important for small n
    tree_criterion="tau",       # Kendall's tau for spanning tree selection
    trunc_lvl=3,                # Only fit trees 1-3; higher trees are mostly noise
    vine_type="rvine",          # R-vine: most flexible structure
    n_threads=4,
)

print(repr(vine))

# COMMAND ----------

vine.fit(losses_df, exposure=exposure)

print(f"\nFitted: {vine.is_fitted}")
print(f"Observations used: {vine.n_obs:,}")
print(repr(vine))

# COMMAND ----------

# MAGIC %md
# MAGIC ## 3. Diagnostics

# COMMAND ----------

diag = vine.diagnostics()
print(repr(diag))

# COMMAND ----------

print("Kendall's tau matrix:")
print(diag.kendall_tau_matrix.round(3).to_string())

# COMMAND ----------

print(f"\nAIC: {diag.aic:.1f}")
print(f"BIC: {diag.bic:.1f}")
print(f"Free parameters: {diag.n_parameters}")

print("\nPair copula summary (tree 1 — unconditional pairs):")
t1 = diag.summary_table().query("tree == 1")
print(t1[["tree", "edge", "family", "params", "tau"]].to_string(index=False))

# COMMAND ----------

# Visualise Kendall's tau heatmap
fig, ax = plt.subplots(figsize=(7, 6))
tau_vals = diag.kendall_tau_matrix.values
im = ax.imshow(tau_vals, vmin=-1, vmax=1, cmap="RdBu_r")
ax.set_xticks(range(len(PERILS)))
ax.set_yticks(range(len(PERILS)))
ax.set_xticklabels(PERILS, rotation=45, ha="right")
ax.set_yticklabels(PERILS)
for i in range(len(PERILS)):
    for j in range(len(PERILS)):
        ax.text(j, i, f"{tau_vals[i, j]:.2f}", ha="center", va="center", fontsize=9)
fig.colorbar(im, ax=ax, label="Kendall's tau")
ax.set_title("Pairwise Kendall's tau — Multi-Peril Dependence")
plt.tight_layout()
plt.savefig("/tmp/tau_heatmap.png", dpi=120)
plt.show()
print("Saved heatmap")

# COMMAND ----------

# MAGIC %md
# MAGIC ## 4. Simulate Loss Scenarios

# COMMAND ----------

# Define marginal distributions (fitted independently — not shown here)
marginals = {
    "flood":      gamma(a=0.8, scale=6_000),
    "subsidence": gamma(a=0.6, scale=4_000),
    "storm":      gamma(a=1.2, scale=900),
    "eow":        gamma(a=1.5, scale=700),
    "fire":       gamma(a=1.0, scale=2_500),
    "theft":      gamma(a=2.0, scale=450),
}

# COMMAND ----------

# Copula-space simulation (uniform [0,1])
u_samples = vine.simulate(10_000, seed=42)
print(f"Copula samples shape: {u_samples.shape}")
print(f"Value range: [{u_samples.values.min():.4f}, {u_samples.values.max():.4f}]")

# COMMAND ----------

# Loss-space simulation (back-transformed via marginal PPFs)
loss_samples = vine.simulate(10_000, marginals=marginals, seed=42)
print(f"\nLoss samples shape: {loss_samples.shape}")
print("\nSample mean by peril (£):")
print(loss_samples.mean().round(0).to_string())
print("\nExpected loss from marginals (£):")
for peril, m in marginals.items():
    print(f"  {peril}: {m.mean():,.0f}")

# COMMAND ----------

# Compare empirical Spearman correlation (fitted data vs simulated)
print("Spearman rank correlation — fitted data:")
print(losses_df.rank().corr(method="spearman").round(3).to_string())

print("\nSpearman rank correlation — simulated (copula captures structure):")
print(loss_samples.rank().corr(method="spearman").round(3).to_string())

# COMMAND ----------

# MAGIC %md
# MAGIC ## 5. Conditional Expectation — Pricing Under Dependence
# MAGIC
# MAGIC A pricing team might ask: "If flood losses are bad (above 90th percentile),
# MAGIC what is our expected escape-of-water loss?" This drives decisions about
# MAGIC cross-peril pricing adjustments.

# COMMAND ----------

flood_90th = marginals["flood"].ppf(0.9)
flood_99th = marginals["flood"].ppf(0.99)

print(f"Flood loss thresholds:")
print(f"  Unconditional E[flood]: £{marginals['flood'].mean():,.0f}")
print(f"  90th percentile: £{flood_90th:,.0f}")
print(f"  99th percentile: £{flood_99th:,.0f}")

# E[eow | flood > 90th]
cond_eow_90 = vine.conditional_expectation(
    target_peril="eow",
    condition_peril="flood",
    threshold=flood_90th,
    n_samples=100_000,
    marginals=marginals,
)

# E[eow | flood > 99th]
cond_eow_99 = vine.conditional_expectation(
    target_peril="eow",
    condition_peril="flood",
    threshold=flood_99th,
    n_samples=100_000,
    marginals=marginals,
)

uncond_eow = marginals["eow"].mean()

print(f"\nUnconditional E[eow]:              £{uncond_eow:,.0f}")
print(f"E[eow | flood > 90th percentile]: £{cond_eow_90:,.0f}  "
      f"({cond_eow_90 / uncond_eow:.1%} of unconditional)")
print(f"E[eow | flood > 99th percentile]: £{cond_eow_99:,.0f}  "
      f"({cond_eow_99 / uncond_eow:.1%} of unconditional)")

# COMMAND ----------

# MAGIC %md
# MAGIC ## 6. Aggregate Portfolio Loss — Solvency II SCR
# MAGIC
# MAGIC The aggregate loss across a 10,000 policy portfolio.
# MAGIC PML at 99.5% is the Solvency II SCR capital requirement.

# COMMAND ----------

N_POLICIES = 10_000
N_SIM = 2_000

print(f"Simulating {N_SIM:,} scenarios for {N_POLICIES:,}-policy portfolio...")

result = vine.aggregate_loss(
    n_sim=N_SIM,
    marginals=marginals,
    n_policies=N_POLICIES,
    seed=42,
)

print(f"\nAggregate loss distribution (£):")
print(f"  Expected loss (E[L]):     £{result.expected_loss:,.0f}")
print(f"  Std deviation:            £{result.simulations.std():,.0f}")
print(f"  CV:                       {result.coefficient_of_variation:.3f}")
print(f"  PML 1-in-100 (99th %ile): £{result.pml_1_in_100:,.0f}")
print(f"  PML 1-in-200 (Solvency II SCR): £{result.pml_1_in_200:,.0f}")
print(f"\n  SCR as multiple of expected loss: {result.pml_1_in_200 / result.expected_loss:.2f}x")

# COMMAND ----------

# Compare with naive independence assumption
# Under independence, aggregate loss is sum of peril means × n_policies
independent_expected = sum(m.mean() for m in marginals.values()) * N_POLICIES
independent_std = np.sqrt(sum(m.std()**2 for m in marginals.values())) * np.sqrt(N_POLICIES)
# 99.5th percentile of normal approximation
from scipy.stats import norm as scipy_norm
independent_scr_approx = independent_expected + scipy_norm.ppf(0.995) * independent_std

print(f"\nComparison: vine vs independence assumption")
print(f"  Vine E[L]:     £{result.expected_loss:,.0f}")
print(f"  Indep E[L]:    £{independent_expected:,.0f}  (should be similar)")
print(f"  Vine SCR:      £{result.pml_1_in_200:,.0f}")
print(f"  Indep SCR:     £{independent_scr_approx:,.0f}  (likely lower — ignores joint tail)")

# COMMAND ----------

# Plot aggregate loss distribution
fig, ax = plt.subplots(figsize=(10, 5))
ax.hist(result.simulations, bins=80, density=True, alpha=0.7, color="steelblue",
        label="Aggregate loss")
ax.axvline(result.expected_loss, color="navy", linestyle="-", lw=2,
           label=f"E[L] = £{result.expected_loss:,.0f}")
ax.axvline(result.pml_1_in_100, color="orange", linestyle="--", lw=2,
           label=f"PML 1-in-100 = £{result.pml_1_in_100:,.0f}")
ax.axvline(result.pml_1_in_200, color="red", linestyle="--", lw=2,
           label=f"PML 1-in-200 (SCR) = £{result.pml_1_in_200:,.0f}")
ax.set_xlabel("Aggregate portfolio loss (£)")
ax.set_ylabel("Density")
ax.set_title(f"Aggregate Loss Distribution — {N_POLICIES:,} policies, {N_SIM:,} simulations")
ax.legend()
plt.tight_layout()
plt.savefig("/tmp/aggregate_loss.png", dpi=120)
plt.show()

# COMMAND ----------

# MAGIC %md
# MAGIC ## 7. Serialise and Restore

# COMMAND ----------

json_str = vine.to_json()
print(f"Serialised model: {len(json_str):,} characters")

import json
envelope = json.loads(json_str)
print(f"\nEnvelope keys: {list(envelope.keys())}")
print(f"Perils: {envelope['perils']}")
print(f"Family set: {envelope['family_set']}")
print(f"Version: {envelope['version']}")

# COMMAND ----------

restored = PerilVine.from_json(json_str)
print(f"Restored model: {repr(restored)}")

# Verify identical simulation output
s1 = vine.simulate(100, seed=7)
s2 = restored.simulate(100, seed=7)
print(f"\nSimulation outputs match: {np.allclose(s1.values, s2.values)}")

# COMMAND ----------

# MAGIC %md
# MAGIC ## Summary
# MAGIC
# MAGIC | Metric | Value |
# MAGIC |--------|-------|
# MAGIC | Perils | 6 |
# MAGIC | Observations | 2,000 |
# MAGIC | Vine structure | R-vine, truncated at tree 3 |
# MAGIC | BIC | See diagnostics |
# MAGIC | Free parameters | See diagnostics |
# MAGIC | E[flood \| storm > 90th] uplift | Computed above |
# MAGIC | PML 1-in-200 (Solvency II) | See aggregate result |
# MAGIC
# MAGIC The vine copula captures tail dependence between perils that the
# MAGIC independence assumption ignores. The SCR under vine copula is typically
# MAGIC materially higher than the independence approximation — the magnitude
# MAGIC depends on the fitted dependence structure.
