{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#default_exp losses.gw_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Giacomini-White Test\n",
    "\n",
    "Test on out-of-sample conditional predictive ability.\n",
    "\n",
    "References:\n",
    "- Giacomini, R., & White, H. (2006). Tests of conditional predictive ability. Econometrica, 74(6), 1545-1578.\n",
    "- https://onlinelibrary.wiley.com/doi/abs/10.1111/j.1468-0262.2006.00718.x\n",
    "- http://www.runmycode.org/companion/view/88"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Newey-West estimator\n",
    "\n",
    "Estimator of the Newey-West Heteroskedasticity and Autocorrelation Consistent Covariance Matrix (Newey-West HAC) to calculate robust error estimations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import os\n",
    "import numpy as np\n",
    "from scipy.stats.distributions import chi2\n",
    "\n",
    "from typing import List\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.colors import ListedColormap\n",
    "from mpl_toolkits.axes_grid1 import make_axes_locatable\n",
    "\n",
    "from matplotlib import rcParams\n",
    "plt.rcParams['font.family'] = 'serif'\n",
    "\n",
    "from matplotlib import cm\n",
    "from matplotlib.colors import ListedColormap, LinearSegmentedColormap\n",
    "\n",
    "FONTSIZE = 17"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "def Newey_West(Z, n_lags):\n",
    "    \"\"\" Newey-West HAC estimator\n",
    "    Parameters\n",
    "    ----------\n",
    "    Z: (n, k) ndarray\n",
    "    n_lags: int\n",
    "        number of lags to consider as available information.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    omega_hat: Newey-West HAC estimator of the covariance matrix\n",
    "    \"\"\"\n",
    "\n",
    "    assert n_lags > 0\n",
    "\n",
    "    n, k = Z.shape\n",
    "\n",
    "    Z = Z - np.ones((n, 1)) * np.mean(Z, axis=0)\n",
    "    gamma = -999 * np.ones((n_lags, k))\n",
    "    omega_hat = (1/n) * np.matmul(np.transpose(Z), Z)\n",
    "\n",
    "    Zlag = np.array([np.pad(Z, ((i,0), (0,0)), mode='constant', \n",
    "                            constant_values = 0)[:n] \n",
    "                     for i in range(1, n_lags + 1)])\n",
    "    gamma = (1/n) * (np.matmul(np.transpose(Z), Zlag) + \n",
    "            np.matmul(np.einsum('ijk -> ikj', Zlag), Z))\n",
    "    weights = 1 - np.array(range(1,n_lags + 1))/(n_lags + 1)\n",
    "    omega_hat = omega_hat + \\\n",
    "                np.sum(gamma * np.expand_dims(weights, \n",
    "                                              axis = (1,2)), \n",
    "                                              axis = 0)\n",
    "    return omega_hat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "def GW_CPA_test(loss1: np.ndarray, \n",
    "                loss2: np.ndarray, \n",
    "                tau: int,\n",
    "                alpha: float=0.05,\n",
    "                conditional: bool=False,\n",
    "                verbose: bool=True):\n",
    "    \"\"\" \n",
    "    Giacomini-White Conditional Predictive Ability Test\n",
    "    Parameters\n",
    "    ----------\n",
    "    loss1: numpy array\n",
    "        losses of model 1\n",
    "    loss2: numpy array\n",
    "        losses of model 2\n",
    "    tau: int\n",
    "        the past information treated as 'available' for the test.\n",
    "    unconditional: boolean, \n",
    "        True if unconditional (DM test), False if conditional (GW test).\n",
    "    verbose: boolean, \n",
    "        True if prints of test are needed\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    test_stat: test statistic of the conditional predictive ability test\n",
    "    crit_val: critical value of the chi-square test for a 5% confidence level\n",
    "    p-vals: (k,) p-value of the test\n",
    "    \"\"\"   \n",
    "\n",
    "    assert len(loss1) == len(loss2)\n",
    "\n",
    "    lossdiff = loss1 - loss2\n",
    "    t = len(loss1)\n",
    "    instruments = np.ones_like(loss1)\n",
    "\n",
    "    if conditional:\n",
    "        instruments = np.hstack((instruments[:t-tau], \n",
    "                                 lossdiff[:-tau]))\n",
    "        lossdiff = lossdiff[tau:]\n",
    "        t = t - tau\n",
    "\n",
    "    reg = instruments * lossdiff\n",
    "    \n",
    "    if tau == 1:\n",
    "\n",
    "        res_beta = np.linalg.lstsq(reg, np.ones((t)), rcond=None)[0]\n",
    "\n",
    "        err = np.ones((t,1)) - reg.dot(res_beta)\n",
    "        r2 = 1 - np.mean(err**2)\n",
    "        test_stat = t * r2\n",
    "    \n",
    "    else:\n",
    "\n",
    "        zbar = np.mean(reg, axis=0)\n",
    "        n_lags = tau - 1\n",
    "        omega = Newey_West(Z=reg, n_lags=n_lags)\n",
    "        test_stat = np.expand_dims(t*zbar, \n",
    "                                   axis=0).dot(np.linalg.inv(omega)).\\\n",
    "                                   dot(zbar)\n",
    "      \n",
    "    test_stat *= np.sign(np.mean(lossdiff))\n",
    "    \n",
    "    q = reg.shape[1]\n",
    "    crit_val = chi2.ppf(1-alpha, df=q)\n",
    "    p_val = 1 - chi2.cdf(test_stat, q)\n",
    "\n",
    "    av_diff_loss = np.mean(loss1-loss2)\n",
    "    s = '+' if np.mean(loss1-loss2) > 0 else '-'\n",
    "    \n",
    "    if verbose:\n",
    "        if conditional: print('\\nConditional test:\\n')\n",
    "        if not conditional: print('\\nUnconditional test:\\n')\n",
    "        print(f'Forecast horizon: {tau}, Nominal Risk Level: {alpha}')\n",
    "        print(f'Test-statistic: {test_stat} ({s})')\n",
    "        print(f'Critical value: {crit_val}')\n",
    "        print(f'p-value: {p_val}\\n')\n",
    "    \n",
    "    return test_stat, crit_val, p_val\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Conditional test:\n",
      "\n",
      "Forecast horizon: 1, Nominal Risk Level: 0.05\n",
      "Test-statistic: -0.06536064947319775 (-)\n",
      "Critical value: 5.991464547107979\n",
      "p-value: 1.0\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-0.06536064947319775, 5.991464547107979, 1.0)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(1)\n",
    "\n",
    "loss1 = np.random.randint(low=1, high=10, size=(10,1))\n",
    "loss2 = np.random.randint(low=1, high=10, size=(10,1))\n",
    "\n",
    "GW_CPA_test(loss1=loss1, loss2=loss2, tau=1, conditional=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "def GW_test_pvals(y: np.ndarray,\n",
    "                  y_hat: np.ndarray,\n",
    "                  horizon: int,\n",
    "                  tau: int,                \n",
    "                  conditional: bool,\n",
    "                  alpha: float=0.05,\n",
    "                  verbose: bool=False):\n",
    "    \"\"\" Function to calculate model-pair-wise GW-Test p-values\n",
    "    Parameters\n",
    "    ----------\n",
    "    y: numpy array\n",
    "        flat array with actual test values\n",
    "    y_hat: numpy array\n",
    "        matrix with predicted values\n",
    "    model_names: string list \n",
    "        with the names of the models.\n",
    "    horizon: int\n",
    "        the multi horizon for which the predictions were created, \n",
    "        the test is performed against over the mean differences \n",
    "        in said multi horizon losses.\n",
    "    tau: int\n",
    "        the past information treated as 'available' for the test.\n",
    "    alpha: float \n",
    "        level of significance for the test.\n",
    "    unconditional: boolean, \n",
    "        True if unconditional (DM test), False if conditional (GW test).\n",
    "    verbose: boolean.\n",
    "        True for partial test results.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    p_vals: (n_models, n_models) \n",
    "            symmetric numpy array with the model-pair-wise p-values.\n",
    "    \n",
    "    \"\"\"\n",
    "    # number of date stamps and de facto forecast creation dates\n",
    "    y_hat = y_hat.T # transpose for good reshape indexes\n",
    "    n_models, n_ds = y_hat.shape\n",
    "    n_fcds = n_ds // horizon\n",
    "\n",
    "    # multi horizon losses\n",
    "    losses = np.abs(y - y_hat)\n",
    "\n",
    "    print(\"losses.shape\", losses.shape)\n",
    "    print(\"n_models, n_ds, n_fcds\")\n",
    "    print(n_models, n_ds, n_fcds)\n",
    "\n",
    "    losses = losses.reshape(n_models, n_fcds, horizon)\n",
    "    losses = np.mean(losses, axis=2)\n",
    "\n",
    "    print(\"losses.shape\", losses.shape)\n",
    "\n",
    "    pvals = np.zeros((n_models, n_models))\n",
    "\n",
    "    for i in range(n_models):\n",
    "        for j in range(n_models):\n",
    "            loss1 = losses[[i], :]\n",
    "            loss2 = losses[[j], :]\n",
    "            _, _, p_val = GW_CPA_test(loss1=loss1.reshape(n_fcds, -1),\n",
    "                                      loss2=loss2.reshape(n_fcds, -1),\n",
    "                                      tau=1,\n",
    "                                      alpha=alpha,\n",
    "                                      conditional=conditional,\n",
    "                                      verbose=verbose)\n",
    "\n",
    "            pvals[i,j] = p_val\n",
    "\n",
    "    return pvals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "def get_nbeatsx_cmap():\n",
    "    cmap = cm.get_cmap('pink', 512)\n",
    "    yellows = cmap(np.linspace(0.5, 0.95, 256))\n",
    "\n",
    "    cmap = cm.get_cmap('Blues', 256)\n",
    "    blues = cmap(np.linspace(0.45, 0.75, 256))\n",
    "\n",
    "    newcolors = np.concatenate([yellows, blues])\n",
    "\n",
    "    #extra = np.array([116/256, 142/256, 157/256, 1])\n",
    "    extra = np.array([66/256, 75/256, 98/256, 1])\n",
    "    #extra = np.array([3/256, 34/256, 71/256, 1])\n",
    "    newcolors[-10:, :] = extra\n",
    "    newcmap = ListedColormap(newcolors)\n",
    "    return newcmap\n",
    "\n",
    "def get_epftoolbox_cmap():\n",
    "    cmap = cm.get_cmap('YlGn_r', 512)\n",
    "    yellows = cmap(np.linspace(0.6, 1.0, 256))\n",
    "\n",
    "    cmap = cm.get_cmap('gist_heat_r', 256)\n",
    "    reds = cmap(np.linspace(0.39, 0.66, 256))\n",
    "\n",
    "    newcolors = np.concatenate([yellows, reds])\n",
    "\n",
    "    #extra = np.array([116/256, 142/256, 157/256, 1])\n",
    "    #extra = np.array([66/256, 75/256, 98/256, 1])\n",
    "    #extra = np.array([3/256, 34/256, 71/256, 1])\n",
    "    extra = np.array([0, 0, 0, 1])\n",
    "    newcolors[-10:, :] = extra\n",
    "    newcmap = ListedColormap(newcolors)\n",
    "    return newcmap\n",
    "\n",
    "def plot_GW_test_pvals(pvals, labels, title):\n",
    "    assert len(pvals)==len(labels), 'Wrong pvals and labels dimensions.'\n",
    "\n",
    "    #plt.rc('text', usetex=True)\n",
    "    plt.rc('axes', labelsize=14)    # fontsize of the x and y labels\n",
    "    plt.rc('xtick', labelsize=14)    # fontsize of the tick labels\n",
    "    plt.rc('ytick', labelsize=14)    # fontsize of the tick labels\n",
    "    plt.rc('axes', titlesize=16)  # fontsize of the figure title\n",
    "\n",
    "    fig = plt.figure(figsize=[6, 6])\n",
    "    ax = plt.axes([.27, .22, .7, .7])\n",
    "\n",
    "    data = np.float32(pvals)\n",
    "\n",
    "    # Colormap with discontinuous limit\n",
    "    #cmap = cm.get_cmap('GnBu', 256)\n",
    "    #cmap = get_nbeatsx_cmap()\n",
    "    cmap = get_epftoolbox_cmap()\n",
    "    mappable = plt.imshow(data, cmap=cmap, vmin=0, vmax=0.1)\n",
    "\n",
    "    ticklabels = labels #[r'$\\textrm{' + e + '}$' for e in labels]\n",
    "    plt.xticks(range(len(labels)), ticklabels, rotation=90., fontsize=FONTSIZE)\n",
    "    plt.yticks(range(len(labels)), ticklabels, fontsize=FONTSIZE)\n",
    "\n",
    "    plt.plot(list(range(len(labels))), \n",
    "             list(range(len(labels))), 'wx', c='white', markersize=FONTSIZE)\n",
    "    plt.title(f'{title}', fontweight='bold', fontsize=FONTSIZE)\n",
    "\n",
    "    # Turn spines off and create black grid.\n",
    "    for edge, spine in ax.spines.items():\n",
    "        spine.set_visible(False)\n",
    "\n",
    "    ax.set_xticks(np.arange(data.shape[1]+1)-.5, minor=True)\n",
    "    ax.set_yticks(np.arange(data.shape[0]+1)-.5, minor=True)\n",
    "    ax.grid(which=\"minor\", color=\"k\", linestyle='-', linewidth=1.5)\n",
    "\n",
    "    # Colorbar\n",
    "    divider = make_axes_locatable(ax)\n",
    "    cax = divider.append_axes(\"right\", size=\"5%\", pad=0.05)\n",
    "    plt.colorbar(mappable, cax=cax)\n",
    "\n",
    "    #fig.tight_layout()\n",
    "    title = title.replace(\" \", \"_\")\n",
    "    title = title.replace(\",\", \"\")\n",
    "    title = title.replace(\"(\", \"\")\n",
    "    title = title.replace(\")\", \"\")\n",
    "    plt.savefig(f'./results/pvals_{title}.pdf', bbox_inches='tight')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TEST/DEBUG GW TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "losses.shape (5, 2400)\n",
      "n_models, n_ds, n_fcds\n",
      "5 2400 100\n",
      "losses.shape (5, 100)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 19800 characters]\n",
      "text/plain": [
       "<Figure size 432x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "np.random.seed(117)\n",
    "\n",
    "# Observed values for 3 different days\n",
    "y = np.random.randint(low=1, high=10, size=(100*24)) \n",
    "\n",
    "# Predicted values for 3 different days, from 5 different models\n",
    "y_hat = np.random.randint(low=1, high=10, size=(100*24, 5))\n",
    "model_names = ['Model1', 'Model2', 'Model3', 'Model4', 'Model5']\n",
    "\n",
    "pvals = GW_test_pvals(y=y, y_hat=y_hat, horizon=24, tau=1,\n",
    "                      conditional=True, alpha=0.05, verbose=False)\n",
    "\n",
    "plot_GW_test_pvals(pvals=pvals, labels=model_names, title='GW_test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pandas as pd\n",
    "\n",
    "# data = pd.read_csv('./results/ver.csv')\n",
    "\n",
    "# model_names = ['AR1', 'ESRNN', 'NBEATS', 'ARX1', 'LEAR', 'DNN', 'NBEATS_X_G', 'NBEATS_X_I']\n",
    "# pvals = GW_test_pvals(y=np.expand_dims(data['y'].to_numpy(), axis=0),\n",
    "#                       y_hat=data.loc[:, model_names].to_numpy(),\n",
    "#                       horizon=24,\n",
    "#                       tau=1,\n",
    "#                       conditional=True,\n",
    "#                       alpha=0.05,\n",
    "#                       verbose=False)\n",
    "\n",
    "# plot_GW_test_pvals(pvals=pvals, labels=model_names, title='GW_test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
