{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp losses.numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NumPy Evaluation Metrics\n",
    "\n",
    "> Metrics for evaluation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from math import sqrt\n",
    "from typing import Optional, Union\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def divide_no_nan(a, b):\n",
    "    \"\"\"\n",
    "    Auxiliary funtion to handle divide by 0\n",
    "    \"\"\"\n",
    "    div = a / b\n",
    "    div[div != div] = 0.0\n",
    "    div[div == float('inf')] = 0.0\n",
    "    return div"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "def metric_protections(y: np.ndarray, y_hat: np.ndarray, weights: np.ndarray):\n",
    "    assert (weights is None) or (np.sum(weights) > 0), 'Sum of weights cannot be 0'\n",
    "    assert (weights is None) or (weights.shape == y_hat.shape), 'Wrong weight dimension'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def mape(y: np.ndarray, y_hat: np.ndarray, \n",
    "         weights: Optional[np.ndarray] = None,\n",
    "         axis: Optional[int] = None) -> Union[float, np.ndarray]:\n",
    "    \"\"\"Calculates Mean Absolute Percentage Error.\n",
    "    \n",
    "    MAPE measures the relative prediction accuracy of a\n",
    "    forecasting method by calculating the percentual deviation\n",
    "    of the prediction and the true value at a given time and\n",
    "    averages these devations over the length of the series.\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    y: numpy array\n",
    "        Actual test values.\n",
    "    y_hat: numpy array\n",
    "        Predicted values.\n",
    "    weights: numpy array, optional\n",
    "        Weights for weighted average.\n",
    "    axis: None or int, optional\n",
    "        Axis or axes along which to average a. \n",
    "        The default, axis=None, will average over all of the \n",
    "        elements of the input array. \n",
    "        If axis is negative it counts \n",
    "        from the last to the first axis.\n",
    " \n",
    "    Returns\n",
    "    -------\n",
    "    mape: numpy array or double\n",
    "        Return the mape along the specified axis.\n",
    "    \"\"\"\n",
    "    metric_protections(y, y_hat, weights)\n",
    "        \n",
    "    delta_y = np.abs(y - y_hat)\n",
    "    scale = np.abs(y)\n",
    "    mape = divide_no_nan(delta_y, scale)\n",
    "    mape = np.average(mape, weights=weights, axis=axis)\n",
    "    mape = 100 * mape\n",
    "    \n",
    "    return mape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def mse(y: np.ndarray, y_hat: np.ndarray, \n",
    "        weights: Optional[np.ndarray] = None,\n",
    "        axis: Optional[int] = None) -> Union[float, np.ndarray]:\n",
    "    \"\"\"Calculates Mean Squared Error.\n",
    "    \n",
    "    MSE measures the prediction accuracy of a\n",
    "    forecasting method by calculating the squared deviation\n",
    "    of the prediction and the true value at a given time and\n",
    "    averages these devations over the length of the series.\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    y: numpy array\n",
    "        Actual test values.\n",
    "    y_hat: numpy array\n",
    "        Predicted values.\n",
    "    weights: numpy array\n",
    "        Weights for weighted average.\n",
    "    axis: None or int, optional\n",
    "        Axis or axes along which to average a. \n",
    "        The default, axis=None, will average over all of the \n",
    "        elements of the input array. \n",
    "        If axis is negative it counts \n",
    "        from the last to the first axis.\n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    mse: numpy array or double\n",
    "        Return the mse along the specified axis.\n",
    "    \"\"\"\n",
    "    metric_protections(y, y_hat, weights)\n",
    "\n",
    "    delta_y = np.square(y - y_hat)\n",
    "    if weights is not None:\n",
    "        mse = np.average(delta_y[~np.isnan(delta_y)], \n",
    "                         weights=weights[~np.isnan(delta_y)], \n",
    "                         axis=axis)\n",
    "    else:\n",
    "        mse = np.nanmean(delta_y, axis=axis)\n",
    "        \n",
    "    return mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def rmse(y: np.ndarray, y_hat: np.ndarray,\n",
    "         weights: Optional[np.ndarray] = None,\n",
    "         axis: Optional[int] = None) -> Union[float, np.ndarray]:\n",
    "    \"\"\"Calculates Root Mean Squared Error.\n",
    "    \n",
    "    RMSE measures the prediction accuracy of a\n",
    "    forecasting method by calculating the squared deviation\n",
    "    of the prediction and the true value at a given time and\n",
    "    averages these devations over the length of the series.\n",
    "    Finally the RMSE will be in the same scale\n",
    "    as the original time series so its comparison with other\n",
    "    series is possible only if they share a common scale.\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    y: numpy array\n",
    "        Actual test values.\n",
    "    y_hat: numpy array\n",
    "        Predicted values.    \n",
    "    weights: numpy array\n",
    "        Weights for weighted average.\n",
    "    axis: None or int, optional\n",
    "        Axis or axes along which to average a. \n",
    "        The default, axis=None, will average over all of the \n",
    "        elements of the input array. \n",
    "        If axis is negative it counts \n",
    "        from the last to the first axis.\n",
    "      \n",
    "    Returns\n",
    "    -------\n",
    "    rmse: numpy array or double\n",
    "        Return the rmse along the specified axis.\n",
    "    \"\"\"\n",
    "\n",
    "    return np.sqrt(mse(y, y_hat, weights, axis))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def smape(y: np.ndarray, y_hat: np.ndarray,\n",
    "          weights: Optional[np.ndarray] = None,\n",
    "          axis: Optional[int] = None) -> Union[float, np.ndarray]:\n",
    "    \"\"\"Calculates Symmetric Mean Absolute Percentage Error.\n",
    "    \n",
    "    SMAPE measures the relative prediction accuracy of a\n",
    "    forecasting method by calculating the relative deviation\n",
    "    of the prediction and the true value scaled by the sum of the\n",
    "    absolute values for the prediction and true value at a\n",
    "    given time, then averages these devations over the length\n",
    "    of the series. This allows the SMAPE to have bounds between\n",
    "    0% and 200% which is desireble compared to normal MAPE that\n",
    "    may be undetermined.\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    y: numpy array\n",
    "        Actual test values.\n",
    "    y_hat: numpy array\n",
    "        Predicted values.\n",
    "    weights: numpy array\n",
    "        Weights for weighted average.\n",
    "    axis: None or int, optional\n",
    "        Axis or axes along which to average a. \n",
    "        The default, axis=None, will average over all of the \n",
    "        elements of the input array. \n",
    "        If axis is negative it counts \n",
    "        from the last to the first axis.\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    smape: numpy array or double\n",
    "        Return the smape along the specified axis.\n",
    "    \"\"\"\n",
    "    metric_protections(y, y_hat, weights)\n",
    "        \n",
    "    delta_y = np.abs(y - y_hat)\n",
    "    scale = np.abs(y) + np.abs(y_hat)\n",
    "    smape = divide_no_nan(delta_y, scale)\n",
    "    smape = 200 * np.average(smape, weights=weights, axis=axis)\n",
    "    \n",
    "    if isinstance(smape, float):\n",
    "        assert smape <= 200, 'SMAPE should be lower than 200'\n",
    "    else:\n",
    "        assert all(smape <= 200), 'SMAPE should be lower than 200'\n",
    "    \n",
    "    return smape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def mase(y: np.ndarray, y_hat: np.ndarray, \n",
    "         y_train: np.ndarray,\n",
    "         seasonality: int,\n",
    "         weights: Optional[np.ndarray] = None,\n",
    "         axis: Optional[int] = None) -> Union[float, np.ndarray]:\n",
    "    \"\"\"Calculates the Mean Absolute Scaled Error.\n",
    "\n",
    "    MASE measures the relative prediction accuracy of a\n",
    "    forecasting method by comparinng the mean absolute errors\n",
    "    of the prediction and the true value against the mean\n",
    "    absolute errors of the seasonal naive model.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    y: numpy array\n",
    "        Actual test values.\n",
    "    y_hat: numpy array\n",
    "        Predicted values.\n",
    "    y_train: numpy array\n",
    "        Actual insample values for Seasonal Naive predictions.\n",
    "    seasonality: int\n",
    "        Main frequency of the time series\n",
    "        Hourly 24,  Daily 7, Weekly 52,\n",
    "        Monthly 12, Quarterly 4, Yearly 1.\n",
    "    weights: numpy array\n",
    "        Weights for weighted average.\n",
    "    axis: None or int, optional\n",
    "        Axis or axes along which to average a. \n",
    "        The default, axis=None, will average over all of the \n",
    "        elements of the input array. \n",
    "        If axis is negative it counts \n",
    "        from the last to the first axis.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    mase: numpy array or double\n",
    "        Return the mase along the specified axis.\n",
    "\n",
    "    References\n",
    "    ----------\n",
    "    [1] https://robjhyndman.com/papers/mase.pdf\n",
    "    \"\"\"    \n",
    "    delta_y = np.abs(y - y_hat)\n",
    "    delta_y = np.average(delta_y, weights=weights, axis=axis)\n",
    "    \n",
    "    scale = np.abs(y_train[:-seasonality] - y_train[seasonality:])\n",
    "    scale = np.average(scale, axis=axis)\n",
    "    \n",
    "    mase = delta_y / scale\n",
    "    \n",
    "    return mase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def mae(y: np.ndarray, y_hat: np.ndarray,\n",
    "        weights: Optional[np.ndarray] = None,\n",
    "        axis: Optional[int] = None) -> Union[float, np.ndarray]:\n",
    "    \"\"\"Calculates Mean Absolute Error.\n",
    "\n",
    "    The mean absolute error \n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    y: numpy array\n",
    "        Actual test values.\n",
    "    y_hat: numpy array\n",
    "        Predicted values.\n",
    "    weights: numpy array\n",
    "        Weights for weighted average.\n",
    "    axis: None or int, optional\n",
    "        Axis or axes along which to average a. \n",
    "        The default, axis=None, will average over all of the \n",
    "        elements of the input array. \n",
    "        If axis is negative it counts \n",
    "        from the last to the first axis.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    mae: numpy array or double\n",
    "        Return the mae along the specified axis.\n",
    "    \"\"\"\n",
    "    metric_protections(y, y_hat, weights)\n",
    "    \n",
    "    delta_y = np.abs(y - y_hat)\n",
    "    if weights is not None:\n",
    "        mae = np.average(delta_y[~np.isnan(delta_y)], \n",
    "                         weights=weights[~np.isnan(delta_y)],\n",
    "                         axis=axis)\n",
    "    else:\n",
    "        mae = np.nanmean(delta_y, axis=axis)\n",
    "        \n",
    "    return mae"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def pinball_loss(y: np.ndarray, y_hat: np.ndarray, tau: float = 0.5, \n",
    "                 weights: Optional[np.ndarray] = None,\n",
    "                 axis: Optional[int] = None) -> Union[float, np.ndarray]:\n",
    "    \"\"\"Calculates the Pinball Loss.\n",
    "\n",
    "    The Pinball loss measures the deviation of a quantile forecast.\n",
    "    By weighting the absolute deviation in a non symmetric way, the\n",
    "    loss pays more attention to under or over estimation.\n",
    "    A common value for tau is 0.5 for the deviation from the median.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    y: numpy array\n",
    "        Actual test values.\n",
    "    y_hat: numpy array\n",
    "        Predicted values.   \n",
    "    weights: numpy array\n",
    "        Weights for weighted average.      \n",
    "    tau: float\n",
    "        Fixes the quantile against which the predictions are compared.\n",
    "    axis: None or int, optional\n",
    "        Axis or axes along which to average a. \n",
    "        The default, axis=None, will average over all of the \n",
    "        elements of the input array. \n",
    "        If axis is negative it counts \n",
    "        from the last to the first axis.\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    pinball loss: numpy array or double\n",
    "        Return the pinball loss along the specified axis.\n",
    "    \"\"\"\n",
    "    metric_protections(y, y_hat, weights)\n",
    "\n",
    "    delta_y = y - y_hat\n",
    "    pinball = np.maximum(tau * delta_y, (tau - 1) * delta_y)\n",
    "\n",
    "    if weights is not None:\n",
    "        pinball = np.average(pinball[~np.isnan(pinball)], \n",
    "                             weights=weights[~np.isnan(pinball)],\n",
    "                             axis=axis)\n",
    "    else:\n",
    "        pinball = np.nanmean(pinball, axis=axis)\n",
    "        \n",
    "    return pinball"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "def rmae(y: np.ndarray, \n",
    "         y_hat1: np.ndarray, y_hat2: np.ndarray, \n",
    "         weights: Optional[np.ndarray] = None,\n",
    "         axis: Optional[int] = None) -> Union[float, np.ndarray]:\n",
    "    \"\"\"Calculates Relative Mean Absolute Error.\n",
    "\n",
    "    The relative mean absolute error of two forecasts.\n",
    "    A number smaller than one implies that the forecast in the \n",
    "    numerator is better than the forecast in the denominator.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    y: numpy array\n",
    "        Actual test values.\n",
    "    y_hat1: numpy array\n",
    "        Predicted values of first model.\n",
    "    y_hat2: numpy array\n",
    "        Predicted values of second model.\n",
    "    weights: numpy array\n",
    "        Weights for weighted average.\n",
    "    axis: None or int, optional\n",
    "        Axis or axes along which to average a. \n",
    "        The default, axis=None, will average over all of the \n",
    "        elements of the input array. \n",
    "        If axis is negative it counts \n",
    "        from the last to the first axis.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    rmae: numpy array or double\n",
    "        Return the rmae along the specified axis.\n",
    "    \"\"\"\n",
    "    numerator = mae(y=y, y_hat=y_hat1, weights=weights, axis=axis)\n",
    "    denominator = mae(y=y, y_hat=y_hat2, weights=weights, axis=axis)\n",
    "    rmae = numerator / denominator\n",
    "    \n",
    "    return rmae"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi-quantile NumPy loss\n",
    "\n",
    "MQLoss definition and testing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def mqloss(y: np.ndarray, y_hat: np.ndarray, \n",
    "           quantiles: np.ndarray, \n",
    "           weights: Optional[np.ndarray] = None,\n",
    "           axis: Optional[int] = None) -> Union[float, np.ndarray]:\n",
    "    \"\"\"Calculates the MultiQuantile loss.\n",
    "\n",
    "    Calculates Average Multi-quantile Loss function, for\n",
    "    a given set of quantiles, based on the absolute \n",
    "    difference between predicted and true values.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    y: numpy array\n",
    "        Actual test values.\n",
    "    y_hat: numpy array (-1, n_quantiles) \n",
    "        Predicted values.\n",
    "    quantiles: numpy array (n_quantiles) \n",
    "        Quantiles to estimate from the distribution of y.\n",
    "    weights: numpy array\n",
    "        Weights for weighted average.      \n",
    "    axis: None or int, optional\n",
    "        Axis or axes along which to average a. \n",
    "        The default, axis=None, will average over all of the \n",
    "        elements of the input array. \n",
    "        If axis is negative it counts \n",
    "        from the last to the first axis.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    mqloss: numpy array or double\n",
    "        Return the mqloss along the specified axis.\n",
    "    \"\"\" \n",
    "    metric_protections(y, y_hat, weights)\n",
    "    n_q = len(quantiles)\n",
    "    \n",
    "    y_rep = np.expand_dims(y, axis=-1)\n",
    "    error = y_hat - y_rep\n",
    "    sq = np.maximum(-error, np.zeros_like(error))\n",
    "    s1_q = np.maximum(error, np.zeros_like(error)) \n",
    "    loss = (quantiles * sq + (1 - quantiles) * s1_q)\n",
    "    loss = np.average(loss, weights=weights, axis=axis)\n",
    "    \n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.21041164, 0.15391882, 0.09449471, 0.18244145, 0.16071395,\n",
       "       0.13142336, 0.20231349, 0.19795971, 0.15299534, 0.16744432,\n",
       "       0.13345362, 0.16700624, 0.10257573, 0.16436093, 0.16341954,\n",
       "       0.15676824, 0.20708862, 0.15711264, 0.21645739, 0.17206218,\n",
       "       0.23661543, 0.16738924, 0.1577535 , 0.16585761, 0.16793635,\n",
       "       0.15740434, 0.13467679, 0.21414516, 0.14007157, 0.27435823,\n",
       "       0.20451858, 0.11715621, 0.12763097, 0.2100278 , 0.10294569,\n",
       "       0.16492288, 0.11550836, 0.19200599, 0.12251248, 0.17777111,\n",
       "       0.20632066, 0.25534812, 0.22850697, 0.2117134 , 0.20114755,\n",
       "       0.22731604, 0.15800838, 0.17286858, 0.14366458, 0.14034129,\n",
       "       0.20393408, 0.13572314, 0.24699116, 0.17890431, 0.11960552,\n",
       "       0.11549357, 0.13940521, 0.13799181, 0.21941735, 0.10874776,\n",
       "       0.16519921, 0.12306689, 0.1858083 , 0.11227548, 0.14478089,\n",
       "       0.23283138, 0.16608539, 0.2198971 , 0.23463294, 0.22970342,\n",
       "       0.16031605, 0.16675414, 0.17982392, 0.13855352, 0.22509618,\n",
       "       0.18950097, 0.2207443 , 0.15073855, 0.19502465, 0.16299692,\n",
       "       0.11750915, 0.17005201, 0.17436329, 0.17390164, 0.14895287,\n",
       "       0.14644051, 0.10446581, 0.13071105, 0.13252147, 0.10070003,\n",
       "       0.1649476 , 0.28120334, 0.13300598, 0.19693479, 0.16165529,\n",
       "       0.184198  , 0.19409348, 0.14772852, 0.11654125, 0.16790548])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = np.random.random(size=(100, 7))\n",
    "y_q = np.random.random(size=(100, 7, 4))\n",
    "weights = np.ones_like(y_q)\n",
    "quantiles = np.array([0.1, 0.2, 0.3, 0.4])\n",
    "\n",
    "mqloss(y, y_q, quantiles, weights=weights, axis=(1, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.16948741448706903"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mqloss(y, y_q, quantiles)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checks for NumPy Evaluation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.3\n",
      "4.777777777777778\n"
     ]
    }
   ],
   "source": [
    "y = np.array([1,1,1,0,0,0,0,0,1, np.nan])\n",
    "y_mask = np.array([1,1,1,1,1,1,1,1,2,0])\n",
    "y_hat = np.array([1,2,3,-4,-5,-6,-7,-8,-9,-10])\n",
    "\n",
    "print(mae(y=y, y_hat=y_hat, weights=y_mask))\n",
    "print(mae(y=y, y_hat=y_hat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.3\n",
      "4.777777777777778\n"
     ]
    }
   ],
   "source": [
    "print(mae(y=y, y_hat=y_hat, weights=y_mask))\n",
    "print(mae(y=y, y_hat=y_hat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
