{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "57e49fa3-3a11-4ed8-a091-035e22363424",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/jonas_h/ch4_py/notebooks/Time Series Analysis/dlm_helper\n"
     ]
    }
   ],
   "source": [
    "cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7a02cc62-6a31-4412-b6c2-5523b7b44c1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import statsmodels.api as sm\n",
    "import numpy as np\n",
    "\n",
    "import dlmhelper.data\n",
    "import dlmhelper.tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6c362837-6665-406d-ac77-8f02ee561e45",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = sm.datasets.nile.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b63314e0-e222-4fc0-8326-9d5173a33c92",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nile River flows at Ashwan 1871-1970\n"
     ]
    }
   ],
   "source": [
    "print(sm.datasets.nile.TITLE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fb751886-fa6d-4b57-bbd7-a5f2c14ad5c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>volume</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1871.0</td>\n",
       "      <td>1120.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1872.0</td>\n",
       "      <td>1160.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1873.0</td>\n",
       "      <td>963.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1874.0</td>\n",
       "      <td>1210.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1875.0</td>\n",
       "      <td>1160.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>1966.0</td>\n",
       "      <td>746.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>1967.0</td>\n",
       "      <td>919.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>1968.0</td>\n",
       "      <td>718.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>1969.0</td>\n",
       "      <td>714.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>1970.0</td>\n",
       "      <td>740.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      year  volume\n",
       "0   1871.0  1120.0\n",
       "1   1872.0  1160.0\n",
       "2   1873.0   963.0\n",
       "3   1874.0  1210.0\n",
       "4   1875.0  1160.0\n",
       "..     ...     ...\n",
       "95  1966.0   746.0\n",
       "96  1967.0   919.0\n",
       "97  1968.0   718.0\n",
       "98  1969.0   714.0\n",
       "99  1970.0   740.0\n",
       "\n",
       "[100 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b069d03-d157-4be8-bd52-279280731c92",
   "metadata": {},
   "source": [
    "Now we create a time series object from this data using the following inputs:\n",
    "\n",
    "- **time & data** : provide the data with corresponding time values\n",
    "- **product_type** : allows to save some identifier with the data\n",
    "- **time_unit** : the time unit declares what a single time step corresponds to\n",
    "- **reference_time** : the reference time in this case is the year 0\n",
    "\n",
    "Alternatively we could pass an array of numpy.datetime64 values to the constructor and only declare the time_unit (since the reference time is already included in the numpy.datetime64 objects):\n",
    "- **dlmhelper.data.TimeSeries(data, time64 = time64,...)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c467431f-7715-4d5a-bac0-1d5192552041",
   "metadata": {},
   "outputs": [],
   "source": [
    "time = np.asarray(dataset.data.year)\n",
    "data = np.asarray(dataset.data.volume)\n",
    "\n",
    "product_type=\"Nile_Flows_Ashwan\"\n",
    "time_unit = \"year\"\n",
    "reference_time = np.datetime64('0000-01-01')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "434d0279-41b8-4a2f-ac0e-3cda05e6a5c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Uncomment to see the docstring for TimeSeries\n",
    "#?dlmhelper.data.TimeSeries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7ede2ec2-5a5f-40fd-abb1-c41e5d7c4857",
   "metadata": {},
   "outputs": [],
   "source": [
    "timeseries = dlmhelper.data.TimeSeries(data, time = time, time_unit=time_unit, reference_time= reference_time, product_type=product_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe992115-a7fc-4e80-b96b-1119553af847",
   "metadata": {},
   "source": [
    "The timeseries can be quickly plotted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "719ff329-6660-45e3-9d05-a2ec50d697b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 12264 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_fig, _ax = timeseries.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df18053b-7eca-4cf5-b06e-c201f3966a5e",
   "metadata": {},
   "source": [
    "Next we perform a single dynamic linear fit on the time series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4241900c-2c53-4a92-9344-cacc5daaa6af",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = dlmhelper.tools.dlm_fit(timeseries, \"Nile_fit_test\",level=True, variable_level=False,trend=True, variable_trend=True,\n",
    "                    seasonal=False, autoregressive=None, irregular=True\n",
    "                       )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ff2facce-f3c2-4bf5-8505-0110475caec8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Uncomment to see the docstring for dlm_fit\n",
    "#?dlmhelper.tools.dlm_fit"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af27a45e-de85-4b88-8f70-85287f5f253d",
   "metadata": {},
   "source": [
    "We can quicky visualize an overview of the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "68c51b09-140a-4fb0-acb1-514562903234",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 41444 characters]\n",
      "text/plain": [
       "<Figure size 576x288 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = result.plot_summary(figsize=(8,4))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c686c61c-f070-49ff-b5ba-9e1a31ae8bfc",
   "metadata": {},
   "source": [
    "It is also possible to perform an ensemble fit on the data using a single command.\n",
    "\n",
    "We vary the variability of the trend and the inclusion of an irregular or autoregressive component"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "af27e773-63da-4483-8319-8cf326806014",
   "metadata": {},
   "outputs": [],
   "source": [
    "seasonal = [False]\n",
    "variable_trend = [True, False]\n",
    "autoregressive = [None,1]\n",
    "irregular = [True, False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "06453da7-ec50-4c81-aca5-c615e27344a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jonas_h/ch4_py/lib/python3.8/site-packages/statsmodels/tsa/statespace/structural.py:508: SpecificationWarning: Specified model does not contain a stochastic element; irregular component added.\n",
      "  warn(\"Specified model does not contain a stochastic element;\"\n"
     ]
    }
   ],
   "source": [
    "result_list = dlmhelper.tools.dlm_ensemble(timeseries, \"Nile_ensemble_test\",\n",
    "                                           seasonal=seasonal, variable_trend = variable_trend,\n",
    "                                           autoregressive=autoregressive, irregular=irregular)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85db0a33-b773-488b-a2e5-dbf0ef40cd94",
   "metadata": {},
   "source": [
    "The results can be quickly listed using the **summary()** method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "32dec29c-3442-4f42-8729-edc20101ebbc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Model    | converged   |     aic |       ll |         ssr |     mse |        cov_level |   cov_trend |   cov_seas |    cov_ar |   cv_amse |   agg |\n",
      "|:---------|:------------|--------:|---------:|------------:|--------:|-----------------:|------------:|-----------:|----------:|----------:|------:|\n",
      "| LTs_A1   | True        | 1262.3  | -628.149 | 2.21447e+06 | 22596.6 |   1238.67        |    7.37248  |          0 |   1238.67 |       nan |   nan |\n",
      "| LT_A1_I  | True        | 1263.12 | -628.561 | 2.22819e+06 | 22736.6 |   1937.13        |    1.12662  |          0 |   5423.81 |       nan |   nan |\n",
      "| LT_A1    | True        | 1264.04 | -630.021 | 2.30372e+06 | 23507.3 |   1033.71        |    0.62355  |          0 |   1033.71 |       nan |   nan |\n",
      "| LTs_A1_I | True        | 1267.71 | -629.856 | 2.26672e+06 | 23129.8 | 818056           |   18.1814   |          0 | 820269    |       nan |   nan |\n",
      "| LTs_I    | True        | 1268.4  | -632.2   | 2.37133e+06 | 24197.3 |    808.834       |    8.43658  |          0 |      0    |       nan |   nan |\n",
      "| LT_I     | True        | 1278.67 | -638.333 | 2.66183e+06 | 27161.5 |    444.203       |    0.271702 |          0 |      0    |       nan |   nan |\n",
      "| LT_I     | True        | 1278.67 | -638.333 | 2.66183e+06 | 27161.5 |    444.203       |    0.271702 |          0 |      0    |       nan |   nan |\n",
      "| LTs      | True        | 1385.57 | -691.786 | 7.76534e+06 | 79238.1 |     -7.72775e-13 |  808.152    |          0 |      0    |       nan |   nan |\n"
     ]
    }
   ],
   "source": [
    "result_list.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f8e374d-98b7-4240-9eca-a3d2336f0c53",
   "metadata": {},
   "source": [
    "The results can be quickly plotted using the **plot_summary()** method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e0c4852f-1d99-4ef2-8c47-c2f074c774af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 31556 characters]\n",
      "text/plain": [
       "<Figure size 864x288 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "result_list.plot_summary(sort=\"aic\", num=4, figsize=(12,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74e326ef-9ca4-4d82-8238-425e02a2a43a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
