{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# uncertainty for regression\n",
    "**based on [this example](https://scikit-learn.org/stable/auto_examples/ensemble/plot_gradient_boosting_quantile.html) in sklearn**\n",
    "\n",
    "the basic idea is to train two extra models, one which is over-confident, one which is under-confident to yield prediction intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 29632 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\"\"\"\n",
    "=====================================================\n",
    "Prediction Intervals for Gradient Boosting Regression\n",
    "=====================================================\n",
    "\n",
    "This example shows how quantile regression can be used\n",
    "to create prediction intervals.\n",
    "\"\"\"\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "np.random.seed(1)\n",
    "\n",
    "\n",
    "def f(x):\n",
    "    \"\"\"The function to predict.\"\"\"\n",
    "    return x * np.sin(x)\n",
    "\n",
    "\n",
    "#----------------------------------------------------------------------\n",
    "#  First the noiseless case\n",
    "X = np.atleast_2d(np.random.uniform(0, 10.0, size=100)).T\n",
    "X = X.astype(np.float32)\n",
    "\n",
    "# Observations\n",
    "y = f(X).ravel()\n",
    "\n",
    "dy = 1.5 + 1.0 * np.random.random(y.shape)\n",
    "noise = np.random.normal(0, dy)\n",
    "y += noise\n",
    "y = y.astype(np.float32)\n",
    "\n",
    "# Mesh the input space for evaluations of the real function, the prediction and\n",
    "# its MSE\n",
    "xx = np.atleast_2d(np.linspace(0, 10, 1000)).T\n",
    "xx = xx.astype(np.float32)\n",
    "\n",
    "alpha = 0.95\n",
    "\n",
    "clf = GradientBoostingRegressor(loss='quantile', alpha=alpha,\n",
    "                                n_estimators=250, max_depth=3,\n",
    "                                learning_rate=.1, min_samples_leaf=9,\n",
    "                                min_samples_split=9)\n",
    "\n",
    "clf.fit(X, y)\n",
    "\n",
    "# Make the prediction on the meshed x-axis\n",
    "y_upper = clf.predict(xx)\n",
    "\n",
    "clf.set_params(alpha=1.0 - alpha)\n",
    "clf.fit(X, y)\n",
    "\n",
    "# Make the prediction on the meshed x-axis\n",
    "y_lower = clf.predict(xx)\n",
    "\n",
    "clf.set_params(loss='ls')\n",
    "clf.fit(X, y)\n",
    "\n",
    "# Make the prediction on the meshed x-axis\n",
    "y_pred = clf.predict(xx)\n",
    "\n",
    "# Plot the function, the prediction and the 90% confidence interval based on\n",
    "# the MSE\n",
    "fig = plt.figure()\n",
    "plt.plot(xx, f(xx), 'g:', label=r'$f(x) = x\\,\\sin(x)$')\n",
    "plt.plot(X, y, 'b.', markersize=10, label=u'Observations')\n",
    "plt.plot(xx, y_pred, 'r-', label=u'Prediction')\n",
    "plt.plot(xx, y_upper, 'k-')\n",
    "plt.plot(xx, y_lower, 'k-')\n",
    "plt.fill(np.concatenate([xx, xx[::-1]]),\n",
    "         np.concatenate([y_upper, y_lower[::-1]]),\n",
    "         alpha=.5, fc='b', ec='None', label='90% prediction interval')\n",
    "plt.xlabel('$x$')\n",
    "plt.ylabel('$f(x)$')\n",
    "plt.ylim(-10, 20)\n",
    "plt.legend(loc='upper left')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}