{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerasy.utils import generateSin\n",
    "from kerasy.utils import root_mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 0\n",
    "xmin,xmax=(0,1)\n",
    "N = 10\n",
    "X = np.linspace(xmin, xmax, 100)\n",
    "Y = np.sin(2*np.pi*X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x, train_y = generateSin(N, xmin=xmin, xmax=xmax, seed=seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerasy.ML.linear import LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "Ms = [2,4,6,8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 103004 characters]\n",
      "text/plain": [
       "<Figure size 864x576 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig=plt.figure(figsize=(12,8))\n",
    "for i,M in enumerate(Ms):\n",
    "    ax=fig.add_subplot(2,2,i+1)\n",
    "    # Training.\n",
    "    model = LinearRegression(basis=\"polynomial\", M=M)\n",
    "    model.fit(train_x,train_y)\n",
    "    # Plot\n",
    "    ax.plot(X, model.predict(X),  color=\"red\", label=\"prediction\")\n",
    "    ax.plot(X, Y,                 color=\"black\", label=\"$sin(2\\pi x)$\")\n",
    "    ax.scatter(train_x, train_y, edgecolors='blue',facecolor=\"white\",label=\"train data\")\n",
    "    ax.set_title(f\"RMSE: {root_mean_squared_error(model.predict(train_x), train_y):.3f}\\npolynomial base transformer(M={M})\", fontsize=14)\n",
    "    ax.set_ylim(-1.2,1.2)\n",
    "    ax.legend()\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "Normalmodel = model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear Regression (Ridge Regularization L2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerasy.ML.linear import LinearRegressionRidge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "lambda_ = 1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 96240 characters]\n",
      "text/plain": [
       "<Figure size 864x576 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig=plt.figure(figsize=(12,8))\n",
    "for i,M in enumerate(Ms):\n",
    "    ax=fig.add_subplot(2,2,i+1)\n",
    "    # Training.\n",
    "    model = LinearRegressionRidge(lambda_, basis=\"polynomial\", M=M)\n",
    "    model.fit(train_x,train_y)\n",
    "    # Plot\n",
    "    ax.plot(X, model.predict(X),  color=\"red\", label=\"prediction\")\n",
    "    ax.plot(X, Y,                 color=\"black\", label=\"$sin(2\\pi x)$\")\n",
    "    ax.scatter(train_x, train_y, edgecolors='blue',facecolor=\"white\",label=\"train data\")\n",
    "    ax.set_title(f\"RMSE: {root_mean_squared_error(model.predict(train_x), train_y):.3f}\\npolynomial base transformer(M={M})\", fontsize=14)\n",
    "    ax.set_ylim(-1.2,1.2)\n",
    "    ax.legend()\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "Ridgemodel = model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear Regression (LASSO Regularization L1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerasy.ML.linear import LinearRegressionLASSO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "lambda_ = 1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100/100 [####################] 100.00% - 0.105s  "
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 103976 characters]\n",
      "text/plain": [
       "<Figure size 864x576 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig=plt.figure(figsize=(12,8))\n",
    "for i,M in enumerate(Ms):\n",
    "    ax=fig.add_subplot(2,2,i+1)\n",
    "    # Training.\n",
    "    model = LinearRegressionLASSO(lambda_, basis=\"polynomial\", M=M)\n",
    "    model.fit(train_x,train_y)\n",
    "    # Plot\n",
    "    ax.plot(X, model.predict(X), color=\"red\", label=\"prediction\")\n",
    "    ax.plot(X, Y,                color=\"black\", label=\"$sin(2\\pi x)$\")\n",
    "    ax.scatter(train_x, train_y, edgecolors='blue',facecolor=\"white\",label=\"train data\")\n",
    "    ax.set_title(f\"RMSE: {root_mean_squared_error(model.predict(train_x), train_y):.3f}\\npolynomial base transformer(M={M})\", fontsize=14)\n",
    "    ax.set_ylim(-1.2,1.2)\n",
    "    ax.legend()\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "LASSOmodel = model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare weights $\\mathbf{w}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 78520 characters]\n",
      "text/plain": [
       "<Figure size 1296x576 with 6 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(18,8))\n",
    "for i,model in enumerate([Normalmodel, Ridgemodel, LASSOmodel]):\n",
    "    ax = fig.add_subplot(2,3,i+4)\n",
    "    ax.bar(np.arange(len(model.w)),model.w, color=\"black\")\n",
    "    ax.set_title(f\"Weights of {model.__class__.__name__}\", fontsize=14)\n",
    "    \n",
    "    ax = fig.add_subplot(2,3,i+1)\n",
    "    ax.plot(X, model.predict(X),  color=\"red\", label=\"prediction\")\n",
    "    ax.plot(X, Y,                 color=\"black\", label=\"$sin(2\\pi x)$\")\n",
    "    ax.scatter(train_x, train_y, edgecolors='blue',facecolor=\"white\",label=\"train data\")\n",
    "    ax.set_title(f\"RMSE: {root_mean_squared_error(model.predict(train_x), train_y):.3f}\\npolynomial base transformer(M={M})\", fontsize=14)\n",
    "    ax.set_ylim(-1.2,1.2)\n",
    "    ax.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BayesianLinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerasy.ML.linear import BayesianLinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha = 1\n",
    "beta = 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 25\n",
    "M = 25\n",
    "train_x, train_y = generateSin(N, xmin=xmin, xmax=xmax, seed=seed)\n",
    "random_idx = np.random.RandomState(seed).choice(N, N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "mus = np.linspace(xmin,xmax,M)\n",
    "sigma = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 137512 characters]\n",
      "text/plain": [
       "<Figure size 1152x576 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(16,8))\n",
    "\n",
    "for i,n in enumerate([1,2,4,25]):\n",
    "    ax = fig.add_subplot(2,2,i+1)\n",
    "    model = BayesianLinearRegression(alpha=alpha, beta=beta, basis=\"gaussian\", mu=mus, sigma=sigma)\n",
    "    model.fit(train_x[random_idx[:n]],train_y[random_idx[:n]])\n",
    "    y_pred, y_std = model.predict(X)\n",
    "    ax.plot(X, Y, color=\"black\", label=\"$\\sin(2\\pi x)$\")\n",
    "    ax.plot(X, y_pred, color=\"red\", label=\"predict($\\mu$)\")\n",
    "    ax.fill_between(X, y_pred-y_std, y_pred+y_std, color=\"pink\", alpha=0.5)\n",
    "    ax.scatter(train_x[random_idx[:n]], train_y[random_idx[:n]],s=150,edgecolors='black',facecolor=\"white\",label=\"train data\")\n",
    "    ax.set_ylim(-1.5,1.5), ax.legend(), ax.set_title(f\"Bayesian Linear Regression (N={n})\")\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evidence Approximation Bayesian Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerasy.ML.linear import EvidenceApproxBayesianRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha = 100\n",
    "beta = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 25\n",
    "Ms = np.arange(10)\n",
    "train_x, train_y = generateSin(N, xmin=xmin, xmax=xmax, seed=seed)\n",
    "random_idx = np.random.RandomState(seed).choice(N, N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "012/100 [##------------------]  12.00% - 0.013s  "
     ]
    }
   ],
   "source": [
    "evidences = []\n",
    "history = []\n",
    "for M in Ms:\n",
    "    model = EvidenceApproxBayesianRegression(alpha=alpha, beta=beta, basis=\"polynomial\", M=M)\n",
    "    model.fit(train_x, train_y)\n",
    "    evidences.append(model.evidence(train_x, train_y))\n",
    "    history.append(model)\n",
    "best_M = Ms[np.argmax(evidences)]\n",
    "best_model = history[np.argmax(evidences)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 18136 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(Ms, evidences, color=\"black\")\n",
    "plt.scatter(Ms, evidences, color=\"black\")\n",
    "plt.scatter(best_M, np.max(evidences), edgecolors='black',facecolor=\"red\", s=100, label=\"Best Model\")\n",
    "plt.title(\"The relationship between 'model evidence' and $M$\"),\n",
    "plt.xlabel(\"$M$\"), plt.ylabel(\"$\\log({model\\ evidence})$\", fontsize=16), plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Best Parameter]\n",
      "M     = 4\n",
      "alpha = 0.006927681904054397\n",
      "beta  = 94.08159213785791\n"
     ]
    }
   ],
   "source": [
    "print(\"[Best Parameter]\")\n",
    "print(f\"M     = {best_M}\")\n",
    "print(f\"alpha = {best_model.alpha}\")\n",
    "print(f\"beta  = {best_model.beta}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 31512 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_pred, y_std = best_model.predict(X)\n",
    "plt.plot(X, Y,color=\"black\", label=\"$\\sin(2\\pi x)$\")\n",
    "plt.plot(X, Y,color=\"red\", label=\"prediction\")\n",
    "plt.fill_between(X, y_pred-y_std, y_pred+y_std, color=\"pink\", alpha=0.5)\n",
    "plt.scatter(train_x,train_y,s=50,edgecolors='black',facecolor=\"white\",label=\"train data\")\n",
    "plt.title(f\"Bayesian Linear Regression (M={best_M})\")\n",
    "plt.legend(), plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
