{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerasy.utils import galleryplot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerasy.utils import generateWhirlpool\n",
    "from kerasy.utils import findLowerUpper\n",
    "from kerasy.ML.svm import BaseSVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 1\n",
    "N = 150\n",
    "MAX_ITERATION = 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 35604 characters]\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_x, train_y = generateWhirlpool(N, xmin=0, xmax=4, plot=True, figsize=(4,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Convert 0 to -1 to suit for the SVM train data format.\n",
      "Convert 1 to  1 to suit for the SVM train data format.\n"
     ]
    }
   ],
   "source": [
    "train_y = BaseSVM.formatting_y(train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x1min, x2min),(x1max, x2max) = findLowerUpper(train_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X,Y = np.meshgrid(\n",
    "    np.linspace(x1min, x1max, 100),\n",
    "    np.linspace(x2min, x2max, 100)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## hard SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerasy.ML.svm import hardSVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "kernels = [\n",
    "    \"linear\",\"polynomial\",\"gaussian\",\n",
    "    \"sigmoid\",\"laplacian\",\"inverse_multiquadric\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "code_folding": [],
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kernel: linear\n",
      " 60/60 [####################] 100.00% - 151.638s  Rate of Support Vector = 46.0%\n",
      "kernel: polynomial\n",
      " 60/60 [####################] 100.00% - 181.798s  Rate of Support Vector = 38.7%\n",
      "kernel: gaussian\n",
      " 60/60 [####################] 100.00% - 159.566s  Rate of Support Vector = 24.0%\n",
      "kernel: sigmoid\n",
      " 60/60 [####################] 100.00% - 301.119s  Rate of Support Vector = 51.3%\n",
      "kernel: laplacian\n",
      " 60/60 [####################] 100.00% - 259.010s  Rate of Support Vector = 32.7%\n",
      "kernel: inverse_multiquadric\n",
      " 60/60 [####################] 100.00% - 186.292s  Rate of Support Vector = 28.0%\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 286012 characters]\n",
      "text/plain": [
       "<Figure size 864x576 with 6 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_fig = len(kernels)\n",
    "n_col = 3\n",
    "n_row = n_fig//n_col if n_fig%n_col==0 else n_fig//n_col+1\n",
    "\n",
    "fig = plt.figure(figsize=(4*n_col,4*n_row))\n",
    "for i,kernel in enumerate(kernels):\n",
    "    ax = fig.add_subplot(n_row,n_col,i+1)\n",
    "    print(f\"kernel: {kernel}\")\n",
    "    model = hardSVC(kernel=kernel)\n",
    "    model.fit(train_x, train_y, max_iter=MAX_ITERATION, sparse_memorize=False)\n",
    "    \n",
    "    Z = np.vectorize(lambda x,y: model.predict(np.array([x,y]).reshape(1,-1)))(X, Y) == 1\n",
    "    ax.pcolor(X, Y, Z, alpha=0.3)\n",
    "    for i in range(N):\n",
    "        alpha = 1 if i in model.isSV() else 0.3\n",
    "        color = \"red\" if train_y[i]==1 else \"blue\"\n",
    "        ax.plot(train_x[i,0], train_x[i,1], marker=\"o\", color=color, markersize=10, alpha=alpha)\n",
    "    ax.set_title(f\"kernel: {kernel}\\naccuracy: {100*model.accuracy(train_x, train_y):.1f}%\", fontsize=14)\n",
    "    ax.set_xlim(x1min,x1max), ax.set_ylim(x2min,x2max)\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Soft SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerasy.ML.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = [\n",
    "    (1e-2,1e-2),(1e-2,1e0),(1e-2,1e2),\n",
    "    (1e1, 1e-2),(1e1, 1e0),(1e1, 1e2),\n",
    "    (1e4, 1e-2),(1e4, 1e0),(1e4, 1e2),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C=0.01, sigma=0.01\n",
      " 05/60 [#-------------------]   8.33% - 5.844s  Rate of Support Vector = 77.3%\n",
      "C=0.01, sigma=1.0\n",
      " 03/60 [#-------------------]   5.00% - 2.343s  Rate of Support Vector = 79.3%\n",
      "C=0.01, sigma=100.0\n",
      " 04/60 [#-------------------]   6.67% - 3.587s  Rate of Support Vector = 78.0%\n",
      "C=10.0, sigma=0.01\n",
      " 05/60 [#-------------------]   8.33% - 9.993s  Rate of Support Vector = 92.0%\n",
      "C=10.0, sigma=1.0\n",
      " 01/60 [--------------------]   1.67% - 0.000s  Rate of Support Vector = 89.3%\n",
      "C=10.0, sigma=100.0\n",
      " 04/60 [#-------------------]   6.67% - 8.346s  Rate of Support Vector = 85.3%\n",
      "C=10000.0, sigma=0.01\n",
      " 03/60 [#-------------------]   5.00% - 5.524s  Rate of Support Vector = 85.3%\n",
      "C=10000.0, sigma=1.0\n",
      " 12/60 [####----------------]  20.00% - 33.190s  Rate of Support Vector = 86.7%\n",
      "C=10000.0, sigma=100.0\n",
      " 05/60 [#-------------------]   8.33% - 11.416s  Rate of Support Vector = 92.0%\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 248396 characters]\n",
      "text/plain": [
       "<Figure size 864x864 with 9 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_fig = len(params)\n",
    "n_col = 3\n",
    "n_row = n_fig//n_col if n_fig%n_col==0 else n_fig//n_col+1\n",
    "\n",
    "fig = plt.figure(figsize=(4*n_col,4*n_row))\n",
    "for i,(C,sigma) in enumerate(params):\n",
    "    ax = fig.add_subplot(n_row,n_col,i+1)\n",
    "    print(f\"C={C}, sigma={sigma}\")\n",
    "    model = SVC(kernel=\"gaussian\",C=C,sigma=sigma)\n",
    "    model.fit(train_x, train_y, max_iter=MAX_ITERATION, sparse_memorize=False)\n",
    "    \n",
    "    Z = np.vectorize(lambda x,y: model.predict(np.array([x,y]).reshape(1,-1)))(X, Y) == 1\n",
    "    ax.pcolor(X, Y, Z, alpha=0.3)\n",
    "    for i in range(N):\n",
    "        alpha = 1 if i in model.isSV() else 0.3\n",
    "        color = \"red\" if train_y[i]==1 else \"blue\"\n",
    "        ax.plot(train_x[i,0], train_x[i,1], marker=\"o\", color=color, markersize=10, alpha=alpha)\n",
    "    ax.set_title(f\"gaussian kernel (C={C}, $\\sigma={sigma}$)\\naccuracy: {100*model.accuracy(train_x, train_y):.1f}%\", fontsize=14)\n",
    "    ax.set_xlim(x1min,x1max), ax.set_ylim(x2min,x2max)\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multiple SVM (Soft)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerasy.utils import generateWholeCakes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "r_high = 5\n",
    "rmin,rmax = -r_high, r_high"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "X,Y = np.meshgrid(\n",
    "    np.linspace(rmin, rmax, 100),\n",
    "    np.linspace(rmin, rmax, 100)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 12676 characters]\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_x, train_y = generateWholeCakes(4, N, r_high=r_high, same=False, plot=True, seed=seed, figsize=(4,4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerasy.ML.svm import MultipleSVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "C,sigma = (1e1, 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 60916 characters]\n",
      "text/plain": [
       "<Figure size 432x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = MultipleSVM(kernel=\"gaussian\", C=C, sigma=sigma)\n",
    "model.fit(train_x, train_y, max_iter=MAX_ITERATION, sparse_memorize=False)\n",
    "Z = np.vectorize(lambda x,y: model.predict(np.array([x,y]).reshape(1,-1)))(X, Y)\n",
    "\n",
    "plt.figure(figsize=(6,6))\n",
    "plt.pcolor(X, Y, Z, alpha=0.3)\n",
    "plt.scatter(train_x[:,0], train_x[:,1], c=train_y, s=100)\n",
    "plt.title(f\"one-versus-the-rest\\ngaussian kernel (C={C}, $\\sigma={sigma}$)\\naccuracy: {100*model.accuracy(train_x, train_y):.1f}%\", fontsize=14)\n",
    "plt.xlim(rmin,rmax), plt.ylim(rmin,rmax)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
