{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "|#|手法|Kerasy|\n",
    "|:--:|:--|:--|\n",
    "|1|<a href=\"#Zeros\">Zeros</a>|`kerasy.initializers.Zeros()`|\n",
    "|2|<a href=\"#Ones\">Ones</a>|`kerasy.initializers.Ones()`|\n",
    "|3|<a href=\"#Constant\">Constant</a>|`kerasy.initializers.Constant(value=0)`|\n",
    "|4|<a href=\"#Random-Normal\">RandomNormal</a>|`kerasy.initializers.RandomNormal(mean=0.0, stddev=0.05, seed=None)`|\n",
    "|5|<a href=\"#Random-Uniform\">RandomUniform</a>|`kerasy.initializers.RandomUniform(minval=-0.05, maxval=0.05, seed=None)`|\n",
    "|6|<a href=\"#Truncated-Normal\">TruncatedNormal</a>|`kerasy.initializers.TruncatedNormal(mean=0.0, stddev=0.05, seed=None)`|\n",
    "|7|<a href=\"#Variance-Scaling\">VarianceScaling</a>|`kerasy.initializers.VarianceScaling(scale=1.0, mode='fan_in', distribution='normal', seed=None)`|\n",
    "|8|<a href=\"#Orthogonal\">Orthogonal</a>|`kerasy.initializers.Orthogonal(gain=1.0, seed=None)`|\n",
    "|9|<a href=\"#Identity\">Identity</a>|`kerasy.initializers.Identity(gain=1.0)`|\n",
    "|10|<a href=\"#Glorot-Normal\">GlorotNormal</a>|`kerasy.initializers.glorot_normal(seed=None)`|\n",
    "|11|<a href=\"#Glorot-Uniform\">GlorotUniform</a>|`kerasy.initializers.glorot_uniform(seed=None)`|\n",
    "|12|<a href=\"#He-Normal\">HeNormal</a>|`kerasy.initializers.he_normal(seed=None)`|\n",
    "|13|<a href=\"#LeCun-Normal\">LeCunNormal</a>|`kerasy.initializers.lecun_normal(seed=None)`|\n",
    "|14|<a href=\"#He-Uniform\">HeUniform</a>|`kerasy.initializers.he_uniform(seed=None)`|\n",
    "|15|<a href=\"#LeCun-Uniform\">LeCunUniform</a>|`kerasy.initializers.lecun_uniform(seed=None)`|"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**※ Followed by [Plot function](#Plot-function)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 50076 characters]\n",
      "text/plain": [
       "<Figure size 1440x864 with 15 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotALLInitializer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Zeros"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generates values initialized to $0$. This is often used as the bias initializer so that your network could learn without affected by bias parameter."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"admonition warning\">\n",
    "  <p class=\"admonition-title\">Warning</p>\n",
    "  <p>If you apply this initializer to edge weights between layers, loss will not convey (back propagation), so notwork could not learn anymore.</p>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def kerasy.initializers.Zeros(shape, dtype=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9064 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMonoInitializer(\"zeros\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ones"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generates values initialized to $1$."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def kerasy.initializers.Ones(shape, dtype=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 8800 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMonoInitializer(\"ones\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Constant"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generates values initialized to user defined **constant (=`value`)**."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def kerasy.initializers.Constant(shape, value=0, dtype=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9296 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMonoInitializer(\"constant\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Normal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generates values from **Normal distribution**."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def kerasy.initializers.RandomNormal(shape, mean=0, stddev=0.05, dtype=None, seed=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9680 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMonoInitializer(\"random_normal\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Uniform"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generates values from **Uniform distribution**."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def kerasy.initializers.RandomUniform(shape, minval=-0.05, maxval=0.05, dtype=None, seed=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9460 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMonoInitializer(\"random_uniform\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Truncated Normal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generates values from **Truncated Normal distribution**, which is derived from a Normal distribution.\n",
    "\n",
    "This distribution leave only the values **within the standard deviation** from the distribution that follows the **Normal distribution**."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"admonition note\">\n",
    "  <p class=\"admonition-title\">Hint</p>\n",
    "  <p>It is recommended to use this distribution for initializing the edge weights between layers.</p>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def kerasy.initializers.TruncatedNormal(shape, mean=0.0, stddev=0.05, dtype=None, seed=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9904 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMonoInitializer(\"truncated_normal\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variance Scaling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Performs scaling according to the size of the weight and generates values from **Truncated Normal** or **Uniform** distributions."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def kerasy.initializers.VarianceScaling(shape, scale=1.0, mode='fan_in', distribution='normal', dtype=None, seed=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 10832 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMonoInitializer(\"variance_scaling\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Orthogonal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generates values as an **Orthogonal matrix**."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def kerasy.initializers.Orthogonal(shape, gain=1.0, dtype=None, seed=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9768 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMonoInitializer(\"orthogonal\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"admonition reference\">\n",
    "  <p class=\"admonition-title\">Reference</p>\n",
    "  <ul>\n",
    "    <li><a href=\"http://arxiv.org/abs/1312.6120\">Exact solutions to the nonlinear dynamics of learning in deeplinear neural networks</a></li>\n",
    "  </ul>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Identity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generates values as an **Identity matrix**."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def kerasy.initializers.Identity(shape, dtype=None, gain=1.0)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9296 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMonoInitializer(\"identity\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"admonition warning\">\n",
    "  <p class=\"admonition-title\">Warning</p>\n",
    "  <p>This initialization can be used when values is a 2-dimentional square matrix.</p>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Glorot Normal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generates values from the following distribution where\n",
    "\n",
    "- $n_{\\text{in}}$: the number of input units\n",
    "- $n_{\\text{out}}$: the number of output units"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\text{Truncated Normal}\\left(0, \\frac{\\sqrt{2}}{\\sqrt{n_{\\text{in}}} + \\sqrt{n_{\\text{out}}}}\\right)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Glorot's objective is to validate\n",
    "\n",
    "- Each **\"activation variances\"** in a layer has the same variance.\n",
    "$$\\forall(i,i^{\\prime}), \\mathrm{Var}\\left[z^i\\right] = \\mathrm{Var}\\left[z^{i^{\\prime}}\\right]\\quad (8)$$\n",
    "- Each **\"activation variances\"** in a layer has the same variance.\n",
    "$$\\forall(i,i^{\\prime}), \\mathrm{Var}\\left[\\frac{\\partial \\text{Cost}}{\\partial s^i}\\right] = \\mathrm{Var}\\left[\\frac{\\partial \\text{Cost}}{\\partial s^{i^{\\prime}}}\\right]\\quad (9)$$\n",
    "\n",
    "These two conditions transform to:\n",
    "\n",
    "$$\n",
    "\\begin{aligned}\n",
    "    \\forall i, n_i\\mathrm{Var}\\left[W^i\\right] &= 1 & (10)\\\\\n",
    "    \\forall i, n_{i+1}\\mathrm{Var}\\left[W^i\\right] &= 1 & (11)\n",
    "\\end{aligned}\n",
    "$$\n",
    "\n",
    "\n",
    "As a compromise between these two constraints, He took \n",
    "\n",
    "$$\\forall i, \\mathrm{Var}\\left[W^i\\right] = \\frac{2}{n_i + n_{i+1}}  \\quad(12)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def kerasy.initializers.GlorotNormal(shape, dtype=None, seed=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 10352 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMonoInitializer(\"glorot_normal\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"admonition reference\">\n",
    "  <p class=\"admonition-title\">Reference</p>\n",
    "  <ul>\n",
    "    <li><a href=\"http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf\">Understanding the difficulty of training deep feedforward neural networks</a></li>\n",
    "  </ul>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Glorot Uniform"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generates values from the following distribution\n",
    "\n",
    "$$\\mathrm{Uni}\\left[-\\frac{\\sqrt{6}}{\\sqrt{n_{\\text{in}} + n_{\\text{out}}}}, \\frac{\\sqrt{6}}{\\sqrt{n_{\\text{in}} + n_{\\text{out}}}}\\right]$$\n",
    "\n",
    "The idea is same with the **Glorot Normal distribution**."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def kerasy.initializer.GlorotUniform(shape, dtype=None, seed=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9572 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMonoInitializer(\"glorot_uniform\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"admonition reference\">\n",
    "  <p class=\"admonition-title\">Reference</p>\n",
    "  <ul>\n",
    "    <li><a href=\"http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf\">Understanding the difficulty of training deep feedforward neural networks</a></li>\n",
    "  </ul>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## He Normal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generates values from the following distribution"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\text{Truncated Normal}\\left(0, \\frac{\\sqrt{2}}{\\sqrt{n_{\\text{in}}}}\\right)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As the **[Glorot Normal](#Glorot-Normal)** has the **restriction** that it valids only when **the activation function is \"origin-symmetric\" and \"linear near the origin\"**, this distribution was invented to deal with the exceptions:\n",
    "\n",
    "- ReLU Activation Function\n",
    "- Convolutional Layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def kerasy.initializer.HeNormal(shape, dtype=None, seed=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9300 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMonoInitializer(\"he_normal\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"admonition reference\">\n",
    "  <p class=\"admonition-title\">Reference</p>\n",
    "  <ul>\n",
    "    <li><a href=\"http://arxiv.org/abs/1502.01852\">Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification</a></li>\n",
    "  </ul>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LeCun Normal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generates the values from **He Normal distribution** scaling to $1/\\sqrt{2}$."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def kerasy.initializer.LeCunNormal(shape, dtype=None, seed=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 10196 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMonoInitializer(\"lecun_normal\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"admonition reference\">\n",
    "  <p class=\"admonition-title\">Reference</p>\n",
    "  <ul>\n",
    "    <li><a href=\"https://arxiv.org/abs/1706.02515\">Self-Normalizing Neural Networks</a></li>\n",
    "    <li><a href=\"http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf\">Efficient BackProp</a></li>\n",
    "  </ul>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## He Uniform"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generates values from the following distribution\n",
    "\n",
    "$$\\mathrm{Uni}\\left[-\\frac{\\sqrt{6}}{\\sqrt{n_{\\text{in}}}}, \\frac{\\sqrt{6}}{\\sqrt{n_{\\text{in}}}}\\right]$$\n",
    "\n",
    "The idea is same with the **He Normal distribution**."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def kerasy.initializer.HeUniform(shape, dtype=None, seed=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9588 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMonoInitializer(\"he_uniform\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"admonition reference\">\n",
    "  <p class=\"admonition-title\">Reference</p>\n",
    "  <ul>\n",
    "    <li><a href=\"http://arxiv.org/abs/1502.01852\">Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification</a></li>\n",
    "  </ul>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LeCun Uniform"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generates the values from **He Uniform distribution** scaling to $1/\\sqrt{2}$."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def kerasy.initializer.LeCunUniform(shape, dtype=None, seed=None)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9440 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMonoInitializer(\"lecun_uniform\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div class=\"admonition reference\">\n",
    "  <p class=\"admonition-title\">Reference</p>\n",
    "  <ul>\n",
    "    <li><a href=\"http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf\">Efficient BackProp</a></li>\n",
    "  </ul>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerasy.models import Sequential\n",
    "from kerasy.layers import Input, Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def visualizeInitialKernel(kernel_name, input_units=1000, bins=30, ax=None):\n",
    "    if ax is None: fig,ax=plt.subplots()\n",
    "    # Create the model.\n",
    "    model = Sequential()\n",
    "    model.add(Input(input_shape=(input_units,)))\n",
    "    model.add(Dense(input_units, kernel_initializer=kernel_name))\n",
    "    model.compile(optimizer='sgd', loss=\"mean_squared_error\")\n",
    "    # Get the layer's kernel(weights).\n",
    "    freq = np.ravel(model.layers[-1].kernel)\n",
    "    ax.hist(freq, bins=bins, density=True, color=\"#722f37\")\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotMonoInitializer(kernel_name, input_units=1000, bins=30):\n",
    "    ax = visualizeInitialKernel(kernel_name, input_units=input_units, bins=bins)\n",
    "    ax.set_xlabel(\"Value\", fontsize=14), ax.set_ylabel(\"Frequency\", fontsize=14)\n",
    "    ax.set_title(f\"Initialize Method: {kernel_name}\", fontsize=14)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerasy.initializers import InitializeHandler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotALLInitializer(n_col=5, input_units=1000, bins=30):\n",
    "    n_fig = len(InitializeHandler)\n",
    "    n_row = n_fig//n_col if n_fig%n_col==0 else n_fig//n_col+1\n",
    "    fig, axes = plt.subplots(n_row, n_col, sharex=\"all\", figsize=(4*n_col, 4*n_row))\n",
    "    for i,kernel_name in enumerate(InitializeHandler.keys()):\n",
    "        ax = visualizeInitialKernel(kernel_name, ax=axes[i//n_col, i%n_col], input_units=input_units, bins=bins)\n",
    "        ax.set_xlim(-0.3, 0.3)\n",
    "        ax.set_title(kernel_name, fontsize=16)\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
