{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:38:06.964358Z",
     "start_time": "2020-09-29T21:38:06.962517Z"
    }
   },
   "outputs": [],
   "source": [
    "# default_exp layers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Layers\n",
    "\n",
    "> Custom activations, layers, and layer blocks are contained in this module."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:38:08.559022Z",
     "start_time": "2020-09-29T21:38:07.355322Z"
    }
   },
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:38:09.098877Z",
     "start_time": "2020-09-29T21:38:08.560936Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:53:57.671847Z",
     "start_time": "2020-09-29T21:53:57.631975Z"
    }
   },
   "outputs": [],
   "source": [
    "# export\n",
    "from functools import partial\n",
    "import numpy as np\n",
    "from operator import add, truediv, sub\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from htools import add_docstring\n",
    "from incendio.data import probabilistic_hash_tensor\n",
    "from incendio.utils import concat, weighted_avg, identity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:38:14.252082Z",
     "start_time": "2020-09-29T21:38:14.214748Z"
    }
   },
   "outputs": [],
   "source": [
    "# Used for testing only.\n",
    "from collections import defaultdict, Counter\n",
    "from itertools import chain\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "from htools import assert_raises, InvalidArgumentError\n",
    "from incendio.data import probabilistic_hash_item\n",
    "import pandas_htools"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:51:57.282507Z",
     "start_time": "2020-09-29T21:51:57.238502Z"
    }
   },
   "outputs": [],
   "source": [
    "# export\n",
    "class GRelu(nn.Module):\n",
    "    \"\"\"Generic ReLU.\"\"\"\n",
    "\n",
    "    def __init__(self, leak=0.0, max=float('inf'), sub=0.0):\n",
    "        super().__init__()\n",
    "        self.leak = leak\n",
    "        self.max = max\n",
    "        self.sub = sub\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"Check which operations are necessary to save computation.\"\"\"\n",
    "        x = F.leaky_relu(x, self.leak) if self.leak else F.relu(x)\n",
    "        if self.sub:\n",
    "            x -= self.sub\n",
    "        if self.max:\n",
    "            x = torch.clamp(x, max=self.max)\n",
    "        return x\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f'GReLU(leak={self.leak}, max={self.max}, sub={self.sub})'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:51:57.825458Z",
     "start_time": "2020-09-29T21:51:57.783681Z"
    }
   },
   "outputs": [],
   "source": [
    "# export\n",
    "JRelu = GRelu(leak=.1, sub=.4, max=6.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:51:58.607738Z",
     "start_time": "2020-09-29T21:51:58.567365Z"
    }
   },
   "outputs": [],
   "source": [
    "# export\n",
    "class Mish(nn.Module):\n",
    "    \"\"\"OOP form of mish activation.\n",
    "\n",
    "    Mish: A Self Regularized Non-Monotonic Neural Activation Function\n",
    "    https://arxiv.org/pdf/1908.08681v1.pdf\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, x):\n",
    "        return x * torch.tanh(F.softplus(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:51:59.150373Z",
     "start_time": "2020-09-29T21:51:59.109130Z"
    }
   },
   "outputs": [],
   "source": [
    "# export\n",
    "def mish(x):\n",
    "    \"\"\"Functional form of mish activation.\n",
    "\n",
    "    Mish: A Self Regularized Non-Monotonic Neural Activation Function\n",
    "    https://arxiv.org/pdf/1908.08681v1.pdf\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    x: torch.Tensor[float]\n",
    "        Input tensor.\n",
    "    Returns\n",
    "    -------\n",
    "    torch.Tensor[float]: Tensor of same shape as input x.\n",
    "    \"\"\"\n",
    "    return x * torch.tanh(F.softplus(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:51:59.921520Z",
     "start_time": "2020-09-29T21:51:59.874476Z"
    }
   },
   "outputs": [],
   "source": [
    "def plot_activations(z, a, mode='scatter', **kwargs):\n",
    "    \"\"\"Plot an input tensor and its corresponding activations.  Both tensors\n",
    "    will be flattened for plotting.\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    z: tf.Tensor\n",
    "        Tensor containing values to plot on the x axis (we can often think of\n",
    "        this as the output of a linear layer, where z=f(x) and a=mish(z)).\n",
    "    a: tf.Tensor\n",
    "        Tensor containing values to plot on y axis.\n",
    "    mode: str\n",
    "        'scatter' for scatter plot or 'plot' for line plot.\n",
    "    kwargs: Values to be passed to the matplotlib plotting function, such as \n",
    "        's' when in 'scatter' mode or 'lw' in 'plot' mode.\n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    None\n",
    "    \"\"\"\n",
    "    plt_func = getattr(plt, mode)\n",
    "    kwargs = kwargs or {}\n",
    "    if mode == 'scatter' and not kwargs:\n",
    "        kwargs = {'s': .75}\n",
    "    plt_func(z.numpy().flatten(), a.numpy().flatten(), **kwargs)\n",
    "    plt.axvline(0, lw=.5, alpha=.5)\n",
    "    plt.axhline(0, lw=.5, alpha=.5)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:52:00.682143Z",
     "start_time": "2020-09-29T21:52:00.630493Z"
    }
   },
   "outputs": [],
   "source": [
    "x = torch.arange(-5, 5, .05)\n",
    "a = mish(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:52:01.598300Z",
     "start_time": "2020-09-29T21:52:01.285693Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 10384 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_activations(x, a, 'plot')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Layer Blocks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:54:06.520212Z",
     "start_time": "2020-09-29T21:54:06.474525Z"
    }
   },
   "outputs": [],
   "source": [
    "# export\n",
    "class ConvBlock(nn.Module):\n",
    "    \"\"\"Create a convolutional block optionally followed by a batch norm layer.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, c_in, c_out, kernel_size=3, norm=True, activation=JRelu,\n",
    "                 **kwargs):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        -----------\n",
    "        c_in: int\n",
    "            # of input channels.\n",
    "        c_out: int\n",
    "            # of output channels.\n",
    "        kernel_size: int\n",
    "            Size of kernel in conv2d layer. An integer argument will be used\n",
    "            as both the height and width.\n",
    "        norm: bool\n",
    "            If True, include a batch norm layer after the conv layer. If False,\n",
    "            no norm layer will be used. Note that batch norm has learnable\n",
    "            affine parameters which remove the need for a bias in the preceding\n",
    "            conv layer. When batch norm is not used, however, the conv layer\n",
    "            will include a bias term.\n",
    "        activation: nn.Module\n",
    "            Activation function to use at the end of the convolutional block.\n",
    "            (In some cases such as our ResBlock implementation, we pass in None\n",
    "            so that an extra addition can be performed before the final\n",
    "            activation.) Do not use the functional form here as it will be\n",
    "            added to a sequential object. This is an object, not a class.\n",
    "        kwargs: any\n",
    "            Additional keyword args are passed to Conv2d. Useful kwargs include\n",
    "            stride, and padding (see pytorch docs for nn.Conv2d).\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.norm = norm\n",
    "        layers = [nn.Conv2d(c_in, c_out, kernel_size, bias=not norm, **kwargs)]\n",
    "        if norm:\n",
    "            layers.append(nn.BatchNorm2d(c_out))\n",
    "        if activation is not None:\n",
    "            layers.append(activation)\n",
    "        self.block = nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.block(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:54:07.148076Z",
     "start_time": "2020-09-29T21:54:07.105892Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ConvBlock(\n",
       "  (block): Sequential(\n",
       "    (0): Conv2d(3, 5, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (1): GReLU(leak=0.1, max=6.0, sub=0.4)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conv = ConvBlock(3, 5, norm=False)\n",
    "conv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:54:07.506915Z",
     "start_time": "2020-09-29T21:54:07.461843Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 5, 2, 2])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.rand(2, 3, 4, 4)\n",
    "conv(x).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:54:12.427555Z",
     "start_time": "2020-09-29T21:54:12.379391Z"
    }
   },
   "outputs": [],
   "source": [
    "# export\n",
    "class ResBlock(nn.Module):\n",
    "\n",
    "    def __init__(self, c_in, kernel_size=3, norm=True, activation=JRelu, \n",
    "                 stride=1, padding=1, skip_size=2, **kwargs):\n",
    "        \"\"\"Residual block using 2D convolutional layers. Note that kernel_size,\n",
    "        stride, and pad must be selected such that the height and width of \n",
    "        the input remain the same.\n",
    "\n",
    "        Parameters\n",
    "        -----------\n",
    "        c_in: int\n",
    "            # of input channels.\n",
    "        kernel_size: int\n",
    "            Size of filter used in convolution. Default 3 (which becomes 3x3).\n",
    "        norm: bool\n",
    "            Specifies whether to include a batch norm layer after each conv\n",
    "            layer.\n",
    "        activation: callable\n",
    "            Activation function to use.\n",
    "        stride: int\n",
    "            # of pixels the filter moves between each convolution. Default 1.\n",
    "        padding: int\n",
    "            Pixel padding around the input. Default 1.\n",
    "        skip_size: int\n",
    "            Number of conv blocks inside the skip connection (default 2).\n",
    "            ResNet paper notes that skipping a single layer did not show\n",
    "            noticeable improvements.\n",
    "        kwargs: any\n",
    "            Additional kwargs to pass to ConvBlock which will in turn pass them\n",
    "            to Conv2d. If you accidentally pass in a 'c_out', it will be \n",
    "            removed since we need all dimensions to remain unchanged.\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        # Ensure we don't accidentally pass in a different c_out.\n",
    "        kwargs.pop('c_out', None)\n",
    "        self.skip_size = skip_size\n",
    "        self.layers = nn.ModuleList([\n",
    "            ConvBlock(c_in, c_in, kernel_size=kernel_size, norm=norm, \n",
    "                      activation=None, stride=stride, padding=padding, \n",
    "                      **kwargs)\n",
    "            for i in range(skip_size)\n",
    "        ])\n",
    "        self.activation = activation\n",
    "\n",
    "    def forward(self, x):\n",
    "        x_out = x\n",
    "        for i, layer in enumerate(self.layers):\n",
    "            x_out = layer(x_out)\n",
    "\n",
    "            # Final activation must be applied after addition.\n",
    "            if i != self.skip_size - 1:\n",
    "                x_out = self.activation(x_out)\n",
    "\n",
    "        return self.activation(x + x_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:54:12.626231Z",
     "start_time": "2020-09-29T21:54:12.574082Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ResBlock(\n",
       "  (layers): ModuleList(\n",
       "    (0): ConvBlock(\n",
       "      (block): Sequential(\n",
       "        (0): Conv2d(4, 4, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(4, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (1): ConvBlock(\n",
       "      (block): Sequential(\n",
       "        (0): Conv2d(4, 4, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(4, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (activation): GReLU(leak=0.1, max=6.0, sub=0.4)\n",
       ")"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ResBlock(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:54:13.288135Z",
     "start_time": "2020-09-29T21:54:13.247101Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ResBlock(\n",
       "  (layers): ModuleList(\n",
       "    (0): ConvBlock(\n",
       "      (block): Sequential(\n",
       "        (0): Conv2d(4, 4, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      )\n",
       "    )\n",
       "    (1): ConvBlock(\n",
       "      (block): Sequential(\n",
       "        (0): Conv2d(4, 4, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (activation): GReLU(leak=0.1, max=6.0, sub=0.4)\n",
       ")"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ResBlock(4, norm=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:54:13.784990Z",
     "start_time": "2020-09-29T21:54:13.739266Z"
    }
   },
   "outputs": [],
   "source": [
    "# export\n",
    "@add_docstring(nn.Conv2d)\n",
    "class ReflectionPaddedConv2d(nn.Module):\n",
    "    \"\"\"Conv2d only allows padding_mode of `zeros` or `circular`. This\n",
    "    layer is a quick way for us to use reflection padding.\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, in_channels, out_channels, padding=1, \n",
    "                 kernel_size=3, **kwargs):\n",
    "        \"\"\"Do not specify a padding mode.\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        if 'padding_mode' in kwargs: \n",
    "            raise InvalidArgumentError('Remove `padding_mode` from arguments.')\n",
    "        self.reflect = nn.ReflectionPad2d(padding)\n",
    "        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size,\n",
    "                              padding=0)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.reflect(x)\n",
    "        return self.conv(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:54:14.968706Z",
     "start_time": "2020-09-29T21:54:14.923790Z"
    }
   },
   "outputs": [],
   "source": [
    "def show_img(img):\n",
    "    plt.imshow(img.permute(1, 2, 0) / 255)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:54:15.504300Z",
     "start_time": "2020-09-29T21:54:15.460224Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ReflectionPaddedConv2d(\n",
       "  (reflect): ReflectionPad2d((2, 2, 2, 2))\n",
       "  (conv): Conv2d(3, 3, kernel_size=(1, 1), stride=(1, 1))\n",
       ")"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rconv = ReflectionPaddedConv2d(3, 3, kernel_size=1, padding=2)\n",
    "rconv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:54:16.370902Z",
     "start_time": "2020-09-29T21:54:16.120469Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 5016 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = torch.randint(255, (1, 3, 3, 3)).float()\n",
    "show_img(x[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:54:17.165969Z",
     "start_time": "2020-09-29T21:54:16.928408Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 4168 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x2 = rconv.reflect(x)\n",
    "show_img(x2[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:54:21.668079Z",
     "start_time": "2020-09-29T21:54:21.627540Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "As expected, got InvalidArgumentError(Remove `padding_mode` from arguments.).\n"
     ]
    }
   ],
   "source": [
    "# Tests\n",
    "assert nn.Conv2d.__doc__ in ReflectionPaddedConv2d.__doc__\n",
    "\n",
    "with assert_raises(InvalidArgumentError):\n",
    "    ReflectionPaddedConv2d(3, 3, padding_mode='zeros')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:54:23.642196Z",
     "start_time": "2020-09-29T21:54:23.588980Z"
    }
   },
   "outputs": [],
   "source": [
    "# export\n",
    "class Dropin(nn.Module):\n",
    "    \"\"\"Additive dropout. This injects small amounts of noise into a model\n",
    "    in the form of randomly generated floats from a zero-centered\n",
    "    gaussian distribution (variance can be adjusted). This does nothing \n",
    "    in eval mode. Unlike Dropout, this does not scale weights during \n",
    "    training since it does not bias them in any direction.\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, scale=.5):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        scale: float\n",
    "            Used to scale the magnitude of the random noise. Keep in mind \n",
    "            that the scalar term is square rooted, so the relationship\n",
    "            will not be linear. Relatively large values (e.g. 1.0) will have\n",
    "            a stronger regularizing effect, while small values (e.g. 0.1)\n",
    "            will have a slight regularizing effect. There is no max value\n",
    "            enforced, so it's up to the user to select a reasonable value.\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.scale = scale\n",
    "        \n",
    "    def forward(self, x):\n",
    "        if not self.training:\n",
    "            return x\n",
    "        \n",
    "        # Storing noise allows us to run diagnostics.\n",
    "        self.noise = torch.randn_like(x) * np.sqrt(self.scale / x.shape[-1])\n",
    "        return x + self.noise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:54:24.049317Z",
     "start_time": "2020-09-29T21:54:24.005320Z"
    }
   },
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.drop = Dropin()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return self.drop(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:54:24.782894Z",
     "start_time": "2020-09-29T21:54:24.717717Z"
    }
   },
   "outputs": [],
   "source": [
    "net = Net()\n",
    "x = torch.randn(8, 128, 128, 3)\n",
    "assert np.corrcoef(net(x).flatten(), x.flatten())[0][1] > .9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:54:25.553591Z",
     "start_time": "2020-09-29T21:54:25.509721Z"
    }
   },
   "outputs": [],
   "source": [
    "net.eval()\n",
    "assert torch.eq(net(x), x).all()\n",
    "assert not net.drop.training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simulate_activation_stats(scale=1.0, trials=10_000):\n",
    "    act_stats = defaultdict(list)\n",
    "    noise_stats = defaultdict(list)\n",
    "    \n",
    "    drop = Dropin(scale)\n",
    "    for _ in range(trials):\n",
    "        x = torch.randn(3, 4, dtype=torch.float)\n",
    "        z = drop(x)\n",
    "        noise = drop.noise\n",
    "        noise_stats['mean'].append(noise.mean())\n",
    "        noise_stats['std'].append(noise.std())\n",
    "        noise_stats['act_corr'].append(\n",
    "            np.corrcoef(z.flatten(), noise.flatten())[0][1]\n",
    "        )\n",
    "        \n",
    "        act_stats['mean'].append(z.mean())\n",
    "        act_stats['std'].append(z.std())\n",
    "        act_stats['x_corr'].append(\n",
    "            np.corrcoef(z.flatten(), x.flatten())[0][1]\n",
    "        )\n",
    "\n",
    "    return pd.DataFrame(dict(\n",
    "        act={k: np.mean(v).round(4) for k, v in act_stats.items()}, \n",
    "        noise={k: np.mean(v).round(4) for k, v in noise_stats.items()}\n",
    "    ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 10\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>act</th>\n",
       "      <th>noise</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.0132</td>\n",
       "      <td>0.0094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.8189</td>\n",
       "      <td>1.5192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x_corr</th>\n",
       "      <td>0.5324</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>act_corr</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.8304</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>act</th>\n",
       "      <th>noise</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-0.0141</td>\n",
       "      <td>0.0034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.0921</td>\n",
       "      <td>0.4870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x_corr</th>\n",
       "      <td>0.8855</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>act_corr</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.4282</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 0.75\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>act</th>\n",
       "      <th>noise</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-0.0015</td>\n",
       "      <td>0.0022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.0633</td>\n",
       "      <td>0.4240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x_corr</th>\n",
       "      <td>0.9100</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>act_corr</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.3899</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 0.5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>act</th>\n",
       "      <th>noise</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.0107</td>\n",
       "      <td>0.0008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.0558</td>\n",
       "      <td>0.3442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x_corr</th>\n",
       "      <td>0.9409</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>act_corr</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.3235</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 0.25\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>act</th>\n",
       "      <th>noise</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.0098</td>\n",
       "      <td>-0.0057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.0013</td>\n",
       "      <td>0.2461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x_corr</th>\n",
       "      <td>0.9667</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>act_corr</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.2298</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 0.1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>act</th>\n",
       "      <th>noise</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-0.0057</td>\n",
       "      <td>-0.0014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.9969</td>\n",
       "      <td>0.1533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>x_corr</th>\n",
       "      <td>0.9868</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>act_corr</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.1394</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for scale in [10, 1, .75, .5, .25, .1]:\n",
    "    print('\\n', scale)\n",
    "    simulate_activation_stats(scale, 1_000).pprint()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "class LinearSkipBlock(nn.Module):\n",
    "    \"\"\"This lets us easily create residual block equivalents with linear\n",
    "    layers.\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, x_dim, layer_dims, op, activation=mish):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        x_dim: int\n",
    "            Size of input tensor.\n",
    "        layer_dims: Iterable[int]\n",
    "            Size of each layer. The length of this list will be the skip size\n",
    "            (2 is probably a reasonable starting point).\n",
    "        op: function\n",
    "            This will be called on the input x and the processed x in the\n",
    "            forward method. This is a concatenation for dense blocks and an\n",
    "            addition for residual blocks, but any operation is possible.\n",
    "        activation: callable\n",
    "            Activation function or callable class. This will be applied after\n",
    "            each layer. The final activation is applied after the `op` function.\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.skip_size = len(layer_dims)\n",
    "        self.activation = activation\n",
    "        self.layers = nn.ModuleList([nn.Linear(d_in, d_out) for d_in, d_out\n",
    "                                     in zip([x_dim]+list(layer_dims), layer_dims)])\n",
    "        self.op = op\n",
    "            \n",
    "    def forward(self, x):\n",
    "        out = x\n",
    "        for i, layer in enumerate(self.layers, 1):\n",
    "            out = layer(out)\n",
    "            if i < self.skip_size: out = self.activation(out)\n",
    "        return self.activation(self.op(x, out))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "class LinearResBlock(LinearSkipBlock):\n",
    "    \"\"\"Equivalent of ResNet block with linear layers.\"\"\"\n",
    "    \n",
    "    def __init__(self, x_dim, hidden_dims, activation=mish):\n",
    "        if hidden_dims[-1] != x_dim:\n",
    "            raise InvalidArgumentError(\n",
    "                'Last hidden dimension must match input dimension.'\n",
    "            )\n",
    "        super().__init__(x_dim, hidden_dims, add, activation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "class LinearDenseBlock(LinearSkipBlock):\n",
    "    \"\"\"Equivalent of DenseNet block with linear layers.\"\"\"\n",
    "    \n",
    "    def __init__(self, x_dim, hidden_dims, activation=mish):\n",
    "        super().__init__(x_dim, hidden_dims, concat, activation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "class WeightedLinearResBlock(LinearSkipBlock):\n",
    "    \"\"\"Like a LinearResBlock but takes a weighted average of the input and output\n",
    "    rather than adding them. Addition gives them equal weight and we may want to \n",
    "    weight the output more heavily.\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, x_dim, hidden_dims, weights=(.25, .75), activation=mish):\n",
    "        super().__init__(x_dim, hidden_dims, \n",
    "                         partial(weighted_avg, weights=list(weights)), activation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embeddings and Encodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:38:31.441836Z",
     "start_time": "2020-09-29T21:38:31.404566Z"
    }
   },
   "outputs": [],
   "source": [
    "# export\n",
    "def trunc_normal_(x, mean=0.0, std=1.0):\n",
    "    \"\"\"Ported from fastai to remove dependency: \n",
    "    \n",
    "    Truncated normal initialization.\n",
    "    From https://discuss.pytorch.org/t/implementing-truncated-normal-initializer/4778/12\n",
    "    \"\"\"\n",
    "    return x.normal_().fmod_(2).mul_(std).add_(mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T22:02:06.933348Z",
     "start_time": "2020-09-29T22:02:06.901304Z"
    }
   },
   "outputs": [],
   "source": [
    "class InitializedEmbedding(nn.Embedding):\n",
    "    \"\"\"Same as nn.Embedding but with truncated normal initialization. This\n",
    "    also differs from fastai's Embedding class in that it allows padding.\"\"\"\n",
    "        \n",
    "    def reset_parameters(self):\n",
    "        with torch.no_grad():\n",
    "            trunc_normal_(self.weight, std=.01)\n",
    "            if self.padding_idx is not None:\n",
    "                torch.zero_(self.weight[self.padding_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T22:02:07.806683Z",
     "start_time": "2020-09-29T22:02:07.774348Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[ 0.0000,  0.0000,  0.0000],\n",
       "        [ 0.0007, -0.0141,  0.0099],\n",
       "        [-0.0017, -0.0158,  0.0135],\n",
       "        [-0.0027,  0.0038,  0.0052]], requires_grad=True)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "InitializedEmbedding(4, 3, 0).weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T22:02:08.746396Z",
     "start_time": "2020-09-29T22:02:08.699341Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[ 0.0038, -0.0103, -0.0074],\n",
       "        [ 0.0104, -0.0036,  0.0039],\n",
       "        [-0.0007,  0.0008,  0.0191],\n",
       "        [ 0.0000,  0.0000,  0.0000]], requires_grad=True)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "InitializedEmbedding(4, 3, 3).weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T22:02:08.746396Z",
     "start_time": "2020-09-29T22:02:08.699341Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[-0.0082,  0.0058, -0.0055],\n",
       "        [-0.0104, -0.0144,  0.0036],\n",
       "        [ 0.0028, -0.0020, -0.0009],\n",
       "        [ 0.0046, -0.0088, -0.0026]], requires_grad=True)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "InitializedEmbedding(4, 3).weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T21:59:56.122552Z",
     "start_time": "2020-09-29T21:59:56.077060Z"
    }
   },
   "outputs": [],
   "source": [
    "# export\n",
    "class BloomEmbedding(nn.Module):\n",
    "    \"\"\"Bloom Embedding layer for memory-efficient word representations.\n",
    "    Each word is encoded by a combination of rows of the embedding\n",
    "    matrix. The number of rows can therefore be far lower than the number\n",
    "    of words in our vocabulary while still providing unique representations.\n",
    "    The reduction in rows allows us to use memory in other ways: a larger\n",
    "    embedding dimension, more or larger layers after the embedding,\n",
    "    larger batch sizes, etc.\n",
    "    \n",
    "    Note that if hashing is done in the Dataset, we could use a simple\n",
    "    nn.EmbeddingBag to achieve the same thing. Many users have reported \n",
    "    poor performance with this layer though (especially on CPU, but in some\n",
    "    cases on GPU) so I stick with the standard Embedding. We also bake in\n",
    "    the truncated normal intialization provided by fastai, with a slight tweak\n",
    "    to allow a row for padding.\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, n_emb=251, emb_dim=100, n_hashes=4, padding_idx=0,\n",
    "                 pre_hashed=False):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        n_emb: int\n",
    "            Number of rows to create in the embedding matrix. A prime\n",
    "            number is recommended. Lower numbers will be more \n",
    "            memory-efficient but increase the chances of collisions.\n",
    "        emb_dim: int\n",
    "            Size of each embedding. If emb_dim=100, each word will\n",
    "            be represented by a 100-dimensional vector.\n",
    "        n_hashes: int\n",
    "            This determines the number of hashes that will be taken\n",
    "            for each word index, and as a result, the number of rows\n",
    "            that will be summed to create each unique representation.\n",
    "            The higher the number, the lower the chances of a collision.\n",
    "        padding_idx: int or None\n",
    "            If an integer is provided, this will set aside the corresponding\n",
    "            row in the embedding matrix as a vector of zeros. If None, no\n",
    "            padding vector will be allocated.\n",
    "        pre_hashed: bool\n",
    "            Pass in True if the input tensor will already be hashed by the time \n",
    "            it enters this layer (you may prefer pre-compute the hashes in the\n",
    "            Dataset to save computation time during training). In this\n",
    "            scenario, the layer is a simple embedding bag with mode \"sum\". \n",
    "            Pass in False if the inputs will be word indices that have not yet\n",
    "            been hashed. In this case, hashing will be done inside the \n",
    "            `forward` call.\n",
    "            \n",
    "        Suggested values for a vocab size of ~30,000:\n",
    "        \n",
    "        | n_emb | n_hashes | unique combos |\n",
    "        |-------|----------|---------------|\n",
    "        | 127   | 5        | 29,998        |\n",
    "        | 251   | 4        | 29,996        |\n",
    "        | 997   | 3        | 29,997        |\n",
    "        | 5,003 | 2        | 29,969        |\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.n_emb = n_emb\n",
    "        self.emb = InitializedEmbedding(n_emb, emb_dim, padding_idx)\n",
    "        self.n_hashes = n_hashes\n",
    "        self.pad_idx = padding_idx\n",
    "        self.pre_hashed = pre_hashed\n",
    "        self.process_fn = identity if pre_hashed else \\\n",
    "            partial(probabilistic_hash_tensor, n_buckets=n_emb,\n",
    "                    n_hashes=n_hashes, pad_idx=pad_idx)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        x: torch.LongTensor\n",
    "            Input tensor of word indices (bs x seq_len) if pre_hashed is False.\n",
    "            Hashed indices (bs x seq_len x n_hashes) if pre_hashed is False.\n",
    "            \n",
    "        Returns\n",
    "        -------\n",
    "        torch.FloatTensor: Words encoded with combination of embeddings.\n",
    "            (bs x seq_len x emb_dim)\n",
    "        \"\"\"\n",
    "        # If not pre-hashed: (bs, seq_len) -> hash -> (bs, seq_len, n_hashes)\n",
    "        hashed = self.process_fn(x)\n",
    "        # (bs, seq_len, n_hashes, emb_dim) -> sum -> (bs, seq_len, emb_dim)\n",
    "        return self.emb(hashed).sum(-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Data(Dataset):\n",
    "    \n",
    "    def __init__(self, sentences, labels, seq_len):\n",
    "        x = [s.split(' ') for s in sentences]\n",
    "        self.w2i = self.make_w2i(x)\n",
    "        self.seq_len = seq_len\n",
    "        self.x = self.encode(x)\n",
    "        self.y = torch.tensor(labels)\n",
    "        \n",
    "    def __getitem__(self, i):\n",
    "        return self.x[i], self.y[i]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.y)\n",
    "    \n",
    "    def make_w2i(self, tok_rows):\n",
    "        return {k: i for i, (k, v) in \n",
    "                enumerate(Counter(chain(*tok_rows)).most_common(), 1)}\n",
    "    \n",
    "    def encode(self, tok_rows):\n",
    "        enc = np.zeros((len(tok_rows), self.seq_len), dtype=int)\n",
    "        for i, row in enumerate(tok_rows):\n",
    "            trunc = [self.w2i.get(w, 0) for w in row[:self.seq_len]]\n",
    "            enc[i, :len(trunc)] = trunc\n",
    "        return torch.tensor(enc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sents = [\n",
    "    'I walked to the store so I hope it is not closed.',\n",
    "    'The theater is closed today and the sky is grey.',\n",
    "    'His dog is brown while hers is grey.'\n",
    "]\n",
    "labels = [0, 1, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([13, 14,  1, 15, 16, 17,  3, 18,  1,  4]), tensor(1))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds = Data(sents, labels, 10)\n",
    "ds[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 2,  5,  6,  3,  7,  8,  2,  9, 10,  1],\n",
       "         [13, 14,  1, 15, 16, 17,  3, 18,  1,  4],\n",
       "         [19, 20,  1, 21, 22, 23,  1,  4,  0,  0]]), tensor([0, 1, 1]))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dl = DataLoader(ds, batch_size=3)\n",
    "x, y = next(iter(dl))\n",
    "x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 2,  5,  6,  3,  7,  8,  2,  9, 10,  1],\n",
       "         [13, 14,  1, 15, 16, 17,  3, 18,  1,  4],\n",
       "         [19, 20,  1, 21, 22, 23,  1,  4,  0,  0]]), tensor([0, 1, 1]))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x, y = next(iter(dl))\n",
    "x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[ 0.0000,  0.0000,  0.0000,  0.0000],\n",
       "        [ 0.0054, -0.0142, -0.0027, -0.0052],\n",
       "        [ 0.0097,  0.0003, -0.0098, -0.0082],\n",
       "        [ 0.0049,  0.0084,  0.0028, -0.0158],\n",
       "        [ 0.0035,  0.0056, -0.0106, -0.0135],\n",
       "        [ 0.0088, -0.0058,  0.0105, -0.0072],\n",
       "        [-0.0003, -0.0004, -0.0012,  0.0142],\n",
       "        [ 0.0089, -0.0114, -0.0001,  0.0037],\n",
       "        [ 0.0089,  0.0007,  0.0076,  0.0034],\n",
       "        [-0.0039,  0.0119,  0.0090, -0.0077],\n",
       "        [ 0.0040, -0.0154,  0.0072, -0.0024]], requires_grad=True)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "be = BloomEmbedding(11, 4)\n",
    "be.emb.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 2,  5,  6,  3,  7,  8,  2,  9, 10,  1],\n",
       "        [13, 14,  1, 15, 16, 17,  3, 18,  1,  4],\n",
       "        [19, 20,  1, 21, 22, 23,  1,  4,  0,  0]])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 10, 4])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# (bs x seq_len) -> (bs -> seq_len -> emb_size)\n",
    "y = be(x)\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0363, -0.0098,  0.0052,  0.0023],\n",
       "        [ 0.0337, -0.0130, -0.0147, -0.0183],\n",
       "        [ 0.0075, -0.0316,  0.0120,  0.0236],\n",
       "        [ 0.0304, -0.0328,  0.0388, -0.0240],\n",
       "        [ 0.0144,  0.0004, -0.0022,  0.0019],\n",
       "        [ 0.0084,  0.0117,  0.0089, -0.0284],\n",
       "        [ 0.0363, -0.0098,  0.0052,  0.0023],\n",
       "        [ 0.0177, -0.0087,  0.0344, -0.0139],\n",
       "        [ 0.0272, -0.0108, -0.0036,  0.0130],\n",
       "        [ 0.0035, -0.0162,  0.0048,  0.0260]], grad_fn=<SelectBackward>)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below, we show by step how to get from x to y. This is meant to demonstrate the basic mechanism, not to show how PyTorch actually implements this under the hood. Let's look at a single row of x, corresponding to 1 sentence where each word is mapped to its index in the vocabulary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 2,  5,  6,  3,  7,  8,  2,  9, 10,  1])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we hash each item."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[8, 2, 7, 8],\n",
       " [2, 8, 1, 2],\n",
       " [6, 6, 10, 10],\n",
       " [10, 5, 5, 5],\n",
       " [6, 9, 7, 2],\n",
       " [5, 9, 4, 0],\n",
       " [8, 2, 7, 8],\n",
       " [5, 10, 8, 9],\n",
       " [7, 8, 6, 2],\n",
       " [6, 10, 6, 0]]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hashed = [probabilistic_hash_item(i.item(), 11, int, 4) for i in x[0]]\n",
    "hashed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then use each row of hashed integers to index into the embedding weight matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([10, 4, 4])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.0089,  0.0007,  0.0076,  0.0034],\n",
       "         [ 0.0097,  0.0003, -0.0098, -0.0082],\n",
       "         [ 0.0089, -0.0114, -0.0001,  0.0037],\n",
       "         [ 0.0089,  0.0007,  0.0076,  0.0034]],\n",
       "\n",
       "        [[ 0.0097,  0.0003, -0.0098, -0.0082],\n",
       "         [ 0.0089,  0.0007,  0.0076,  0.0034],\n",
       "         [ 0.0054, -0.0142, -0.0027, -0.0052],\n",
       "         [ 0.0097,  0.0003, -0.0098, -0.0082]]], grad_fn=<SliceBackward>)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output = []\n",
    "for row in hashed:\n",
    "    row_out = be.emb.weight[row]\n",
    "    output.append(row_out)\n",
    "output = torch.stack(output)\n",
    "print(output.shape)\n",
    "output[:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, we sum up the embedding rows. Above, each word is represented by four rows of the embedding matrix. After summing, we get a single vector for each word."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0363, -0.0098,  0.0052,  0.0023],\n",
       "        [ 0.0337, -0.0130, -0.0147, -0.0183],\n",
       "        [ 0.0075, -0.0316,  0.0120,  0.0236],\n",
       "        [ 0.0304, -0.0328,  0.0388, -0.0240],\n",
       "        [ 0.0144,  0.0004, -0.0022,  0.0019],\n",
       "        [ 0.0084,  0.0117,  0.0089, -0.0284],\n",
       "        [ 0.0363, -0.0098,  0.0052,  0.0023],\n",
       "        [ 0.0177, -0.0087,  0.0344, -0.0139],\n",
       "        [ 0.0272, -0.0108, -0.0036,  0.0130],\n",
       "        [ 0.0035, -0.0162,  0.0048,  0.0260]], grad_fn=<SumBackward1>)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output = output.sum(-2)\n",
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " Notice that the values now match the output of our embedding layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert torch.isclose(output, y[0]).all()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Axial encodings are intended to work as positional embeddings for transformer-like architectures. It's possible they could work for word embeddings as well, similar to our use of Bloom embeddings. However, the standard version of axial encodings results in similar vectors for adjacent indices - this makes some sense for positional indices, but for word indices it might require some additional preprocessing. For example, we could compress word embeddings down to 1 dimension and sort them, or simply sort by number of occurrences in our corpus which could be considered to be doing the same thing. Large chunks of the outputs vectors will be shared among different inputs, whereas Bloom embeddings seem like they would have a greater capacity to avoid this issue."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T22:00:03.487055Z",
     "start_time": "2020-09-29T22:00:03.452331Z"
    }
   },
   "outputs": [],
   "source": [
    "# export\n",
    "class AxialEncoding(nn.Module):\n",
    "    \"\"\"Axial encodings. These are intended to encode position in a sequence\n",
    "    (e.g. index in a sentence). It's possible we could adapt these for use as\n",
    "    word embeddings but this would likely require some experimentation (for\n",
    "    example, words would likely need to be sorted in a thoughtful manner \n",
    "    (e.g. pre-trained embeddings compressed to 1D?) since adjacent inputs will\n",
    "    share half of their encodings).\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, vocab_dim, emb_dim, pad_idx=None):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        vocab_dim: int\n",
    "            Number of words in vocab (or max sequence length if being used for\n",
    "            positional encodings).\n",
    "        emb_dim: int\n",
    "            Size of embedding vectors (often numbers like 50, 100, 300).\n",
    "        pad_idx: int or None\n",
    "            If necessary, pass in an integer to represent padding. Otherwise\n",
    "            no rows are reserved for padding.\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        if emb_dim % 2 != 0:\n",
    "            raise ValueError('emb_dim must be an even number.')\n",
    "            \n",
    "        self.v = self._decompose_mult(vocab_dim)\n",
    "        self.e = self._decompose_add(emb_dim)\n",
    "        self.emb = nn.ModuleList(InitializedEmbedding(self.v, self.e, pad_idx) \n",
    "                                 for _ in range(2))\n",
    "    \n",
    "    def _decompose_mult(self, dim):\n",
    "        return int(np.ceil(np.sqrt(dim)))\n",
    "    \n",
    "    def _decompose_add(self, dim):\n",
    "        return int(np.ceil(dim / 2))\n",
    "    \n",
    "    def forward(self, idx):\n",
    "        return torch.cat([self.emb[0](idx%self.v), self.emb[1](idx//self.v)], \n",
    "                         dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-29T22:00:20.818511Z",
     "start_time": "2020-09-29T22:00:20.771943Z"
    }
   },
   "outputs": [],
   "source": [
    "# export\n",
    "class MultiAxialEncoding(nn.Module):\n",
    "    \"\"\"Adapted axial encodings to allow for more than 2 embedding matrices.\n",
    "    These are intended to encode position in a sequence (e.g. index in a \n",
    "    sentence) but might work as word embeddings. This version may be better \n",
    "    suited for that use case because using more blocks results in fewer shared\n",
    "    numbers in the output vectors of adjacent inputs.\n",
    "    \n",
    "    Some experimentation is still required for this use case (for\n",
    "    example, words would likely need to be sorted in a thoughtful manner \n",
    "    (e.g. pre-trained embeddings compressed to 1D?) since adjacent inputs will\n",
    "    share half of their encodings).\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, vocab_dim, emb_dim, n_blocks=2, pre_hashed=False, \n",
    "                 pad_idx=None):\n",
    "        super().__init__()\n",
    "        # Must set n_blocks before computing v or e.\n",
    "        self.n_blocks = n_blocks\n",
    "        self.v = self._decompose_mult(vocab_dim)\n",
    "        self.e = self._decompose_add(emb_dim)\n",
    "        self.pre_hashed = pre_hashed\n",
    "        # Must set emb blocks before defining process_fn.\n",
    "        self.emb = nn.ModuleList(InitializedEmbedding(self.v, self.e, pad_idx) \n",
    "                                 for _ in range(n_blocks))\n",
    "        self.process_fn = identity if pre_hashed else \\\n",
    "            partial(probabilistic_hash_tensor, n_buckets=self.v, \n",
    "                    n_hashes=len(self.emb), pad_idx=pad_idx)\n",
    "        self.emb_dim = self.e * self.n_blocks\n",
    "    \n",
    "    def _decompose_mult(self, dim):\n",
    "        return int(np.ceil(dim ** (1 / self.n_blocks)))\n",
    "    \n",
    "    def _decompose_add(self, dim):\n",
    "        return int(np.ceil(dim // self.n_blocks))\n",
    "    \n",
    "    def forward(self, idx):\n",
    "        # Hashed shape: (bs, seq_len, n_hashes)\n",
    "        xhash = self.process_fn(idx)\n",
    "        # Each embedding takes in a tensor of shape (bs, seq_len).\n",
    "        res_blocks = [e(hashed.squeeze()) for e, hashed in \n",
    "                      zip(self.emb, torch.chunk(xhash, xhash.shape[0], -1))]\n",
    "        return torch.cat(res_blocks, dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reduction_ratio(ax, vocab_size, emb_dim):\n",
    "    \"\"\"For testing purposes. Lets us compare the number of weights in a\n",
    "    traditional embedding matrix vs. the number of weights in our axial\n",
    "    encoding.\n",
    "    \"\"\"\n",
    "    normal_n = vocab_size * emb_dim\n",
    "    ax_n = sum(e.weight.numel() for e in ax.emb)\n",
    "    print('Normal embedding weights:', normal_n)\n",
    "    print('Axial encoding weights:', ax_n)\n",
    "    print('Difference:', normal_n - ax_n)\n",
    "    print('Ratio:', normal_n / ax_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([12, 2])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "AxialEncoding(\n",
       "  (emb): ModuleList(\n",
       "    (0): Embedding(174, 50)\n",
       "    (1): Embedding(174, 50)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_size = 30_000\n",
    "emb_dim = 100\n",
    "bs = 12\n",
    "\n",
    "ax = AxialEncoding(vocab_size, emb_dim)\n",
    "x = torch.randint(0, vocab_size, (bs, 2))\n",
    "print(x.shape)\n",
    "ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([12, 2, 100])\n"
     ]
    }
   ],
   "source": [
    "res = ax(x)\n",
    "print(res.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Normal embedding weights: 3000000\n",
      "Axial encoding weights: 17400\n",
      "Difference: 2982600\n",
      "Ratio: 172.41379310344828\n"
     ]
    }
   ],
   "source": [
    "reduction_ratio(ax, vocab_size, emb_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([12, 2])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "MultiAxialEncoding(\n",
       "  (emb): ModuleList(\n",
       "    (0): Embedding(14, 25)\n",
       "    (1): Embedding(14, 25)\n",
       "    (2): Embedding(14, 25)\n",
       "    (3): Embedding(14, 25)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_size = 30_000\n",
    "emb_dim = 100\n",
    "bs = 12\n",
    "\n",
    "ax = MultiAxialEncoding(vocab_size, emb_dim, 4)\n",
    "x = torch.randint(0, vocab_size, (bs, 2))\n",
    "print(x.shape)\n",
    "ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([12, 2, 100])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res1 = ax(x)\n",
    "res1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultiAxialEncoding(\n",
       "  (emb): ModuleList(\n",
       "    (0): Embedding(14, 25)\n",
       "    (1): Embedding(14, 25)\n",
       "    (2): Embedding(14, 25)\n",
       "    (3): Embedding(14, 25)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_size = 30_000\n",
    "emb_dim = 100\n",
    "bs = 12\n",
    "\n",
    "ax_pre = MultiAxialEncoding(vocab_size, emb_dim, 4, pre_hashed=True)\n",
    "ax_pre"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By setting the weights of our pre-hashed embedding to the weights of our hashing embedding, we can check that the outputs are ultimately the same."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e, e_pre in zip(ax.emb, ax_pre.emb):\n",
    "    e_pre.weight.data = e.weight.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([12, 2, 100])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xhash = probabilistic_hash_tensor(x, 14, 4)\n",
    "res2 = ax_pre(xhash)\n",
    "res2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(res1 == res2).all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Normal embedding weights: 3000000\n",
      "Axial encoding weights: 1400\n",
      "Difference: 2998600\n",
      "Ratio: 2142.8571428571427\n"
     ]
    }
   ],
   "source": [
    "reduction_ratio(ax_pre, vocab_size, emb_dim)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I imagine that as we increase `n_blocks`, there's likely a point where we simply won't have enough weights to encode the amount of information that's present in the data. It would take some experimentation to find where that line is, however."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultiAxialEncoding(\n",
       "  (emb): ModuleList(\n",
       "    (0): Embedding(4, 12)\n",
       "    (1): Embedding(4, 12)\n",
       "    (2): Embedding(4, 12)\n",
       "    (3): Embedding(4, 12)\n",
       "    (4): Embedding(4, 12)\n",
       "    (5): Embedding(4, 12)\n",
       "    (6): Embedding(4, 12)\n",
       "    (7): Embedding(4, 12)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ax_large = MultiAxialEncoding(vocab_size, emb_dim, 8, pre_hashed=True)\n",
    "ax_large"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Normal embedding weights: 3000000\n",
      "Axial encoding weights: 384\n",
      "Difference: 2999616\n",
      "Ratio: 7812.5\n"
     ]
    }
   ],
   "source": [
    "reduction_ratio(ax_large, vocab_size, emb_dim)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
