{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Linear():\n",
    "    def forward(input):\n",
    "        return input\n",
    "\n",
    "    def diff(a):\n",
    "        dhda = a\n",
    "        return dhda \n",
    "\n",
    "class Tanh():\n",
    "    def forward(input):\n",
    "        return np.tanh(input)\n",
    "\n",
    "    def diff(a):\n",
    "        dhda = 1-np.tanh(a)**2\n",
    "        return dhda\n",
    "\n",
    "ActivationHandler = {\n",
    "    'linear' : Linear,\n",
    "    'tanh'   : Tanh,\n",
    "}\n",
    "\n",
    "def ActivationFunc(activation_func_name):\n",
    "    return ActivationHandler[activation_func_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Zeros(shape, dtype=None):\n",
    "    return np.zeros(shape=shape, dtype=dtype)\n",
    "\n",
    "def RandomNormal(shape):\n",
    "    return np.random.normal(loc=0, scale=0.05, size=shape)\n",
    "\n",
    "InitializeHandler = {\n",
    "    'zeros': Zeros,\n",
    "    'random_normal': RandomNormal,\n",
    "}\n",
    "\n",
    "def Initializer(initializer_name):\n",
    "    return InitializeHandler[initializer_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {},
   "outputs": [],
   "source": [
    "class mean_squared_error():\n",
    "    def loss(y_true, y_pred):\n",
    "        return np.mean(np.square(y_pred - y_true), axis=-1)\n",
    "    def diff(y_true, y_pred):\n",
    "        return y_pred-y_true\n",
    "\n",
    "LossHandler = {\n",
    "    'mean_squared_error' : mean_squared_error,\n",
    "}\n",
    "\n",
    "def LossFunc(loss_func_name):\n",
    "    return LossHandler[loss_func_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Layers():\n",
    "    NLayers=1\n",
    "\n",
    "    def __init__(self, name, units):\n",
    "        self.name=f\"Layer{Layers.NLayers}.{name}\"\n",
    "        self.outdim=units\n",
    "        self.w = None\n",
    "        Layers.NLayers+=1\n",
    "\n",
    "    def build(self, indim):\n",
    "        self.indim = indim\n",
    "        self.w = np.c_[\n",
    "            self.kernel_initializer(shape=(self.outdim,self.indim)),\n",
    "            self.bias_initializer(shape=(self.outdim,1))\n",
    "        ]\n",
    "\n",
    "class Input(Layers):\n",
    "    def __init__(self, inputdim):\n",
    "        super().__init__(\"inputs\", inputdim)\n",
    "\n",
    "class Dense(Layers):\n",
    "    def __init__(self, units, activation='linear', kernel_initializer='random_normal', bias_initializer='zeros'):\n",
    "        \"\"\"\n",
    "        @param units             : (tuple) dimensionality of the (input space, output space).\n",
    "        @param activation        : (str) Activation function to use.\n",
    "        @param kernel_initializer: (str) Initializer for the `kernel` weights matrix.\n",
    "        @param bias_initializer  : (str) Initializer for the bias vector.\n",
    "        \"\"\"        \n",
    "        super().__init__(\"dense\", units)\n",
    "        self.kernel_initializer = Initializer(kernel_initializer)\n",
    "        self.bias_initializer   = Initializer(bias_initializer)\n",
    "        self.h = ActivationFunc(activation)\n",
    "        self.z = None\n",
    "        self.a = None\n",
    "\n",
    "    def forward(self, input):\n",
    "        \"\"\" @param input: shape=(Din,) \"\"\"\n",
    "        z_in = np.append(input,1) # shape=(Din+1,)            \n",
    "        a = self.w.dot(z_in)      # (Dout,Din+1) @ (Din+1,) = (Dout,)\n",
    "        z_out = self.h.forward(a) # shape=(Dout,)\n",
    "        self.z = z_in\n",
    "        self.a = a\n",
    "        return z_out\n",
    "\n",
    "    def backprop(self, dEdz_out):\n",
    "        \"\"\" @param dEdz_out: shape=(Dout,) \"\"\"\n",
    "        dEda = self.h.diff(self.a)*dEdz_out # δ, shape=(Dout,)\n",
    "        dEdz_in = self.w.T.dot(dEda)        # (Din+1,Dout) @ (Dout,) = (Din+1,)\n",
    "        self.update(dEda)\n",
    "        return dEdz_in[:-1]                 # shape=(Din,) term of bias is not propagated.\n",
    "\n",
    "    def update(self, delta, ALPHA=1):\n",
    "        \"\"\" @param delta: shape=(Dout,) \"\"\"\n",
    "        dw = np.outer(delta, self.z) # (Dout,) × (Din+1,) = (Dout,Din+1)\n",
    "        self.w -= ALPHA*dw # update. w → w + ALPHA*dw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Sequential():\n",
    "    def __init__(self, layer=None):\n",
    "        self.layers = []\n",
    "        self.epochs = 0\n",
    "        if layer is not None: self.add(layer)\n",
    "        self.loss = None\n",
    "        self.config = None\n",
    "\n",
    "    def add(self, layer):\n",
    "        \"\"\"Adds a layer instance.\"\"\"\n",
    "        self.layers.append(layer)\n",
    "\n",
    "    def compile(self, loss, input_shape=None):\n",
    "        \"\"\" Creates the layer weights. \"\"\"\n",
    "        self.loss = LossFunc(loss)\n",
    "        units = [l.outdim for l in self.layers]\n",
    "        for i,l in enumerate(self.layers):\n",
    "            if l.name[-6:] == \"inputs\": continue\n",
    "            l.build(indim=units[i-1])\n",
    "\n",
    "    def fit(self, x_train, y_train, epochs=1000):\n",
    "        goal_epochs = self.epochs+epochs\n",
    "        digit=len(str(goal_epochs))\n",
    "        for e in range(epochs):\n",
    "            for x,y in zip(x_train,y_train):\n",
    "                out = self.forward(x)\n",
    "                self.backprop(y, out)\n",
    "            self.epochs+=1\n",
    "            y_pred = self.predict(x_train)\n",
    "            mse = np.mean((y_pred-y_train)**2)\n",
    "            if self.epochs % 100 == 99: print(f'[{self.epochs+1:{digit}d}/{goal_epochs:{digit}d}] mse={mse:{4}f}')\n",
    "\n",
    "    def forward(self, input):\n",
    "        out=input\n",
    "        for l in self.layers:\n",
    "            if l.name[-6:] == \"inputs\": continue\n",
    "            out=l.forward(out)            \n",
    "        return out\n",
    "\n",
    "    def backprop(self, y_true, out):\n",
    "        dEdz_out = self.loss.diff(y_true, out)\n",
    "        for l in reversed(self.layers):\n",
    "            if l.name[-6:] == \"inputs\": continue\n",
    "            dEdz_out = l.backprop(dEdz_out)\n",
    "\n",
    "    def predict(self, x_train):\n",
    "        if np.ndim(x_train) == 1: \n",
    "            return self.forward(x_train)\n",
    "        else:\n",
    "            return np.array([self.forward(x) for x in x_train])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential(Input(1))\n",
    "model.add(Dense(10, activation=\"tanh\", kernel_initializer=\"random_normal\", bias_initializer=\"zeros\"))\n",
    "model.add(Dense(10, activation=\"tanh\", kernel_initializer=\"random_normal\", bias_initializer=\"zeros\"))\n",
    "model.add(Dense(1, activation=\"tanh\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Units:   [1, 10, 10, 1]\n",
      "Layers:  ['Layer5.inputs', 'Layer6.dense', 'Layer7.dense', 'Layer8.dense']\n",
      "Weights: [None, None, None, None]\n",
      "Weights: [None, (10, 2), (10, 11), (1, 11)]\n"
     ]
    }
   ],
   "source": [
    "print(\"Units:   {}\".format([l.outdim for l in model.layers]))\n",
    "print(\"Layers:  {}\".format([l.name for l in model.layers]))\n",
    "print(\"Weights: {}\".format([l.w for l in model.layers]))\n",
    "model.compile(loss=\"mean_squared_error\")\n",
    "print(\"Weights: {}\".format([l.w.shape if l.w is not None else None for l in model.layers]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 1000\n",
    "func = lambda x:x**2\n",
    "X = np.linspace(-1, 1, N).reshape(-1,1)\n",
    "Y = np.vectorize(func)(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.array([\n",
    " [ 0.04294521,  0.        ],\n",
    " [-0.07202989,  0.        ],\n",
    " [-0.05457115,  0.        ],\n",
    "])\n",
    "\n",
    "b = np.array([\n",
    " [-0.02091271,  0.03322586, -0.02828983,  0.        ],\n",
    " [ 0.07530736, -0.00955775,  0.09933972,  0.        ],\n",
    " [-0.07459121, -0.01057377, -0.01888303,  0.        ],\n",
    "])\n",
    "\n",
    "c = np.array([[ 0.03243209, -0.04209234,  0.03517436,  0.        ]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = [a,b,c]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {},
   "outputs": [],
   "source": [
    "for layer,w in zip(model.layers[1:],tmp):\n",
    "    layer.w = w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, 2)\n",
      "(3, 4)\n",
      "(1, 4)\n"
     ]
    }
   ],
   "source": [
    "for layer in model.layers[1:]:\n",
    "    print(layer.w.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.04294521  0.        ]\n",
      " [-0.07202989  0.        ]\n",
      " [-0.05457115  0.        ]]\n",
      "[[-0.02091271  0.03322586 -0.02828983  0.        ]\n",
      " [ 0.07530736 -0.00955775  0.09933972  0.        ]\n",
      " [-0.07459121 -0.01057377 -0.01888303  0.        ]]\n",
      "[[ 0.03243209 -0.04209234  0.03517436  0.        ]]\n"
     ]
    }
   ],
   "source": [
    "for layer,w in zip(model.layers[1:],tmp):\n",
    "    print(layer.w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = np.arange(len(X))\n",
    "np.random.shuffle(idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = X[idx,:]\n",
    "Y = Y[idx,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x1170de470>"
      ]
     },
     "execution_count": 316,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 10876 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(X,Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[100/300] mse=0.173272\n",
      "[200/300] mse=0.173272\n",
      "[300/300] mse=0.173273\n"
     ]
    }
   ],
   "source": [
    "model.fit(X,Y,epochs=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred = model.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 14156 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(X, Y_pred, label=\"Neural Network\", color=\"red\")\n",
    "plt.scatter(X, Y, s=1, label=\"data\", color=\"blue\")\n",
    "plt.xlabel(\"x\")\n",
    "plt.ylabel(\"y\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.0068016   0.20578694]\n",
      " [-0.02798395  0.02394519]\n",
      " [-0.05357066  0.27076291]]\n",
      "[[-0.10808446  0.03842617 -0.12193693 -1.86998864]\n",
      " [ 0.08101969  0.02720275  0.1587066   2.1721532 ]\n",
      " [-0.14922808 -0.01347841 -0.10957356 -1.95819482]]\n",
      "[[ 2.68843758 -3.74570328  2.95286747  2.47395041]]\n"
     ]
    }
   ],
   "source": [
    "for l in model.layers[1:]:\n",
    "    print(l.w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kerasy.utils import make_batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_train_samples = N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "batches = make_batches(num_train_samples, 12)\n",
    "index_array = np.arange(num_train_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 18584 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for batch_index, (batch_start, batch_end) in enumerate(batches):\n",
    "    np.random.shuffle(index_array)\n",
    "    batch_ids = index_array[batch_start:batch_end]\n",
    "    for bs, (x_train, y_train) in enumerate(zip(X[batch_ids], Y[batch_ids])):\n",
    "        plt.scatter(x_train, model.forward(x_train))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.91391391] [0.83523864]\n",
      "[-0.0950951] [0.00904308]\n",
      "[-0.0970971] [0.00942785]\n",
      "[-0.1011011] [0.01022143]\n"
     ]
    }
   ],
   "source": [
    "for bs, (x_train, y_train) in enumerate(zip(X[batch_ids], Y[batch_ids])):\n",
    "    print(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16,\n",
       "       17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33,\n",
       "       34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50,\n",
       "       51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67,\n",
       "       68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84,\n",
       "       85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99])"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
