{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from skimage.transform import resize as imresize\n",
    "import cv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('jawi-to-malay-dataset.pkl', 'rb') as fopen:\n",
    "    dataset = pickle.load(fopen)\n",
    "    \n",
    "train_X = dataset['train_X']\n",
    "train_labels = dataset['train_Y']\n",
    "test_X = dataset['test_X']\n",
    "test_labels = dataset['test_Y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 18596 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(train_X[-1])\n",
    "plt.title(train_labels[-1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "charset = list(set(''.join(train_labels + test_labels)))\n",
    "num_classes = len(charset) + 2\n",
    "encode_maps = {}\n",
    "decode_maps = {}\n",
    "for i, char in enumerate(charset, 3):\n",
    "    encode_maps[char] = i\n",
    "    decode_maps[i] = char\n",
    "    \n",
    "SPACE_INDEX = 0\n",
    "SPACE_TOKEN = '<PAD>'\n",
    "encode_maps[SPACE_TOKEN] = SPACE_INDEX\n",
    "decode_maps[SPACE_INDEX] = SPACE_TOKEN\n",
    "\n",
    "GO_INDEX = 1\n",
    "GO_TOKEN = '<GO>'\n",
    "encode_maps[GO_TOKEN] = GO_INDEX\n",
    "decode_maps[GO_INDEX] = GO_TOKEN\n",
    "\n",
    "EOS_INDEX = 2\n",
    "EOS_TOKEN = '<EOS>'\n",
    "encode_maps[EOS_TOKEN] = EOS_INDEX\n",
    "decode_maps[EOS_INDEX] = EOS_TOKEN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{3: 'h',\n",
       " 4: 'q',\n",
       " 5: 'l',\n",
       " 6: 'r',\n",
       " 7: 'x',\n",
       " 8: 'u',\n",
       " 9: 'i',\n",
       " 10: 't',\n",
       " 11: 'p',\n",
       " 12: \"'\",\n",
       " 13: 'y',\n",
       " 14: 'c',\n",
       " 15: 'b',\n",
       " 16: 'm',\n",
       " 17: ' ',\n",
       " 18: 'n',\n",
       " 19: 'z',\n",
       " 20: 'a',\n",
       " 21: 's',\n",
       " 22: 'v',\n",
       " 23: 'o',\n",
       " 24: '?',\n",
       " 25: 'e',\n",
       " 26: 'w',\n",
       " 27: 'j',\n",
       " 28: 'g',\n",
       " 29: 'd',\n",
       " 30: 'k',\n",
       " 31: 'f',\n",
       " 32: '-',\n",
       " 0: '<PAD>',\n",
       " 1: '<GO>',\n",
       " 2: '<EOS>'}"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decode_maps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[6, 20, 10, 20, 11, 20, 18, 2]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[encode_maps[c] for c in train_labels[0]] + [2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = 1\n",
    "PAD = 0\n",
    "EOS = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_height = 60\n",
    "image_width = 240\n",
    "image_channel = 1\n",
    "max_stepsize = 128\n",
    "num_hidden = 256\n",
    "epoch = 20\n",
    "batch_size = 128\n",
    "initial_learning_rate = 1e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_Y = []\n",
    "for label in train_labels:\n",
    "    train_Y.append([encode_maps[c] for c in label] + [EOS])\n",
    "    \n",
    "test_Y = []\n",
    "for label in test_labels:\n",
    "    test_Y.append([encode_maps[c] for c in label] + [EOS])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://github.com/guillaumegenthial/im2latex/blob/master/model/components/attention_mechanism.py\n",
    "\n",
    "class AttentionMechanism(object):\n",
    "    \"\"\"Class to compute attention over an image\"\"\"\n",
    "\n",
    "    def __init__(self, img, dim_e, tiles=1):\n",
    "        \"\"\"Stores the image under the right shape.\n",
    "        We loose the H, W dimensions and merge them into a single\n",
    "        dimension that corresponds to \"regions\" of the image.\n",
    "        Args:\n",
    "            img: (tf.Tensor) image\n",
    "            dim_e: (int) dimension of the intermediary vector used to\n",
    "                compute attention\n",
    "            tiles: (int) default 1, input to context h may have size\n",
    "                    (tile * batch_size, ...)\n",
    "        \"\"\"\n",
    "        if len(img.shape) == 3:\n",
    "            self._img = img\n",
    "        elif len(img.shape) == 4:\n",
    "            N    = tf.shape(img)[0]\n",
    "            H, W = tf.shape(img)[1], tf.shape(img)[2] # image\n",
    "            C    = img.shape[3].value                 # channels\n",
    "            self._img = tf.reshape(img, shape=[N, H*W, C])\n",
    "        else:\n",
    "            print(\"Image shape not supported\")\n",
    "            raise NotImplementedError\n",
    "\n",
    "        # dimensions\n",
    "        self._n_regions  = tf.shape(self._img)[1]\n",
    "        self._n_channels = self._img.shape[2].value\n",
    "        self._dim_e      = dim_e\n",
    "        self._tiles      = tiles\n",
    "        self._scope_name = \"att_mechanism\"\n",
    "\n",
    "        # attention vector over the image\n",
    "        self._att_img = tf.layers.dense(\n",
    "            inputs=self._img,\n",
    "            units=self._dim_e,\n",
    "            use_bias=False,\n",
    "            name=\"att_img\")\n",
    "\n",
    "\n",
    "    def context(self, h):\n",
    "        \"\"\"Computes attention\n",
    "        Args:\n",
    "            h: (batch_size, num_units) hidden state\n",
    "        Returns:\n",
    "            c: (batch_size, channels) context vector\n",
    "        \"\"\"\n",
    "        with tf.variable_scope(self._scope_name):\n",
    "            if self._tiles > 1:\n",
    "                att_img = tf.expand_dims(self._att_img, axis=1)\n",
    "                att_img = tf.tile(att_img, multiples=[1, self._tiles, 1, 1])\n",
    "                att_img = tf.reshape(att_img, shape=[-1, self._n_regions,\n",
    "                        self._dim_e])\n",
    "                img = tf.expand_dims(self._img, axis=1)\n",
    "                img = tf.tile(img, multiples=[1, self._tiles, 1, 1])\n",
    "                img = tf.reshape(img, shape=[-1, self._n_regions,\n",
    "                        self._n_channels])\n",
    "            else:\n",
    "                att_img = self._att_img\n",
    "                img     = self._img\n",
    "\n",
    "            # computes attention over the hidden vector\n",
    "            att_h = tf.layers.dense(inputs=h, units=self._dim_e, use_bias=False)\n",
    "\n",
    "            # sums the two contributions\n",
    "            att_h = tf.expand_dims(att_h, axis=1)\n",
    "            att = tf.tanh(att_img + att_h)\n",
    "\n",
    "            # computes scalar product with beta vector\n",
    "            # works faster with a matmul than with a * and a tf.reduce_sum\n",
    "            att_beta = tf.get_variable(\"att_beta\", shape=[self._dim_e, 1],\n",
    "                    dtype=tf.float32)\n",
    "            att_flat = tf.reshape(att, shape=[-1, self._dim_e])\n",
    "            e = tf.matmul(att_flat, att_beta)\n",
    "            e = tf.reshape(e, shape=[-1, self._n_regions])\n",
    "\n",
    "            # compute weights\n",
    "            a = tf.nn.softmax(e)\n",
    "            a = tf.expand_dims(a, axis=-1)\n",
    "            c = tf.reduce_sum(a * img, axis=1)\n",
    "\n",
    "            return c\n",
    "\n",
    "\n",
    "    def initial_cell_state(self, cell):\n",
    "        \"\"\"Returns initial state of a cell computed from the image\n",
    "        Assumes cell.state_type is an instance of named_tuple.\n",
    "        Ex: LSTMStateTuple\n",
    "        Args:\n",
    "            cell: (instance of RNNCell) must define _state_size\n",
    "        \"\"\"\n",
    "        _states_0 = []\n",
    "        for hidden_name in cell._state_size._fields:\n",
    "            hidden_dim = getattr(cell._state_size, hidden_name)\n",
    "            h = self.initial_state(hidden_name, hidden_dim)\n",
    "            _states_0.append(h)\n",
    "\n",
    "        initial_state_cell = type(cell.state_size)(*_states_0)\n",
    "\n",
    "        return initial_state_cell\n",
    "\n",
    "\n",
    "    def initial_state(self, name, dim):\n",
    "        \"\"\"Returns initial state of dimension specified by dim\"\"\"\n",
    "        with tf.variable_scope(self._scope_name):\n",
    "            img_mean = tf.reduce_mean(self._img, axis=1)\n",
    "            W = tf.get_variable(\"W_{}_0\".format(name), shape=[self._n_channels,\n",
    "                    dim])\n",
    "            b = tf.get_variable(\"b_{}_0\".format(name), shape=[dim])\n",
    "            h = tf.tanh(tf.matmul(img_mean, W) + b)\n",
    "\n",
    "            return h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://github.com/guillaumegenthial/im2latex/blob/master/model/components/attention_cell.py\n",
    "\n",
    "import collections\n",
    "from tensorflow.contrib.rnn import RNNCell, LSTMStateTuple\n",
    "\n",
    "\n",
    "AttentionState = collections.namedtuple(\"AttentionState\", (\"cell_state\", \"o\"))\n",
    "\n",
    "\n",
    "class AttentionCell(RNNCell):\n",
    "    def __init__(self, cell, attention_mechanism, dropout, dim_e,\n",
    "                 dim_o, num_units,\n",
    "        num_proj, dtype=tf.float32):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            cell: (RNNCell)\n",
    "            attention_mechanism: (AttentionMechanism)\n",
    "            dropout: (tf.float)\n",
    "            attn_cell_config: (dict) hyper params\n",
    "        \"\"\"\n",
    "        # variables and tensors\n",
    "        self._cell                = cell\n",
    "        self._attention_mechanism = attention_mechanism\n",
    "        self._dropout             = dropout\n",
    "\n",
    "        # hyperparameters and shapes\n",
    "        self._n_channels     = self._attention_mechanism._n_channels\n",
    "        self._dim_e          = dim_e\n",
    "        self._dim_o          = dim_o\n",
    "        self._num_units      = num_units\n",
    "        self._num_proj       = num_proj\n",
    "        self._dtype          = dtype\n",
    "\n",
    "        # for RNNCell\n",
    "        self._state_size = AttentionState(self._cell._state_size, self._dim_o)\n",
    "\n",
    "\n",
    "    @property\n",
    "    def state_size(self):\n",
    "        return self._state_size\n",
    "\n",
    "\n",
    "    @property\n",
    "    def output_size(self):\n",
    "        return self._num_proj\n",
    "\n",
    "\n",
    "    @property\n",
    "    def output_dtype(self):\n",
    "        return self._dtype\n",
    "\n",
    "\n",
    "    def initial_state(self):\n",
    "        \"\"\"Returns initial state for the lstm\"\"\"\n",
    "        initial_cell_state = self._attention_mechanism.initial_cell_state(self._cell)\n",
    "        initial_o          = self._attention_mechanism.initial_state(\"o\", self._dim_o)\n",
    "\n",
    "        return AttentionState(initial_cell_state, initial_o)\n",
    "\n",
    "\n",
    "    def step(self, embedding, attn_cell_state):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            embedding: shape = (batch_size, dim_embeddings) embeddings\n",
    "                from previous time step\n",
    "            attn_cell_state: (AttentionState) state from previous time step\n",
    "        \"\"\"\n",
    "        prev_cell_state, o = attn_cell_state\n",
    "\n",
    "        scope = tf.get_variable_scope()\n",
    "        with tf.variable_scope(scope):\n",
    "            # compute new h\n",
    "            x                     = tf.concat([embedding, o], axis=-1)\n",
    "            new_h, new_cell_state = self._cell.__call__(x, prev_cell_state)\n",
    "            new_h = tf.nn.dropout(new_h, self._dropout)\n",
    "\n",
    "            # compute attention\n",
    "            c = self._attention_mechanism.context(new_h)\n",
    "\n",
    "            # compute o\n",
    "            o_W_c = tf.get_variable(\"o_W_c\", dtype=tf.float32,\n",
    "                    shape=(self._n_channels, self._dim_o))\n",
    "            o_W_h = tf.get_variable(\"o_W_h\", dtype=tf.float32,\n",
    "                    shape=(self._num_units, self._dim_o))\n",
    "\n",
    "            new_o = tf.tanh(tf.matmul(new_h, o_W_h) + tf.matmul(c, o_W_c))\n",
    "            new_o = tf.nn.dropout(new_o, self._dropout)\n",
    "\n",
    "            y_W_o = tf.get_variable(\"y_W_o\", dtype=tf.float32,\n",
    "                    shape=(self._dim_o, self._num_proj))\n",
    "            logits = tf.matmul(new_o, y_W_o)\n",
    "\n",
    "            # new Attn cell state\n",
    "            new_state = AttentionState(new_cell_state, new_o)\n",
    "\n",
    "            return logits, new_state\n",
    "\n",
    "\n",
    "    def __call__(self, inputs, state):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            inputs: the embedding of the previous word for training only\n",
    "            state: (AttentionState) (h, o) where h is the hidden state and\n",
    "                o is the vector used to make the prediction of\n",
    "                the previous word\n",
    "        \"\"\"\n",
    "        new_output, new_state = self.step(inputs, state)\n",
    "\n",
    "        return (new_output, new_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "import math\n",
    "import numpy as np\n",
    "from six.moves import xrange\n",
    "import tensorflow as tf\n",
    "\n",
    "\n",
    "# taken from https://github.com/tensorflow/tensor2tensor/blob/37465a1759e278e8f073cd04cd9b4fe377d3c740/tensor2tensor/layers/common_attention.py\n",
    "\n",
    "# taken from https://raw.githubusercontent.com/guillaumegenthial/im2latex/master/model/components/positional.py\n",
    "\n",
    "def add_timing_signal_nd(x, min_timescale=1.0, max_timescale=1.0e4):\n",
    "    \"\"\"Adds a bunch of sinusoids of different frequencies to a Tensor.\n",
    "\n",
    "    Each channel of the input Tensor is incremented by a sinusoid of a difft\n",
    "    frequency and phase in one of the positional dimensions.\n",
    "\n",
    "    This allows attention to learn to use absolute and relative positions.\n",
    "    Timing signals should be added to some precursors of both the query and the\n",
    "    memory inputs to attention.\n",
    "\n",
    "    The use of relative position is possible because sin(a+b) and cos(a+b) can\n",
    "    be experessed in terms of b, sin(a) and cos(a).\n",
    "\n",
    "    x is a Tensor with n \"positional\" dimensions, e.g. one dimension for a\n",
    "    sequence or two dimensions for an image\n",
    "\n",
    "    We use a geometric sequence of timescales starting with\n",
    "    min_timescale and ending with max_timescale.  The number of different\n",
    "    timescales is equal to channels // (n * 2). For each timescale, we\n",
    "    generate the two sinusoidal signals sin(timestep/timescale) and\n",
    "    cos(timestep/timescale).  All of these sinusoids are concatenated in\n",
    "    the channels dimension.\n",
    "\n",
    "    Args:\n",
    "        x: a Tensor with shape [batch, d1 ... dn, channels]\n",
    "        min_timescale: a float\n",
    "        max_timescale: a float\n",
    "\n",
    "    Returns:\n",
    "        a Tensor the same shape as x.\n",
    "\n",
    "    \"\"\"\n",
    "    static_shape = x.get_shape().as_list()\n",
    "    num_dims = len(static_shape) - 2\n",
    "    channels = tf.shape(x)[-1]\n",
    "    num_timescales = channels // (num_dims * 2)\n",
    "    log_timescale_increment = (\n",
    "            math.log(float(max_timescale) / float(min_timescale)) /\n",
    "            (tf.to_float(num_timescales) - 1))\n",
    "    inv_timescales = min_timescale * tf.exp(\n",
    "            tf.to_float(tf.range(num_timescales)) * -log_timescale_increment)\n",
    "    for dim in xrange(num_dims):\n",
    "        length = tf.shape(x)[dim + 1]\n",
    "        position = tf.to_float(tf.range(length))\n",
    "        scaled_time = tf.expand_dims(position, 1) * tf.expand_dims(\n",
    "                inv_timescales, 0)\n",
    "        signal = tf.concat([tf.sin(scaled_time), tf.cos(scaled_time)], axis=1)\n",
    "        prepad = dim * 2 * num_timescales\n",
    "        postpad = channels - (dim + 1) * 2 * num_timescales\n",
    "        signal = tf.pad(signal, [[0, 0], [prepad, postpad]])\n",
    "        for _ in xrange(1 + dim):\n",
    "            signal = tf.expand_dims(signal, 0)\n",
    "        for _ in xrange(num_dims - 1 - dim):\n",
    "            signal = tf.expand_dims(signal, -2)\n",
    "        x += signal\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "attention_size = 256\n",
    "size_layer = 512\n",
    "embedded_size = 256\n",
    "beam_width = 15\n",
    "learning_rate = 1e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CNN part I took from https://github.com/guillaumegenthial/im2latex/blob/master/model/encoder.py\n",
    "# I use tf.contrib.seq2seq as decoder part\n",
    "\n",
    "class Model:\n",
    "    def __init__(self):\n",
    "        self.X = tf.placeholder(tf.float32, shape=(None, 60, 240, 1))\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        x_len = tf.shape(self.X)[2] // 2\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        \n",
    "        decoder_embeddings = tf.Variable(tf.random_uniform([len(encode_maps), embedded_size], -1, 1))\n",
    "        \n",
    "        img = self.X\n",
    "        \n",
    "        out = tf.layers.conv2d(img, 64, 3, 1, \"SAME\",\n",
    "                activation=tf.nn.relu)\n",
    "        out = tf.layers.max_pooling2d(out, 2, 2, \"SAME\")\n",
    "\n",
    "        out = tf.layers.conv2d(out, 128, 3, 1, \"SAME\",\n",
    "                activation=tf.nn.relu)\n",
    "        out = tf.layers.max_pooling2d(out, 2, 2, \"SAME\")\n",
    "\n",
    "        out = tf.layers.conv2d(out, 256, 3, 1, \"SAME\",\n",
    "                activation=tf.nn.relu)\n",
    "\n",
    "        out = tf.layers.conv2d(out, 256, 3, 1, \"SAME\",\n",
    "                activation=tf.nn.relu)\n",
    "        out = tf.layers.max_pooling2d(out, (2, 1), (2, 1), \"SAME\")\n",
    "        out = tf.layers.conv2d(out, 512, 3, 1, \"SAME\",\n",
    "                activation=tf.nn.relu)\n",
    "        out = tf.layers.max_pooling2d(out, (1, 2), (1, 2), \"SAME\")\n",
    "        out = tf.layers.conv2d(out, 512, 3, 1, \"VALID\",\n",
    "                activation=tf.nn.relu)\n",
    "        img = add_timing_signal_nd(out)\n",
    "        print(img)\n",
    "        \n",
    "        with tf.variable_scope(\"attn_cell\", reuse=False):\n",
    "            attn_meca = AttentionMechanism(img, attention_size)\n",
    "            recu_cell = tf.nn.rnn_cell.LSTMCell(size_layer)\n",
    "            attn_cell = AttentionCell(recu_cell, attn_meca, 1.0,\n",
    "                        attention_size, attention_size, size_layer, len(encode_maps))\n",
    "\n",
    "            encoder_state = attn_cell.initial_state()\n",
    "\n",
    "            training_helper = tf.contrib.seq2seq.ScheduledEmbeddingTrainingHelper(\n",
    "                    inputs = tf.nn.embedding_lookup(decoder_embeddings, decoder_input),\n",
    "                    sequence_length = self.Y_seq_len,\n",
    "                    embedding = decoder_embeddings,\n",
    "                    sampling_probability = 0.5,\n",
    "                    time_major = False)\n",
    "            training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                    cell = attn_cell,\n",
    "                    helper = training_helper,\n",
    "                    initial_state = encoder_state,\n",
    "                    output_layer = None)\n",
    "            training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                    decoder = training_decoder,\n",
    "                    impute_finished = True,\n",
    "                    maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "        \n",
    "        with tf.variable_scope(\"attn_cell\", reuse=True):\n",
    "            attn_meca = AttentionMechanism(img, attention_size, tiles=beam_width)\n",
    "            recu_cell = tf.nn.rnn_cell.LSTMCell(size_layer, reuse = True)\n",
    "            attn_cell = AttentionCell(recu_cell, attn_meca, 1.0,\n",
    "                        attention_size, attention_size, size_layer, len(encode_maps))\n",
    "            \n",
    "            encoder_state = attn_cell.initial_state()\n",
    "            \n",
    "            predicting_decoder = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "                cell = attn_cell,\n",
    "                embedding = decoder_embeddings,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS,\n",
    "                initial_state = tf.contrib.seq2seq.tile_batch(encoder_state, beam_width),\n",
    "                beam_width = beam_width,\n",
    "                output_layer = None,\n",
    "                length_penalty_weight = 0.0)\n",
    "            predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = predicting_decoder,\n",
    "                impute_finished = False,\n",
    "                maximum_iterations = x_len)\n",
    "            \n",
    "        self.training_logits = training_decoder_output.rnn_output\n",
    "        self.predicting_ids = predicting_decoder_output.predicted_ids\n",
    "        self.predicting_ids = tf.identity(self.predicting_ids, name = 'logits')\n",
    "        \n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/client/session.py:1735: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"add_1:0\", shape=(?, 6, 28, 512), dtype=float32)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/gradients_util.py:93: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Variable',\n",
       " 'conv2d/kernel',\n",
       " 'conv2d/bias',\n",
       " 'conv2d_1/kernel',\n",
       " 'conv2d_1/bias',\n",
       " 'conv2d_2/kernel',\n",
       " 'conv2d_2/bias',\n",
       " 'conv2d_3/kernel',\n",
       " 'conv2d_3/bias',\n",
       " 'conv2d_4/kernel',\n",
       " 'conv2d_4/bias',\n",
       " 'conv2d_5/kernel',\n",
       " 'conv2d_5/bias',\n",
       " 'attn_cell/att_img/kernel',\n",
       " 'attn_cell/att_mechanism/W_c_0',\n",
       " 'attn_cell/att_mechanism/b_c_0',\n",
       " 'attn_cell/att_mechanism/W_h_0',\n",
       " 'attn_cell/att_mechanism/b_h_0',\n",
       " 'attn_cell/att_mechanism/W_o_0',\n",
       " 'attn_cell/att_mechanism/b_o_0',\n",
       " 'attn_cell/decoder/lstm_cell/kernel',\n",
       " 'attn_cell/decoder/lstm_cell/bias',\n",
       " 'attn_cell/decoder/att_mechanism/dense/kernel',\n",
       " 'attn_cell/decoder/o_W_c',\n",
       " 'attn_cell/decoder/o_W_h',\n",
       " 'attn_cell/decoder/y_W_o',\n",
       " 'logits']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'adam' not in n.name\n",
    "        and 'Adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'im2latex/model.ckpt'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'im2latex/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from im2latex/model.ckpt\n",
      "INFO:tensorflow:Froze 27 variables.\n",
      "INFO:tensorflow:Converted 27 variables to const ops.\n",
      "805 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('im2latex', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'attn_cell/decoder/transpose:0' shape=(?, ?, 33) dtype=float32>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.training_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3.68607, 0.0)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_x = train_X[:5]\n",
    "batch_x = np.array(batch_x).reshape((len(batch_x), image_height, image_width,image_channel))\n",
    "y = train_Y[:5]\n",
    "batch_y, _ = pad_sentence_batch(y, 0)\n",
    "loss, logits, acc = sess.run([model.cost, model.training_logits, model.accuracy], feed_dict = {model.X: batch_x,\n",
    "                                                          model.Y: batch_y})\n",
    "loss, acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 120, 15)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_x = train_X[:5]\n",
    "batch_x = np.array(batch_x).reshape((len(batch_x), image_height, image_width,image_channel))\n",
    "y = train_Y[:5]\n",
    "batch_y, _ = pad_sentence_batch(y, 0)\n",
    "logits = sess.run(model.predicting_ids, feed_dict = {model.X: batch_x})\n",
    "logits.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 507/507 [01:32<00:00,  5.51it/s, accuracy=0.972, cost=0.0901]\n",
      "minibatch loop: 100%|██████████| 127/127 [00:10<00:00, 12.59it/s, accuracy=0.878, cost=0.437]\n",
      "minibatch loop:   0%|          | 1/507 [00:00<01:29,  5.68it/s, accuracy=0.949, cost=0.156]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, training avg loss 0.152301, training avg acc 0.951534\n",
      "epoch 1, testing avg loss 0.412375, testing avg acc 0.886601\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 507/507 [01:31<00:00,  5.52it/s, accuracy=0.973, cost=0.0828]\n",
      "minibatch loop: 100%|██████████| 127/127 [00:09<00:00, 12.75it/s, accuracy=0.882, cost=0.463]\n",
      "minibatch loop:   0%|          | 1/507 [00:00<01:28,  5.69it/s, accuracy=0.949, cost=0.149]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 2, training avg loss 0.147481, training avg acc 0.953340\n",
      "epoch 2, testing avg loss 0.399478, testing avg acc 0.887377\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 507/507 [01:31<00:00,  5.51it/s, accuracy=0.954, cost=0.163] \n",
      "minibatch loop: 100%|██████████| 127/127 [00:10<00:00, 12.61it/s, accuracy=0.885, cost=0.439]\n",
      "minibatch loop:   0%|          | 1/507 [00:00<01:27,  5.76it/s, accuracy=0.944, cost=0.185]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 3, training avg loss 0.139094, training avg acc 0.955725\n",
      "epoch 3, testing avg loss 0.450588, testing avg acc 0.873517\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 507/507 [01:31<00:00,  5.52it/s, accuracy=0.974, cost=0.0827]\n",
      "minibatch loop: 100%|██████████| 127/127 [00:10<00:00, 12.55it/s, accuracy=0.872, cost=0.46] \n",
      "minibatch loop:   0%|          | 1/507 [00:00<01:30,  5.57it/s, accuracy=0.942, cost=0.179]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 4, training avg loss 0.133433, training avg acc 0.957887\n",
      "epoch 4, testing avg loss 0.446984, testing avg acc 0.876499\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 507/507 [01:31<00:00,  5.51it/s, accuracy=0.982, cost=0.0629]\n",
      "minibatch loop: 100%|██████████| 127/127 [00:10<00:00, 12.58it/s, accuracy=0.897, cost=0.451]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 5, training avg loss 0.125402, training avg acc 0.960689\n",
      "epoch 5, testing avg loss 0.405598, testing avg acc 0.888860\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import tqdm\n",
    "\n",
    "for e in range(5):\n",
    "    pbar = tqdm.tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'minibatch loop')\n",
    "    train_loss, train_acc, test_loss, test_acc = [], [], [], []\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i : index]\n",
    "        batch_x = np.array(batch_x).reshape((len(batch_x), image_height, image_width,image_channel))\n",
    "        y = train_Y[i : index]\n",
    "        batch_y, _ = pad_sentence_batch(y, 0)\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y}\n",
    "        accuracy, loss, _ = sess.run([model.accuracy,model.cost,model.optimizer],\n",
    "                                    feed_dict = feed)\n",
    "        train_loss.append(loss)\n",
    "        train_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    \n",
    "    pbar = tqdm.tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'minibatch loop')\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i : index]\n",
    "        batch_x = np.array(batch_x).reshape((len(batch_x), image_height, image_width,image_channel))\n",
    "        y = test_Y[i : index]\n",
    "        batch_y, _ = pad_sentence_batch(y, 0)\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y,}\n",
    "        accuracy, loss = sess.run([model.accuracy,model.cost],\n",
    "                                    feed_dict = feed)\n",
    "\n",
    "        test_loss.append(loss)\n",
    "        test_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    print('epoch %d, training avg loss %f, training avg acc %f'%(e+1,\n",
    "                                                                 np.mean(train_loss),np.mean(train_acc)))\n",
    "    print('epoch %d, testing avg loss %f, testing avg acc %f'%(e+1,\n",
    "                                                              np.mean(test_loss),np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from im2latex/model.ckpt\n",
      "INFO:tensorflow:Froze 27 variables.\n",
      "INFO:tensorflow:Converted 27 variables to const ops.\n",
      "805 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "saver.save(sess, 'im2latex/model.ckpt')\n",
    "freeze_graph('im2latex', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16, 15)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decoded = sess.run(model.predicting_ids, feed_dict = {model.X: batch_x[:1]})[0]\n",
    "decoded.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "abad sehingga\n",
      "abad sehingga\n",
      "abad sehingga\n",
      "abad sehingga\n",
      "abad sehingga\n",
      "abad sehingga\n",
      "abad sehingga\n",
      "abad sehingga\n",
      "abad sehingga\n",
      "abad sehingga\n",
      "abad sehingga\n",
      "abad sehingga\n",
      "abad sehingga\n",
      "abad sehingga\n",
      "abad sehingga\n"
     ]
    }
   ],
   "source": [
    "for i in range(decoded.shape[1]):\n",
    "    d = decoded[:,0]\n",
    "    print(''.join([decode_maps[i] for i in d if i not in [0,1,2]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 13252 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(cv2.flip(batch_x[0][:,:,0], 1))\n",
    "decoded = ''.join([decode_maps[i] for i in decoded[:,0] if i not in [0,1,2]])\n",
    "actual = ''.join([decode_maps[i] for i in batch_y[0] if i not in [0,1,2]])\n",
    "plt.title('predict: %s, actual: %s'%(decoded, actual))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = load_graph('im2latex/frozen_model.pb')\n",
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "logits = g.get_tensor_by_name('import/logits:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/client/session.py:1735: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    }
   ],
   "source": [
    "test_sess = tf.InteractiveSession(graph = g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21, 15)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decoded = test_sess.run(logits, feed_dict = {x: batch_x[10:11]})[0]\n",
    "decoded.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "betina sesetengah\n",
      "betina sesetengah\n",
      "betina sesetengah\n",
      "betina sesetengah\n",
      "betina sesetengah\n",
      "betina sesetengah\n",
      "betina sesetengah\n",
      "betina sesetengah\n",
      "betina sesetengah\n",
      "betina sesetengah\n",
      "betina sesetengah\n",
      "betina sesetengah\n",
      "betina sesetengah\n",
      "betina sesetengah\n",
      "betina sesetengah\n"
     ]
    }
   ],
   "source": [
    "for i in range(decoded.shape[1]):\n",
    "    d = decoded[:,0]\n",
    "    print(''.join([decode_maps[i] for i in d if i not in [0,1,2]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 17108 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(cv2.flip(batch_x[10][:,:,0], 1))\n",
    "decoded = ''.join([decode_maps[i] for i in decoded[:,0] if i not in [0,1,2]])\n",
    "actual = ''.join([decode_maps[i] for i in batch_y[10] if i not in [0,1,2]])\n",
    "plt.title('predict: %s, actual: %s'%(decoded, actual))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
