{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import re\n",
    "import os\n",
    "from sklearn.metrics import classification_report\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15194"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('entities-bm-normalize-v3.txt','r') as fopen:\n",
    "    texts= list(filter(None, fopen.read().split('\\n')))\n",
    "len(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tag2idx = {'PAD': 0}\n",
    "pos2idx = {'PAD': 0}\n",
    "char2idx = {'PAD': 0}\n",
    "tag_idx = 1\n",
    "char_idx = 1\n",
    "pos_idx = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_texts = texts\n",
    "# dataset is too small\n",
    "test_texts = texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_word(word, lower=True):\n",
    "    if lower:\n",
    "        word = word.lower()\n",
    "    else:\n",
    "        if word.isupper():\n",
    "            word = word.title()\n",
    "    word = re.sub('[^A-Za-z0-9\\- ]+', '', word)\n",
    "    if word.isdigit():\n",
    "        word = 'NUM'\n",
    "    return word\n",
    "\n",
    "def read_file(f):\n",
    "    global tag_idx, char_idx, pos_idx\n",
    "    words, tags, poss, X, Y, Y_pos = [], [], [], [], [], []\n",
    "    for line in f:\n",
    "        line = line.strip()\n",
    "        if (len(line) == 0 or line.startswith(\"-DOCSTART-\")):\n",
    "            continue\n",
    "        else:\n",
    "            ls = line.split(' ')\n",
    "            try:\n",
    "                word, tag, pos = ls[0],ls[1],ls[2]\n",
    "            except:\n",
    "                continue\n",
    "            word = process_word(word)\n",
    "            if len(word) < 1:\n",
    "                continue\n",
    "            char_ids = []\n",
    "            for c in word:\n",
    "                if c not in char2idx:\n",
    "                    char2idx[c] = char_idx\n",
    "                    char_idx += 1\n",
    "                char_ids.append(char2idx[c])\n",
    "            words += [word]\n",
    "            tags += [tag]\n",
    "            poss += [pos]\n",
    "            X.append(char_ids)\n",
    "            if tag not in tag2idx:\n",
    "                tag2idx[tag] = tag_idx\n",
    "                tag_idx += 1\n",
    "            if pos not in pos2idx:\n",
    "                pos2idx[pos] = pos_idx\n",
    "                pos_idx += 1\n",
    "            Y.append(tag2idx[tag])\n",
    "            Y_pos.append(pos2idx[pos])\n",
    "                        \n",
    "    return words, tags, poss, X, np.array(Y), np.array(Y_pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_words, train_tags, train_poss, train_X, train_Y, train_Y_pos = read_file(train_texts)\n",
    "test_words, test_tags, test_poss, test_X, test_Y, test_Y_pos = read_file(test_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size,\n",
    "                 dict_size, dimension_output, dimension_output_pos, learning_rate):\n",
    "        \n",
    "        def cells(size, reuse=False):\n",
    "            return tf.nn.rnn_cell.LSTMCell(size,initializer=tf.orthogonal_initializer(),reuse=reuse)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.float32, [None, dimension_output])\n",
    "        self.Y_pos = tf.placeholder(tf.float32, [None, dimension_output_pos])\n",
    "        encoder_embeddings = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))\n",
    "        encoder_embedded = tf.nn.embedding_lookup(encoder_embeddings, self.X)\n",
    "        \n",
    "        for n in range(num_layers):\n",
    "            (out_fw, out_bw), (state_fw, state_bw) = tf.nn.bidirectional_dynamic_rnn(\n",
    "                cell_fw = cells(size_layer // 2),\n",
    "                cell_bw = cells(size_layer // 2),\n",
    "                inputs = encoder_embedded,\n",
    "                dtype = tf.float32,\n",
    "                scope = 'bidirectional_rnn_%d'%(n))\n",
    "            encoder_embedded = tf.concat((out_fw, out_bw), 2)\n",
    "        W = tf.get_variable('w',shape=(size_layer, dimension_output),initializer=tf.orthogonal_initializer())\n",
    "        b = tf.get_variable('b',shape=(dimension_output),initializer=tf.zeros_initializer())\n",
    "        W_pos = tf.get_variable('w_pos',shape=(size_layer, dimension_output_pos),initializer=tf.orthogonal_initializer())\n",
    "        b_pos = tf.get_variable('b_pos',shape=(dimension_output_pos),initializer=tf.zeros_initializer())\n",
    "        self.logits = tf.add(tf.matmul(encoder_embedded[:, -1], W),b,name='logits')\n",
    "        self.logits_pos = tf.add(tf.matmul(encoder_embedded[:, -1], W_pos),b_pos,name='logits_pos')\n",
    "        cost_entity = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits, labels = self.Y))\n",
    "        cost_pos = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits_pos, labels = self.Y_pos))\n",
    "        self.cost = cost_entity + cost_pos\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        correct_pred = tf.equal(tf.argmax(self.logits, 1), tf.argmax(self.Y, 1))\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "        correct_pred_pos = tf.equal(tf.argmax(self.logits_pos, 1), tf.argmax(self.Y_pos, 1))\n",
    "        self.accuracy_pos = tf.reduce_mean(tf.cast(correct_pred_pos, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_onehot = np.zeros((train_Y.shape[0],len(tag2idx)))\n",
    "train_onehot[np.arange(train_Y.shape[0]),train_Y] = 1.0\n",
    "train_onehot_pos = np.zeros((train_Y_pos.shape[0],len(pos2idx)))\n",
    "train_onehot_pos[np.arange(train_Y_pos.shape[0]),train_Y_pos] = 1.0\n",
    "\n",
    "test_onehot = np.zeros((test_Y.shape[0],len(tag2idx)))\n",
    "test_onehot[np.arange(test_Y.shape[0]),test_Y] = 1.0\n",
    "test_onehot_pos = np.zeros((test_Y_pos.shape[0],len(pos2idx)))\n",
    "test_onehot_pos[np.arange(test_Y_pos.shape[0]),test_Y_pos] = 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 128\n",
    "num_layers = 2\n",
    "embedded_size = 128\n",
    "dimension_output = len(tag2idx)\n",
    "dimension_output_pos = len(pos2idx)\n",
    "learning_rate = 1e-3\n",
    "batch_size = 32\n",
    "idx2tag={idx: tag for tag, idx in tag2idx.items()}\n",
    "idx2pos={idx: tag for tag, idx in pos2idx.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('char-settings.json','w') as fopen:\n",
    "    fopen.write(json.dumps({'idx2tag':idx2tag,'idx2pos':idx2pos,'tag2idx':tag2idx,'pos2idx':pos2idx,'char2idx':char2idx}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic, UNK=3):\n",
    "    maxlen = max([len(i) for i in corpus])\n",
    "    X = np.zeros((len(corpus),maxlen))\n",
    "    for i in range(len(corpus)):\n",
    "        for no, k in enumerate(corpus[i][:maxlen][::-1]):\n",
    "            try:\n",
    "                X[i,-1 - no]=dic[k]\n",
    "            except Exception as e:\n",
    "                X[i,-1 - no]=UNK\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-7-8fac7491dc94>:28: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(size_layer,num_layers,embedded_size,len(char2idx),\n",
    "              dimension_output,dimension_output_pos,learning_rate)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.000000, current acc: 0.780716\n",
      "time taken: 18.98798632621765\n",
      "epoch: 0, training loss: 1.998942, training acc: 0.721154, valid loss: 1.550866, valid acc: 0.780716\n",
      "\n",
      "epoch: 1, pass acc: 0.780716, current acc: 0.803385\n",
      "time taken: 21.58083415031433\n",
      "epoch: 1, training loss: 1.366895, training acc: 0.804387, valid loss: 1.301349, valid acc: 0.803385\n",
      "\n",
      "epoch: 2, pass acc: 0.803385, current acc: 0.836104\n",
      "time taken: 21.7098650932312\n",
      "epoch: 2, training loss: 1.168944, training acc: 0.833267, valid loss: 1.146023, valid acc: 0.836104\n",
      "\n",
      "epoch: 3, pass acc: 0.836104, current acc: 0.848524\n",
      "time taken: 21.69019603729248\n",
      "epoch: 3, training loss: 1.051429, training acc: 0.847990, valid loss: 1.027778, valid acc: 0.848524\n",
      "\n",
      "epoch: 4, pass acc: 0.848524, current acc: 0.860143\n",
      "time taken: 21.572304248809814\n",
      "epoch: 4, training loss: 0.960130, training acc: 0.858774, valid loss: 0.942974, valid acc: 0.860143\n",
      "\n",
      "epoch: 5, pass acc: 0.860143, current acc: 0.866687\n",
      "time taken: 21.70699381828308\n",
      "epoch: 5, training loss: 0.887664, training acc: 0.868122, valid loss: 0.894153, valid acc: 0.866687\n",
      "\n",
      "epoch: 6, pass acc: 0.866687, current acc: 0.873130\n",
      "time taken: 21.673323154449463\n",
      "epoch: 6, training loss: 0.827795, training acc: 0.876536, valid loss: 0.842749, valid acc: 0.873130\n",
      "\n",
      "epoch: 7, pass acc: 0.873130, current acc: 0.880809\n",
      "time taken: 21.63108539581299\n",
      "epoch: 7, training loss: 0.775000, training acc: 0.882979, valid loss: 0.795866, valid acc: 0.880809\n",
      "\n",
      "epoch: 8, pass acc: 0.880809, current acc: 0.886652\n",
      "time taken: 21.54202914237976\n",
      "epoch: 8, training loss: 0.727128, training acc: 0.889790, valid loss: 0.748803, valid acc: 0.886652\n",
      "\n",
      "epoch: 9, pass acc: 0.886652, current acc: 0.890391\n",
      "time taken: 21.570520639419556\n",
      "epoch: 9, training loss: 0.687533, training acc: 0.893997, valid loss: 0.720360, valid acc: 0.890391\n",
      "\n",
      "epoch: 10, pass acc: 0.890391, current acc: 0.894464\n",
      "time taken: 21.52001643180847\n",
      "epoch: 10, training loss: 0.654449, training acc: 0.898271, valid loss: 0.700037, valid acc: 0.894464\n",
      "\n",
      "epoch: 11, pass acc: 0.894464, current acc: 0.898805\n",
      "time taken: 21.498887300491333\n",
      "epoch: 11, training loss: 0.625374, training acc: 0.901175, valid loss: 0.669348, valid acc: 0.898805\n",
      "\n",
      "epoch: 12, pass acc: 0.898805, current acc: 0.899840\n",
      "time taken: 21.723352432250977\n",
      "epoch: 12, training loss: 0.598619, training acc: 0.905883, valid loss: 0.642673, valid acc: 0.899840\n",
      "\n",
      "epoch: 13, pass acc: 0.899840, current acc: 0.900741\n",
      "time taken: 21.67806577682495\n",
      "epoch: 13, training loss: 0.577554, training acc: 0.906784, valid loss: 0.630975, valid acc: 0.900741\n",
      "\n",
      "epoch: 14, pass acc: 0.900741, current acc: 0.904447\n",
      "time taken: 21.69010305404663\n",
      "epoch: 14, training loss: 0.551874, training acc: 0.910690, valid loss: 0.599477, valid acc: 0.904447\n",
      "\n",
      "time taken: 21.657179355621338\n",
      "epoch: 15, training loss: 0.529899, training acc: 0.912627, valid loss: 0.596786, valid acc: 0.903579\n",
      "\n",
      "epoch: 16, pass acc: 0.904447, current acc: 0.905716\n",
      "time taken: 21.785433292388916\n",
      "epoch: 16, training loss: 0.513279, training acc: 0.914463, valid loss: 0.587909, valid acc: 0.905716\n",
      "\n",
      "time taken: 21.778388023376465\n",
      "epoch: 17, training loss: 0.502226, training acc: 0.916500, valid loss: 0.577847, valid acc: 0.904848\n",
      "\n",
      "epoch: 18, pass acc: 0.905716, current acc: 0.907252\n",
      "time taken: 21.614745616912842\n",
      "epoch: 18, training loss: 0.487408, training acc: 0.917902, valid loss: 0.567347, valid acc: 0.907252\n",
      "\n",
      "time taken: 21.671206951141357\n",
      "epoch: 19, training loss: 0.470046, training acc: 0.920072, valid loss: 0.569621, valid acc: 0.905182\n",
      "\n",
      "time taken: 21.67682981491089\n",
      "epoch: 20, training loss: 0.457097, training acc: 0.920873, valid loss: 0.556407, valid acc: 0.906116\n",
      "\n",
      "time taken: 21.55426859855652\n",
      "epoch: 21, training loss: 0.444981, training acc: 0.922476, valid loss: 0.563894, valid acc: 0.905883\n",
      "\n",
      "epoch: 22, pass acc: 0.907252, current acc: 0.907853\n",
      "time taken: 21.584643840789795\n",
      "epoch: 22, training loss: 0.434219, training acc: 0.923044, valid loss: 0.541347, valid acc: 0.907853\n",
      "\n",
      "time taken: 21.46364688873291\n",
      "epoch: 23, training loss: 0.424346, training acc: 0.924546, valid loss: 0.543344, valid acc: 0.907285\n",
      "\n",
      "epoch: 24, pass acc: 0.907853, current acc: 0.909555\n",
      "time taken: 21.512551069259644\n",
      "epoch: 24, training loss: 0.419888, training acc: 0.924713, valid loss: 0.522833, valid acc: 0.909555\n",
      "\n",
      "epoch: 25, pass acc: 0.909555, current acc: 0.910891\n",
      "time taken: 21.498022317886353\n",
      "epoch: 25, training loss: 0.413125, training acc: 0.925915, valid loss: 0.520470, valid acc: 0.910891\n",
      "\n",
      "epoch: 26, pass acc: 0.910891, current acc: 0.911291\n",
      "time taken: 21.52986454963684\n",
      "epoch: 26, training loss: 0.400837, training acc: 0.928018, valid loss: 0.517079, valid acc: 0.911291\n",
      "\n",
      "epoch: 27, pass acc: 0.911291, current acc: 0.912360\n",
      "time taken: 21.65718722343445\n",
      "epoch: 27, training loss: 0.399081, training acc: 0.926349, valid loss: 0.507206, valid acc: 0.912360\n",
      "\n",
      "epoch: 28, pass acc: 0.912360, current acc: 0.913962\n",
      "time taken: 21.579139471054077\n",
      "epoch: 28, training loss: 0.393547, training acc: 0.928653, valid loss: 0.498560, valid acc: 0.913962\n",
      "\n",
      "time taken: 21.56552815437317\n",
      "epoch: 29, training loss: 0.388136, training acc: 0.928819, valid loss: 0.502940, valid acc: 0.912961\n",
      "\n",
      "epoch: 30, pass acc: 0.913962, current acc: 0.914363\n",
      "time taken: 21.55526614189148\n",
      "epoch: 30, training loss: 0.384601, training acc: 0.929354, valid loss: 0.493290, valid acc: 0.914363\n",
      "\n",
      "time taken: 21.478503465652466\n",
      "epoch: 31, training loss: 0.373340, training acc: 0.930055, valid loss: 0.498301, valid acc: 0.913261\n",
      "\n",
      "time taken: 21.574644565582275\n",
      "epoch: 32, training loss: 0.372059, training acc: 0.930489, valid loss: 0.499049, valid acc: 0.913929\n",
      "\n",
      "epoch: 33, pass acc: 0.914363, current acc: 0.916233\n",
      "time taken: 21.4410138130188\n",
      "epoch: 33, training loss: 0.368516, training acc: 0.931190, valid loss: 0.483331, valid acc: 0.916233\n",
      "\n",
      "time taken: 21.53644108772278\n",
      "epoch: 34, training loss: 0.366300, training acc: 0.931424, valid loss: 0.488928, valid acc: 0.914497\n",
      "\n",
      "time taken: 21.391112327575684\n",
      "epoch: 35, training loss: 0.364563, training acc: 0.930489, valid loss: 0.508152, valid acc: 0.911659\n",
      "\n",
      "time taken: 21.41468048095703\n",
      "epoch: 36, training loss: 0.359991, training acc: 0.929955, valid loss: 0.494809, valid acc: 0.914330\n",
      "\n",
      "time taken: 21.350648164749146\n",
      "epoch: 37, training loss: 0.352404, training acc: 0.932258, valid loss: 0.511083, valid acc: 0.912727\n",
      "\n",
      "time taken: 21.41197109222412\n",
      "epoch: 38, training loss: 0.351894, training acc: 0.930622, valid loss: 0.488564, valid acc: 0.914363\n",
      "\n",
      "break epoch:39\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 5, 0, 0, 0\n",
    "while True:\n",
    "    lasttime = time.time()\n",
    "    if CURRENT_CHECKPOINT == EARLY_STOPPING:\n",
    "        print('break epoch:%d\\n'%(EPOCH))\n",
    "        break\n",
    "        \n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    for i in range(0, (len(train_X) // batch_size) * batch_size, batch_size):\n",
    "        batch_x = str_idx(train_words[i:i+batch_size],char2idx)\n",
    "        acc_pos, acc, loss, _ = sess.run([model.accuracy_pos, model.accuracy, model.cost, model.optimizer], \n",
    "                           feed_dict = {model.X : batch_x, model.Y : train_onehot[i:i+batch_size],\n",
    "                                       model.Y_pos: train_onehot_pos[i:i+batch_size]})\n",
    "        train_loss += loss\n",
    "        train_acc += ((acc+acc_pos)/2)\n",
    "    \n",
    "    for i in range(0, (len(test_X) // batch_size) * batch_size, batch_size):\n",
    "        batch_x = str_idx(test_words[i:i+batch_size],char2idx)\n",
    "        acc_pos, acc, loss = sess.run([model.accuracy_pos, model.accuracy, model.cost], \n",
    "                           feed_dict = {model.X : batch_x, model.Y : test_onehot[i:i+batch_size],\n",
    "                                       model.Y_pos : test_onehot_pos[i:i+batch_size]})\n",
    "        test_loss += loss\n",
    "        test_acc += ((acc+acc_pos)/2)\n",
    "    \n",
    "    train_loss /= (len(train_X) // batch_size)\n",
    "    train_acc /= (len(train_X) // batch_size)\n",
    "    test_loss /= (len(test_X) // batch_size)\n",
    "    test_acc /= (len(test_X) // batch_size)\n",
    "    \n",
    "    if test_acc > CURRENT_ACC:\n",
    "        print('epoch: %d, pass acc: %f, current acc: %f'%(EPOCH,CURRENT_ACC, test_acc))\n",
    "        CURRENT_ACC = test_acc\n",
    "        CURRENT_CHECKPOINT = 0\n",
    "    else:\n",
    "        CURRENT_CHECKPOINT += 1\n",
    "        \n",
    "    print('time taken:', time.time()-lasttime)\n",
    "    print('epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'%(EPOCH,train_loss,\n",
    "                                                                                          train_acc,test_loss,\n",
    "                                                                                          test_acc))\n",
    "    EPOCH += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_Y, label_pos, predicted_Y, predicted_pos = [], [], [], []\n",
    "for i in range(0, (len(test_X) // batch_size) * batch_size, batch_size):\n",
    "    batch_x = str_idx(test_words[i:i+batch_size],char2idx)\n",
    "    label_Y += test_Y[i:i+batch_size].tolist()\n",
    "    label_pos += test_Y_pos[i:i+batch_size].tolist()\n",
    "    logits, logits_pos = sess.run([tf.argmax(model.logits,1),tf.argmax(model.logits_pos,1)], \n",
    "                                  feed_dict = {model.X : batch_x})\n",
    "    predicted_Y += logits.tolist()\n",
    "    predicted_pos += logits_pos.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "       TIME       0.74      0.82      0.78       474\n",
      "        ART       0.95      0.96      0.96     11530\n",
      "          I       0.86      0.80      0.83      1347\n",
      "        LOC       0.88      0.90      0.89       174\n",
      "        ORG       0.53      0.72      0.61       367\n",
      "        PRN       0.65      0.80      0.72       173\n",
      "        PAD       0.70      0.55      0.61       144\n",
      "       NORP       0.72      0.60      0.65       575\n",
      "        DOC       0.77      0.71      0.74        85\n",
      "          O       0.86      0.51      0.64       105\n",
      "      EVENT       0.00      0.00      0.00         1\n",
      "        LAW       1.00      1.00      1.00         1\n",
      "\n",
      "avg / total       0.91      0.91      0.91     14976\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/sklearn/metrics/classification.py:1428: UserWarning: labels size, 12, does not match size of target_names, 13\n",
      "  .format(len(labels), len(target_names))\n",
      "/usr/local/lib/python3.5/dist-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(label_Y, predicted_Y, target_names=tag2idx.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         GN       0.91      0.96      0.94      8478\n",
      "         KJ       0.96      0.95      0.96       109\n",
      "         NO       0.97      0.71      0.82      2606\n",
      "       NORP       0.95      0.99      0.97      1170\n",
      "         KP       0.98      0.99      0.99       374\n",
      "          O       0.53      0.68      0.59       204\n",
      "       ARAH       0.84      0.99      0.91       660\n",
      "    KPEMERI       0.98      0.97      0.98       641\n",
      "         KB       0.98      0.99      0.98       287\n",
      "    PISAHAN       0.00      0.00      0.00        11\n",
      "       NAFI       1.00      0.98      0.99       144\n",
      "         KA       0.88      1.00      0.93         7\n",
      "         KS       0.93      0.93      0.93       103\n",
      "         KH       1.00      1.00      1.00        53\n",
      "   KPENGUAT       0.95      0.98      0.97        58\n",
      "   KPENEGAS       0.71      1.00      0.83         5\n",
      "         KT       0.97      0.93      0.95        30\n",
      "        PAD       1.00      1.00      1.00        11\n",
      "         KN       0.83      0.79      0.81        19\n",
      "         KM       0.00      0.00      0.00         1\n",
      " KETERANGAN       1.00      0.50      0.67         4\n",
      "       SUKU       0.33      1.00      0.50         1\n",
      "\n",
      "avg / total       0.92      0.92      0.92     14976\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/sklearn/metrics/classification.py:1428: UserWarning: labels size, 22, does not match size of target_names, 23\n",
      "  .format(len(labels), len(target_names))\n",
      "/usr/local/lib/python3.5/dist-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(label_pos, predicted_pos, target_names=pos2idx.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_entity(string):\n",
    "    batch_x = str_idx([process_word(w) for w in string.split()],char2idx)\n",
    "    logits, logits_pos = sess.run([tf.argmax(model.logits,1),tf.argmax(model.logits_pos,1)],feed_dict={model.X:batch_x})\n",
    "    for no, i in enumerate(string.split()):\n",
    "        print(i,idx2tag[logits[no]],idx2pos[logits_pos[no]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KUALA LOC KN\n",
      "LUMPUR: LOC KN\n",
      "Sempena O KN\n",
      "sambutan O KN\n",
      "Aidilfitri EVENT KN\n",
      "minggu O KN\n",
      "depan, O KN\n",
      "Perdana PRN KN\n",
      "Menteri PRN KN\n",
      "Tun PRN KN\n",
      "Dr PRN KN\n",
      "Mahathir PRN KN\n",
      "Mohamad PRN KN\n",
      "dan O KH\n",
      "Menteri PRN KN\n",
      "Pengangkutan LAW KN\n",
      "Anthony PRN KN\n",
      "Loke PRN KN\n",
      "Siew PRN KN\n",
      "Fook PRN KN\n",
      "menitipkan O KN\n",
      "pesanan O KN\n",
      "khas O KN\n",
      "kepada O KS\n",
      "orang O KN\n",
      "ramai O KN\n",
      "yang O KETERANGAN\n",
      "mahu O KN\n",
      "pulang O KN\n",
      "ke O KS\n",
      "kampung LOC KN\n",
      "halaman LOC KN\n",
      "masing-masing. O KN\n",
      "Dalam O KS\n",
      "video O KN\n",
      "pendek O KN\n",
      "terbitan O KN\n",
      "Jabatan NORP KN\n",
      "Keselamatan O KN\n",
      "Jalan LOC KN\n",
      "Raya ART KN\n",
      "(JKJR) PRN KN\n",
      "itu, O GN\n",
      "Dr PRN KN\n",
      "Mahathir PRN KN\n",
      "menasihati O KJ\n",
      "mereka O GN\n",
      "supaya O KH\n",
      "berhenti O KJ\n",
      "berehat O KA\n",
      "dan O KH\n",
      "tidur O KN\n",
      "sebentar O KETERANGAN\n",
      "sekiranya O KN\n",
      "mengantuk O KJ\n",
      "ketika O KN\n",
      "memandu. O KJ\n"
     ]
    }
   ],
   "source": [
    "get_entity('KUALA LUMPUR: Sempena sambutan Aidilfitri minggu depan, Perdana Menteri Tun Dr Mahathir Mohamad dan Menteri Pengangkutan Anthony Loke Siew Fook menitipkan pesanan khas kepada orang ramai yang mahu pulang ke kampung halaman masing-masing. Dalam video pendek terbitan Jabatan Keselamatan Jalan Raya (JKJR) itu, Dr Mahathir menasihati mereka supaya berhenti berehat dan tidur sebentar  sekiranya mengantuk ketika memandu.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/husein/space/text-dataset/entities/bm/char/char-entity-pos.ckpt'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.global_variables())\n",
    "saver.save(sess, os.getcwd()+\"/char/char-entity-pos.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# only load Variables, placeholder for input, and our logits\n",
    "strings=','.join([n.name for n in tf.get_default_graph().as_graph_def().node if \"Variable\" in n.op or n.name.find('Placeholder') >= 0 or n.name.find('logits') == 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            \"directory: %s\" % model_dir)\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "    \n",
    "    absolute_model_dir = \"/\".join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + \"/frozen_model.pb\"\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph=tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(input_checkpoint + '.meta', clear_devices=clear_devices)\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(\",\")\n",
    "        ) \n",
    "        with tf.gfile.GFile(output_graph, \"wb\") as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print(\"%d ops in the final graph.\" % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /home/husein/space/text-dataset/entities/bm/char/char-entity-pos.ckpt\n",
      "INFO:tensorflow:Froze 41 variables.\n",
      "Converted 41 variables to const ops.\n",
      "550 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph(\"char\", strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, \"rb\") as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "g=load_graph('char/frozen_model.pb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "import/Placeholder\n",
      "import/Placeholder_1\n",
      "import/Placeholder_2\n",
      "import/Variable\n",
      "import/Variable/read\n",
      "import/embedding_lookup\n",
      "import/bidirectional_rnn_0/fw/fw/Rank\n",
      "import/bidirectional_rnn_0/fw/fw/range/start\n",
      "import/bidirectional_rnn_0/fw/fw/range/delta\n",
      "import/bidirectional_rnn_0/fw/fw/range\n",
      "import/bidirectional_rnn_0/fw/fw/concat/values_0\n",
      "import/bidirectional_rnn_0/fw/fw/concat/axis\n",
      "import/bidirectional_rnn_0/fw/fw/concat\n",
      "import/bidirectional_rnn_0/fw/fw/transpose\n",
      "import/bidirectional_rnn_0/fw/fw/Shape\n",
      "import/bidirectional_rnn_0/fw/fw/strided_slice/stack\n",
      "import/bidirectional_rnn_0/fw/fw/strided_slice/stack_1\n",
      "import/bidirectional_rnn_0/fw/fw/strided_slice/stack_2\n",
      "import/bidirectional_rnn_0/fw/fw/strided_slice\n",
      "import/bidirectional_rnn_0/fw/fw/LSTMCellZeroState/ExpandDims/dim\n",
      "import/bidirectional_rnn_0/fw/fw/LSTMCellZeroState/ExpandDims\n",
      "import/bidirectional_rnn_0/fw/fw/LSTMCellZeroState/Const\n",
      "import/bidirectional_rnn_0/fw/fw/LSTMCellZeroState/concat/axis\n",
      "import/bidirectional_rnn_0/fw/fw/LSTMCellZeroState/concat\n",
      "import/bidirectional_rnn_0/fw/fw/LSTMCellZeroState/zeros/Const\n",
      "import/bidirectional_rnn_0/fw/fw/LSTMCellZeroState/zeros\n",
      "import/bidirectional_rnn_0/fw/fw/LSTMCellZeroState/ExpandDims_2/dim\n",
      "import/bidirectional_rnn_0/fw/fw/LSTMCellZeroState/ExpandDims_2\n",
      "import/bidirectional_rnn_0/fw/fw/LSTMCellZeroState/Const_2\n",
      "import/bidirectional_rnn_0/fw/fw/LSTMCellZeroState/concat_1/axis\n",
      "import/bidirectional_rnn_0/fw/fw/LSTMCellZeroState/concat_1\n",
      "import/bidirectional_rnn_0/fw/fw/LSTMCellZeroState/zeros_1/Const\n",
      "import/bidirectional_rnn_0/fw/fw/LSTMCellZeroState/zeros_1\n",
      "import/bidirectional_rnn_0/fw/fw/Shape_1\n",
      "import/bidirectional_rnn_0/fw/fw/strided_slice_1/stack\n",
      "import/bidirectional_rnn_0/fw/fw/strided_slice_1/stack_1\n",
      "import/bidirectional_rnn_0/fw/fw/strided_slice_1/stack_2\n",
      "import/bidirectional_rnn_0/fw/fw/strided_slice_1\n",
      "import/bidirectional_rnn_0/fw/fw/time\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArray\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArray_1\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/Shape\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/strided_slice/stack\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/strided_slice/stack_1\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/strided_slice/stack_2\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/strided_slice\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/range/start\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/range/delta\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/range\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3\n",
      "import/bidirectional_rnn_0/fw/fw/while/iteration_counter\n",
      "import/bidirectional_rnn_0/fw/fw/while/Enter\n",
      "import/bidirectional_rnn_0/fw/fw/while/Enter_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/Enter_2\n",
      "import/bidirectional_rnn_0/fw/fw/while/Enter_3\n",
      "import/bidirectional_rnn_0/fw/fw/while/Enter_4\n",
      "import/bidirectional_rnn_0/fw/fw/while/Merge\n",
      "import/bidirectional_rnn_0/fw/fw/while/Merge_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/Merge_2\n",
      "import/bidirectional_rnn_0/fw/fw/while/Merge_3\n",
      "import/bidirectional_rnn_0/fw/fw/while/Merge_4\n",
      "import/bidirectional_rnn_0/fw/fw/while/Less\n",
      "import/bidirectional_rnn_0/fw/fw/while/Less/Enter\n",
      "import/bidirectional_rnn_0/fw/fw/while/Less_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/LogicalAnd\n",
      "import/bidirectional_rnn_0/fw/fw/while/LoopCond\n",
      "import/bidirectional_rnn_0/fw/fw/while/Switch\n",
      "import/bidirectional_rnn_0/fw/fw/while/Switch_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/Switch_2\n",
      "import/bidirectional_rnn_0/fw/fw/while/Switch_3\n",
      "import/bidirectional_rnn_0/fw/fw/while/Switch_4\n",
      "import/bidirectional_rnn_0/fw/fw/while/Identity\n",
      "import/bidirectional_rnn_0/fw/fw/while/Identity_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/Identity_2\n",
      "import/bidirectional_rnn_0/fw/fw/while/Identity_3\n",
      "import/bidirectional_rnn_0/fw/fw/while/Identity_4\n",
      "import/bidirectional_rnn_0/fw/fw/while/add/y\n",
      "import/bidirectional_rnn_0/fw/fw/while/add\n",
      "import/bidirectional_rnn_0/fw/fw/while/TensorArrayReadV3\n",
      "import/bidirectional_rnn_0/fw/fw/while/TensorArrayReadV3/Enter\n",
      "import/bidirectional_rnn_0/fw/fw/while/TensorArrayReadV3/Enter_1\n",
      "import/bidirectional_rnn_0/fw/lstm_cell/kernel\n",
      "import/bidirectional_rnn_0/fw/lstm_cell/kernel/read\n",
      "import/bidirectional_rnn_0/fw/lstm_cell/bias\n",
      "import/bidirectional_rnn_0/fw/lstm_cell/bias/read\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/concat/axis\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/concat\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/MatMul\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/MatMul/Enter\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/BiasAdd\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/BiasAdd/Enter\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/split/split_dim\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/split\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/add/y\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/add\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/Sigmoid\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/mul\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/Sigmoid_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/Tanh\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/mul_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/add_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/Sigmoid_2\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/Tanh_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/lstm_cell/mul_2\n",
      "import/bidirectional_rnn_0/fw/fw/while/TensorArrayWrite/TensorArrayWriteV3\n",
      "import/bidirectional_rnn_0/fw/fw/while/TensorArrayWrite/TensorArrayWriteV3/Enter\n",
      "import/bidirectional_rnn_0/fw/fw/while/add_1/y\n",
      "import/bidirectional_rnn_0/fw/fw/while/add_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/NextIteration\n",
      "import/bidirectional_rnn_0/fw/fw/while/NextIteration_1\n",
      "import/bidirectional_rnn_0/fw/fw/while/NextIteration_2\n",
      "import/bidirectional_rnn_0/fw/fw/while/NextIteration_3\n",
      "import/bidirectional_rnn_0/fw/fw/while/NextIteration_4\n",
      "import/bidirectional_rnn_0/fw/fw/while/Exit_2\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayStack/TensorArraySizeV3\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayStack/range/start\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayStack/range/delta\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayStack/range\n",
      "import/bidirectional_rnn_0/fw/fw/TensorArrayStack/TensorArrayGatherV3\n",
      "import/bidirectional_rnn_0/fw/fw/Rank_1\n",
      "import/bidirectional_rnn_0/fw/fw/range_1/start\n",
      "import/bidirectional_rnn_0/fw/fw/range_1/delta\n",
      "import/bidirectional_rnn_0/fw/fw/range_1\n",
      "import/bidirectional_rnn_0/fw/fw/concat_2/values_0\n",
      "import/bidirectional_rnn_0/fw/fw/concat_2/axis\n",
      "import/bidirectional_rnn_0/fw/fw/concat_2\n",
      "import/bidirectional_rnn_0/fw/fw/transpose_1\n",
      "import/bidirectional_rnn_0/bw/ReverseV2/axis\n",
      "import/bidirectional_rnn_0/bw/ReverseV2\n",
      "import/bidirectional_rnn_0/bw/bw/Rank\n",
      "import/bidirectional_rnn_0/bw/bw/range/start\n",
      "import/bidirectional_rnn_0/bw/bw/range/delta\n",
      "import/bidirectional_rnn_0/bw/bw/range\n",
      "import/bidirectional_rnn_0/bw/bw/concat/values_0\n",
      "import/bidirectional_rnn_0/bw/bw/concat/axis\n",
      "import/bidirectional_rnn_0/bw/bw/concat\n",
      "import/bidirectional_rnn_0/bw/bw/transpose\n",
      "import/bidirectional_rnn_0/bw/bw/Shape\n",
      "import/bidirectional_rnn_0/bw/bw/strided_slice/stack\n",
      "import/bidirectional_rnn_0/bw/bw/strided_slice/stack_1\n",
      "import/bidirectional_rnn_0/bw/bw/strided_slice/stack_2\n",
      "import/bidirectional_rnn_0/bw/bw/strided_slice\n",
      "import/bidirectional_rnn_0/bw/bw/LSTMCellZeroState/ExpandDims/dim\n",
      "import/bidirectional_rnn_0/bw/bw/LSTMCellZeroState/ExpandDims\n",
      "import/bidirectional_rnn_0/bw/bw/LSTMCellZeroState/Const\n",
      "import/bidirectional_rnn_0/bw/bw/LSTMCellZeroState/concat/axis\n",
      "import/bidirectional_rnn_0/bw/bw/LSTMCellZeroState/concat\n",
      "import/bidirectional_rnn_0/bw/bw/LSTMCellZeroState/zeros/Const\n",
      "import/bidirectional_rnn_0/bw/bw/LSTMCellZeroState/zeros\n",
      "import/bidirectional_rnn_0/bw/bw/LSTMCellZeroState/ExpandDims_2/dim\n",
      "import/bidirectional_rnn_0/bw/bw/LSTMCellZeroState/ExpandDims_2\n",
      "import/bidirectional_rnn_0/bw/bw/LSTMCellZeroState/Const_2\n",
      "import/bidirectional_rnn_0/bw/bw/LSTMCellZeroState/concat_1/axis\n",
      "import/bidirectional_rnn_0/bw/bw/LSTMCellZeroState/concat_1\n",
      "import/bidirectional_rnn_0/bw/bw/LSTMCellZeroState/zeros_1/Const\n",
      "import/bidirectional_rnn_0/bw/bw/LSTMCellZeroState/zeros_1\n",
      "import/bidirectional_rnn_0/bw/bw/Shape_1\n",
      "import/bidirectional_rnn_0/bw/bw/strided_slice_1/stack\n",
      "import/bidirectional_rnn_0/bw/bw/strided_slice_1/stack_1\n",
      "import/bidirectional_rnn_0/bw/bw/strided_slice_1/stack_2\n",
      "import/bidirectional_rnn_0/bw/bw/strided_slice_1\n",
      "import/bidirectional_rnn_0/bw/bw/time\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArray\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArray_1\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/Shape\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/strided_slice/stack\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/strided_slice/stack_1\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/strided_slice/stack_2\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/strided_slice\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/range/start\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/range/delta\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/range\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3\n",
      "import/bidirectional_rnn_0/bw/bw/while/iteration_counter\n",
      "import/bidirectional_rnn_0/bw/bw/while/Enter\n",
      "import/bidirectional_rnn_0/bw/bw/while/Enter_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/Enter_2\n",
      "import/bidirectional_rnn_0/bw/bw/while/Enter_3\n",
      "import/bidirectional_rnn_0/bw/bw/while/Enter_4\n",
      "import/bidirectional_rnn_0/bw/bw/while/Merge\n",
      "import/bidirectional_rnn_0/bw/bw/while/Merge_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/Merge_2\n",
      "import/bidirectional_rnn_0/bw/bw/while/Merge_3\n",
      "import/bidirectional_rnn_0/bw/bw/while/Merge_4\n",
      "import/bidirectional_rnn_0/bw/bw/while/Less\n",
      "import/bidirectional_rnn_0/bw/bw/while/Less/Enter\n",
      "import/bidirectional_rnn_0/bw/bw/while/Less_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/LogicalAnd\n",
      "import/bidirectional_rnn_0/bw/bw/while/LoopCond\n",
      "import/bidirectional_rnn_0/bw/bw/while/Switch\n",
      "import/bidirectional_rnn_0/bw/bw/while/Switch_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/Switch_2\n",
      "import/bidirectional_rnn_0/bw/bw/while/Switch_3\n",
      "import/bidirectional_rnn_0/bw/bw/while/Switch_4\n",
      "import/bidirectional_rnn_0/bw/bw/while/Identity\n",
      "import/bidirectional_rnn_0/bw/bw/while/Identity_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/Identity_2\n",
      "import/bidirectional_rnn_0/bw/bw/while/Identity_3\n",
      "import/bidirectional_rnn_0/bw/bw/while/Identity_4\n",
      "import/bidirectional_rnn_0/bw/bw/while/add/y\n",
      "import/bidirectional_rnn_0/bw/bw/while/add\n",
      "import/bidirectional_rnn_0/bw/bw/while/TensorArrayReadV3\n",
      "import/bidirectional_rnn_0/bw/bw/while/TensorArrayReadV3/Enter\n",
      "import/bidirectional_rnn_0/bw/bw/while/TensorArrayReadV3/Enter_1\n",
      "import/bidirectional_rnn_0/bw/lstm_cell/kernel\n",
      "import/bidirectional_rnn_0/bw/lstm_cell/kernel/read\n",
      "import/bidirectional_rnn_0/bw/lstm_cell/bias\n",
      "import/bidirectional_rnn_0/bw/lstm_cell/bias/read\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/concat/axis\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/concat\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/MatMul\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/MatMul/Enter\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/BiasAdd\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/BiasAdd/Enter\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/split/split_dim\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/split\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/add/y\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/add\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/Sigmoid\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/mul\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/Sigmoid_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/Tanh\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/mul_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/add_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/Sigmoid_2\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/Tanh_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/lstm_cell/mul_2\n",
      "import/bidirectional_rnn_0/bw/bw/while/TensorArrayWrite/TensorArrayWriteV3\n",
      "import/bidirectional_rnn_0/bw/bw/while/TensorArrayWrite/TensorArrayWriteV3/Enter\n",
      "import/bidirectional_rnn_0/bw/bw/while/add_1/y\n",
      "import/bidirectional_rnn_0/bw/bw/while/add_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/NextIteration\n",
      "import/bidirectional_rnn_0/bw/bw/while/NextIteration_1\n",
      "import/bidirectional_rnn_0/bw/bw/while/NextIteration_2\n",
      "import/bidirectional_rnn_0/bw/bw/while/NextIteration_3\n",
      "import/bidirectional_rnn_0/bw/bw/while/NextIteration_4\n",
      "import/bidirectional_rnn_0/bw/bw/while/Exit_2\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayStack/TensorArraySizeV3\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayStack/range/start\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayStack/range/delta\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayStack/range\n",
      "import/bidirectional_rnn_0/bw/bw/TensorArrayStack/TensorArrayGatherV3\n",
      "import/bidirectional_rnn_0/bw/bw/Rank_1\n",
      "import/bidirectional_rnn_0/bw/bw/range_1/start\n",
      "import/bidirectional_rnn_0/bw/bw/range_1/delta\n",
      "import/bidirectional_rnn_0/bw/bw/range_1\n",
      "import/bidirectional_rnn_0/bw/bw/concat_2/values_0\n",
      "import/bidirectional_rnn_0/bw/bw/concat_2/axis\n",
      "import/bidirectional_rnn_0/bw/bw/concat_2\n",
      "import/bidirectional_rnn_0/bw/bw/transpose_1\n",
      "import/ReverseV2/axis\n",
      "import/ReverseV2\n",
      "import/concat/axis\n",
      "import/concat\n",
      "import/bidirectional_rnn_1/fw/fw/Rank\n",
      "import/bidirectional_rnn_1/fw/fw/range/start\n",
      "import/bidirectional_rnn_1/fw/fw/range/delta\n",
      "import/bidirectional_rnn_1/fw/fw/range\n",
      "import/bidirectional_rnn_1/fw/fw/concat/values_0\n",
      "import/bidirectional_rnn_1/fw/fw/concat/axis\n",
      "import/bidirectional_rnn_1/fw/fw/concat\n",
      "import/bidirectional_rnn_1/fw/fw/transpose\n",
      "import/bidirectional_rnn_1/fw/fw/Shape\n",
      "import/bidirectional_rnn_1/fw/fw/strided_slice/stack\n",
      "import/bidirectional_rnn_1/fw/fw/strided_slice/stack_1\n",
      "import/bidirectional_rnn_1/fw/fw/strided_slice/stack_2\n",
      "import/bidirectional_rnn_1/fw/fw/strided_slice\n",
      "import/bidirectional_rnn_1/fw/fw/LSTMCellZeroState/ExpandDims/dim\n",
      "import/bidirectional_rnn_1/fw/fw/LSTMCellZeroState/ExpandDims\n",
      "import/bidirectional_rnn_1/fw/fw/LSTMCellZeroState/Const\n",
      "import/bidirectional_rnn_1/fw/fw/LSTMCellZeroState/concat/axis\n",
      "import/bidirectional_rnn_1/fw/fw/LSTMCellZeroState/concat\n",
      "import/bidirectional_rnn_1/fw/fw/LSTMCellZeroState/zeros/Const\n",
      "import/bidirectional_rnn_1/fw/fw/LSTMCellZeroState/zeros\n",
      "import/bidirectional_rnn_1/fw/fw/LSTMCellZeroState/ExpandDims_2/dim\n",
      "import/bidirectional_rnn_1/fw/fw/LSTMCellZeroState/ExpandDims_2\n",
      "import/bidirectional_rnn_1/fw/fw/LSTMCellZeroState/Const_2\n",
      "import/bidirectional_rnn_1/fw/fw/LSTMCellZeroState/concat_1/axis\n",
      "import/bidirectional_rnn_1/fw/fw/LSTMCellZeroState/concat_1\n",
      "import/bidirectional_rnn_1/fw/fw/LSTMCellZeroState/zeros_1/Const\n",
      "import/bidirectional_rnn_1/fw/fw/LSTMCellZeroState/zeros_1\n",
      "import/bidirectional_rnn_1/fw/fw/Shape_1\n",
      "import/bidirectional_rnn_1/fw/fw/strided_slice_1/stack\n",
      "import/bidirectional_rnn_1/fw/fw/strided_slice_1/stack_1\n",
      "import/bidirectional_rnn_1/fw/fw/strided_slice_1/stack_2\n",
      "import/bidirectional_rnn_1/fw/fw/strided_slice_1\n",
      "import/bidirectional_rnn_1/fw/fw/time\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArray\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArray_1\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/Shape\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/strided_slice/stack\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/strided_slice/stack_1\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/strided_slice/stack_2\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/strided_slice\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/range/start\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/range/delta\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/range\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3\n",
      "import/bidirectional_rnn_1/fw/fw/while/iteration_counter\n",
      "import/bidirectional_rnn_1/fw/fw/while/Enter\n",
      "import/bidirectional_rnn_1/fw/fw/while/Enter_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/Enter_2\n",
      "import/bidirectional_rnn_1/fw/fw/while/Enter_3\n",
      "import/bidirectional_rnn_1/fw/fw/while/Enter_4\n",
      "import/bidirectional_rnn_1/fw/fw/while/Merge\n",
      "import/bidirectional_rnn_1/fw/fw/while/Merge_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/Merge_2\n",
      "import/bidirectional_rnn_1/fw/fw/while/Merge_3\n",
      "import/bidirectional_rnn_1/fw/fw/while/Merge_4\n",
      "import/bidirectional_rnn_1/fw/fw/while/Less\n",
      "import/bidirectional_rnn_1/fw/fw/while/Less/Enter\n",
      "import/bidirectional_rnn_1/fw/fw/while/Less_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/LogicalAnd\n",
      "import/bidirectional_rnn_1/fw/fw/while/LoopCond\n",
      "import/bidirectional_rnn_1/fw/fw/while/Switch\n",
      "import/bidirectional_rnn_1/fw/fw/while/Switch_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/Switch_2\n",
      "import/bidirectional_rnn_1/fw/fw/while/Switch_3\n",
      "import/bidirectional_rnn_1/fw/fw/while/Switch_4\n",
      "import/bidirectional_rnn_1/fw/fw/while/Identity\n",
      "import/bidirectional_rnn_1/fw/fw/while/Identity_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/Identity_2\n",
      "import/bidirectional_rnn_1/fw/fw/while/Identity_3\n",
      "import/bidirectional_rnn_1/fw/fw/while/Identity_4\n",
      "import/bidirectional_rnn_1/fw/fw/while/add/y\n",
      "import/bidirectional_rnn_1/fw/fw/while/add\n",
      "import/bidirectional_rnn_1/fw/fw/while/TensorArrayReadV3\n",
      "import/bidirectional_rnn_1/fw/fw/while/TensorArrayReadV3/Enter\n",
      "import/bidirectional_rnn_1/fw/fw/while/TensorArrayReadV3/Enter_1\n",
      "import/bidirectional_rnn_1/fw/lstm_cell/kernel\n",
      "import/bidirectional_rnn_1/fw/lstm_cell/kernel/read\n",
      "import/bidirectional_rnn_1/fw/lstm_cell/bias\n",
      "import/bidirectional_rnn_1/fw/lstm_cell/bias/read\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/concat/axis\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/concat\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/MatMul\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/MatMul/Enter\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/BiasAdd\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/BiasAdd/Enter\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/split/split_dim\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/split\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/add/y\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/add\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/Sigmoid\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/mul\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/Sigmoid_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/Tanh\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/mul_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/add_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/Sigmoid_2\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/Tanh_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/lstm_cell/mul_2\n",
      "import/bidirectional_rnn_1/fw/fw/while/TensorArrayWrite/TensorArrayWriteV3\n",
      "import/bidirectional_rnn_1/fw/fw/while/TensorArrayWrite/TensorArrayWriteV3/Enter\n",
      "import/bidirectional_rnn_1/fw/fw/while/add_1/y\n",
      "import/bidirectional_rnn_1/fw/fw/while/add_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/NextIteration\n",
      "import/bidirectional_rnn_1/fw/fw/while/NextIteration_1\n",
      "import/bidirectional_rnn_1/fw/fw/while/NextIteration_2\n",
      "import/bidirectional_rnn_1/fw/fw/while/NextIteration_3\n",
      "import/bidirectional_rnn_1/fw/fw/while/NextIteration_4\n",
      "import/bidirectional_rnn_1/fw/fw/while/Exit_2\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayStack/TensorArraySizeV3\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayStack/range/start\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayStack/range/delta\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayStack/range\n",
      "import/bidirectional_rnn_1/fw/fw/TensorArrayStack/TensorArrayGatherV3\n",
      "import/bidirectional_rnn_1/fw/fw/Rank_1\n",
      "import/bidirectional_rnn_1/fw/fw/range_1/start\n",
      "import/bidirectional_rnn_1/fw/fw/range_1/delta\n",
      "import/bidirectional_rnn_1/fw/fw/range_1\n",
      "import/bidirectional_rnn_1/fw/fw/concat_2/values_0\n",
      "import/bidirectional_rnn_1/fw/fw/concat_2/axis\n",
      "import/bidirectional_rnn_1/fw/fw/concat_2\n",
      "import/bidirectional_rnn_1/fw/fw/transpose_1\n",
      "import/bidirectional_rnn_1/bw/ReverseV2/axis\n",
      "import/bidirectional_rnn_1/bw/ReverseV2\n",
      "import/bidirectional_rnn_1/bw/bw/Rank\n",
      "import/bidirectional_rnn_1/bw/bw/range/start\n",
      "import/bidirectional_rnn_1/bw/bw/range/delta\n",
      "import/bidirectional_rnn_1/bw/bw/range\n",
      "import/bidirectional_rnn_1/bw/bw/concat/values_0\n",
      "import/bidirectional_rnn_1/bw/bw/concat/axis\n",
      "import/bidirectional_rnn_1/bw/bw/concat\n",
      "import/bidirectional_rnn_1/bw/bw/transpose\n",
      "import/bidirectional_rnn_1/bw/bw/Shape\n",
      "import/bidirectional_rnn_1/bw/bw/strided_slice/stack\n",
      "import/bidirectional_rnn_1/bw/bw/strided_slice/stack_1\n",
      "import/bidirectional_rnn_1/bw/bw/strided_slice/stack_2\n",
      "import/bidirectional_rnn_1/bw/bw/strided_slice\n",
      "import/bidirectional_rnn_1/bw/bw/LSTMCellZeroState/ExpandDims/dim\n",
      "import/bidirectional_rnn_1/bw/bw/LSTMCellZeroState/ExpandDims\n",
      "import/bidirectional_rnn_1/bw/bw/LSTMCellZeroState/Const\n",
      "import/bidirectional_rnn_1/bw/bw/LSTMCellZeroState/concat/axis\n",
      "import/bidirectional_rnn_1/bw/bw/LSTMCellZeroState/concat\n",
      "import/bidirectional_rnn_1/bw/bw/LSTMCellZeroState/zeros/Const\n",
      "import/bidirectional_rnn_1/bw/bw/LSTMCellZeroState/zeros\n",
      "import/bidirectional_rnn_1/bw/bw/LSTMCellZeroState/ExpandDims_2/dim\n",
      "import/bidirectional_rnn_1/bw/bw/LSTMCellZeroState/ExpandDims_2\n",
      "import/bidirectional_rnn_1/bw/bw/LSTMCellZeroState/Const_2\n",
      "import/bidirectional_rnn_1/bw/bw/LSTMCellZeroState/concat_1/axis\n",
      "import/bidirectional_rnn_1/bw/bw/LSTMCellZeroState/concat_1\n",
      "import/bidirectional_rnn_1/bw/bw/LSTMCellZeroState/zeros_1/Const\n",
      "import/bidirectional_rnn_1/bw/bw/LSTMCellZeroState/zeros_1\n",
      "import/bidirectional_rnn_1/bw/bw/Shape_1\n",
      "import/bidirectional_rnn_1/bw/bw/strided_slice_1/stack\n",
      "import/bidirectional_rnn_1/bw/bw/strided_slice_1/stack_1\n",
      "import/bidirectional_rnn_1/bw/bw/strided_slice_1/stack_2\n",
      "import/bidirectional_rnn_1/bw/bw/strided_slice_1\n",
      "import/bidirectional_rnn_1/bw/bw/time\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArray\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArray_1\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/Shape\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/strided_slice/stack\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/strided_slice/stack_1\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/strided_slice/stack_2\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/strided_slice\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/range/start\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/range/delta\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/range\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3\n",
      "import/bidirectional_rnn_1/bw/bw/while/iteration_counter\n",
      "import/bidirectional_rnn_1/bw/bw/while/Enter\n",
      "import/bidirectional_rnn_1/bw/bw/while/Enter_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/Enter_2\n",
      "import/bidirectional_rnn_1/bw/bw/while/Enter_3\n",
      "import/bidirectional_rnn_1/bw/bw/while/Enter_4\n",
      "import/bidirectional_rnn_1/bw/bw/while/Merge\n",
      "import/bidirectional_rnn_1/bw/bw/while/Merge_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/Merge_2\n",
      "import/bidirectional_rnn_1/bw/bw/while/Merge_3\n",
      "import/bidirectional_rnn_1/bw/bw/while/Merge_4\n",
      "import/bidirectional_rnn_1/bw/bw/while/Less\n",
      "import/bidirectional_rnn_1/bw/bw/while/Less/Enter\n",
      "import/bidirectional_rnn_1/bw/bw/while/Less_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/LogicalAnd\n",
      "import/bidirectional_rnn_1/bw/bw/while/LoopCond\n",
      "import/bidirectional_rnn_1/bw/bw/while/Switch\n",
      "import/bidirectional_rnn_1/bw/bw/while/Switch_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/Switch_2\n",
      "import/bidirectional_rnn_1/bw/bw/while/Switch_3\n",
      "import/bidirectional_rnn_1/bw/bw/while/Switch_4\n",
      "import/bidirectional_rnn_1/bw/bw/while/Identity\n",
      "import/bidirectional_rnn_1/bw/bw/while/Identity_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/Identity_2\n",
      "import/bidirectional_rnn_1/bw/bw/while/Identity_3\n",
      "import/bidirectional_rnn_1/bw/bw/while/Identity_4\n",
      "import/bidirectional_rnn_1/bw/bw/while/add/y\n",
      "import/bidirectional_rnn_1/bw/bw/while/add\n",
      "import/bidirectional_rnn_1/bw/bw/while/TensorArrayReadV3\n",
      "import/bidirectional_rnn_1/bw/bw/while/TensorArrayReadV3/Enter\n",
      "import/bidirectional_rnn_1/bw/bw/while/TensorArrayReadV3/Enter_1\n",
      "import/bidirectional_rnn_1/bw/lstm_cell/kernel\n",
      "import/bidirectional_rnn_1/bw/lstm_cell/kernel/read\n",
      "import/bidirectional_rnn_1/bw/lstm_cell/bias\n",
      "import/bidirectional_rnn_1/bw/lstm_cell/bias/read\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/concat/axis\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/concat\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/MatMul\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/MatMul/Enter\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/BiasAdd\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/BiasAdd/Enter\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/split/split_dim\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/split\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/add/y\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/add\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/Sigmoid\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/mul\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/Sigmoid_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/Tanh\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/mul_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/add_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/Sigmoid_2\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/Tanh_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/lstm_cell/mul_2\n",
      "import/bidirectional_rnn_1/bw/bw/while/TensorArrayWrite/TensorArrayWriteV3\n",
      "import/bidirectional_rnn_1/bw/bw/while/TensorArrayWrite/TensorArrayWriteV3/Enter\n",
      "import/bidirectional_rnn_1/bw/bw/while/add_1/y\n",
      "import/bidirectional_rnn_1/bw/bw/while/add_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/NextIteration\n",
      "import/bidirectional_rnn_1/bw/bw/while/NextIteration_1\n",
      "import/bidirectional_rnn_1/bw/bw/while/NextIteration_2\n",
      "import/bidirectional_rnn_1/bw/bw/while/NextIteration_3\n",
      "import/bidirectional_rnn_1/bw/bw/while/NextIteration_4\n",
      "import/bidirectional_rnn_1/bw/bw/while/Exit_2\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayStack/TensorArraySizeV3\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayStack/range/start\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayStack/range/delta\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayStack/range\n",
      "import/bidirectional_rnn_1/bw/bw/TensorArrayStack/TensorArrayGatherV3\n",
      "import/bidirectional_rnn_1/bw/bw/Rank_1\n",
      "import/bidirectional_rnn_1/bw/bw/range_1/start\n",
      "import/bidirectional_rnn_1/bw/bw/range_1/delta\n",
      "import/bidirectional_rnn_1/bw/bw/range_1\n",
      "import/bidirectional_rnn_1/bw/bw/concat_2/values_0\n",
      "import/bidirectional_rnn_1/bw/bw/concat_2/axis\n",
      "import/bidirectional_rnn_1/bw/bw/concat_2\n",
      "import/bidirectional_rnn_1/bw/bw/transpose_1\n",
      "import/ReverseV2_1/axis\n",
      "import/ReverseV2_1\n",
      "import/concat_1/axis\n",
      "import/concat_1\n",
      "import/w\n",
      "import/w/read\n",
      "import/b\n",
      "import/b/read\n",
      "import/w_pos\n",
      "import/w_pos/read\n",
      "import/b_pos\n",
      "import/b_pos/read\n",
      "import/strided_slice/stack\n",
      "import/strided_slice/stack_1\n",
      "import/strided_slice/stack_2\n",
      "import/strided_slice\n",
      "import/MatMul\n",
      "import/logits\n",
      "import/strided_slice_1/stack\n",
      "import/strided_slice_1/stack_1\n",
      "import/strided_slice_1/stack_2\n",
      "import/strided_slice_1\n",
      "import/MatMul_1\n",
      "import/logits_pos\n",
      "import/beta1_power\n",
      "import/beta2_power\n",
      "import/Variable/Adam\n",
      "import/Variable/Adam_1\n",
      "import/bidirectional_rnn_0/fw/lstm_cell/kernel/Adam\n",
      "import/bidirectional_rnn_0/fw/lstm_cell/kernel/Adam_1\n",
      "import/bidirectional_rnn_0/fw/lstm_cell/bias/Adam\n",
      "import/bidirectional_rnn_0/fw/lstm_cell/bias/Adam_1\n",
      "import/bidirectional_rnn_0/bw/lstm_cell/kernel/Adam\n",
      "import/bidirectional_rnn_0/bw/lstm_cell/kernel/Adam_1\n",
      "import/bidirectional_rnn_0/bw/lstm_cell/bias/Adam\n",
      "import/bidirectional_rnn_0/bw/lstm_cell/bias/Adam_1\n",
      "import/bidirectional_rnn_1/fw/lstm_cell/kernel/Adam\n",
      "import/bidirectional_rnn_1/fw/lstm_cell/kernel/Adam_1\n",
      "import/bidirectional_rnn_1/fw/lstm_cell/bias/Adam\n",
      "import/bidirectional_rnn_1/fw/lstm_cell/bias/Adam_1\n",
      "import/bidirectional_rnn_1/bw/lstm_cell/kernel/Adam\n",
      "import/bidirectional_rnn_1/bw/lstm_cell/kernel/Adam_1\n",
      "import/bidirectional_rnn_1/bw/lstm_cell/bias/Adam\n",
      "import/bidirectional_rnn_1/bw/lstm_cell/bias/Adam_1\n",
      "import/w/Adam\n",
      "import/w/Adam_1\n",
      "import/b/Adam\n",
      "import/b/Adam_1\n",
      "import/w_pos/Adam\n",
      "import/w_pos/Adam_1\n",
      "import/b_pos/Adam\n",
      "import/b_pos/Adam_1\n"
     ]
    }
   ],
   "source": [
    "for op in g.get_operations():\n",
    "    print(op.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "logits = g.get_tensor_by_name('import/logits:0')\n",
    "pos_logits = g.get_tensor_by_name('import/logits_pos:0')\n",
    "test_sess = tf.InteractiveSession(graph=g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32, 13)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_sess.run([logits,pos_logits], feed_dict={x:batch_x})[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
