{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Attention:\n",
    "    def __init__(self,hidden_size):\n",
    "        self.hidden_size = hidden_size\n",
    "        self.dense_layer = tf.layers.Dense(hidden_size)\n",
    "        self.v = tf.random_normal([hidden_size],mean=0,stddev=1/np.sqrt(hidden_size))\n",
    "        \n",
    "    def score(self, hidden_tensor, encoder_outputs):\n",
    "        energy = tf.nn.tanh(self.dense_layer(tf.concat([hidden_tensor,encoder_outputs],2)))\n",
    "        energy = tf.transpose(energy,[0,2,1])\n",
    "        batch_size = tf.shape(encoder_outputs)[0]\n",
    "        v = tf.expand_dims(tf.tile(tf.expand_dims(self.v,0),[batch_size,1]),1)\n",
    "        energy = tf.matmul(v,energy)\n",
    "        return tf.squeeze(energy,1)\n",
    "    \n",
    "    def __call__(self, hidden, encoder_outputs):\n",
    "        seq_len = tf.shape(encoder_outputs)[1]\n",
    "        batch_size = tf.shape(encoder_outputs)[0]\n",
    "        H = tf.tile(tf.expand_dims(hidden, 1),[1,seq_len,1])\n",
    "        attn_energies = self.score(H,encoder_outputs)\n",
    "        return tf.expand_dims(tf.nn.softmax(attn_energies),1)\n",
    "\n",
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        dict_size,\n",
    "        size_layers,\n",
    "        learning_rate,\n",
    "        maxlen,\n",
    "        num_blocks = 3,\n",
    "    ):\n",
    "        block_size = size_layers\n",
    "        self.BEFORE = tf.placeholder(tf.int32,[None,maxlen])\n",
    "        self.INPUT = tf.placeholder(tf.int32,[None,maxlen])\n",
    "        self.AFTER = tf.placeholder(tf.int32,[None,maxlen])\n",
    "        self.batch_size = tf.shape(self.INPUT)[0]\n",
    "        self.output_layer = tf.layers.Dense(dict_size, name=\"output_layer\")\n",
    "        self.output_layer.build(size_layers)\n",
    "        self.embeddings = tf.Variable(tf.random_uniform([dict_size, size_layers], -1, 1))\n",
    "        embedded = tf.nn.embedding_lookup(self.embeddings, self.INPUT)\n",
    "        self.attention = Attention(size_layers)\n",
    "\n",
    "        def residual_block(x, size, rate, block, reuse = False):\n",
    "            with tf.variable_scope(\n",
    "                'block_%d_%d' % (block, rate), reuse = reuse\n",
    "            ):\n",
    "                attn_weights = self.attention(tf.reduce_sum(x,axis=1), x)\n",
    "                conv_filter = tf.layers.conv1d(\n",
    "                    attn_weights,\n",
    "                    x.shape[2] // 4,\n",
    "                    kernel_size = size,\n",
    "                    strides = 1,\n",
    "                    padding = 'same',\n",
    "                    dilation_rate = rate,\n",
    "                    activation = tf.nn.tanh,\n",
    "                )\n",
    "                conv_gate = tf.layers.conv1d(\n",
    "                    x,\n",
    "                    x.shape[2] // 4,\n",
    "                    kernel_size = size,\n",
    "                    strides = 1,\n",
    "                    padding = 'same',\n",
    "                    dilation_rate = rate,\n",
    "                    activation = tf.nn.sigmoid,\n",
    "                )\n",
    "                out = tf.multiply(conv_filter, conv_gate)\n",
    "                out = tf.layers.conv1d(\n",
    "                    out,\n",
    "                    block_size,\n",
    "                    kernel_size = 1,\n",
    "                    strides = 1,\n",
    "                    padding = 'same',\n",
    "                    activation = tf.nn.tanh,\n",
    "                )\n",
    "                return tf.add(x, out), out\n",
    "\n",
    "        forward = tf.layers.conv1d(\n",
    "            embedded, block_size, kernel_size = 1, strides = 1, padding = 'SAME'\n",
    "        )\n",
    "        zeros = tf.zeros_like(forward)\n",
    "        for i in range(num_blocks):\n",
    "            for r in [1, 2, 4, 8, 16]:\n",
    "                forward, s = residual_block(\n",
    "                    forward, size = 7, rate = r, block = i\n",
    "                )\n",
    "                zeros = tf.add(zeros, s)\n",
    "        forward = tf.layers.conv1d(\n",
    "            zeros,\n",
    "            block_size,\n",
    "            kernel_size = 1,\n",
    "            strides = 1,\n",
    "            padding = 'SAME',\n",
    "            activation = tf.nn.tanh,\n",
    "        )\n",
    "        self.get_thought = tf.reduce_sum(forward,axis=1, name = 'logits')\n",
    "        \n",
    "        def decoder(labels, reuse):\n",
    "            decoder_in = tf.nn.embedding_lookup(self.embeddings, labels)\n",
    "            forward = tf.layers.conv1d(\n",
    "                decoder_in, block_size, kernel_size = 1, strides = 1, padding = 'SAME'\n",
    "            )\n",
    "            zeros = tf.zeros_like(forward)\n",
    "            for r in [8, 16, 24]:\n",
    "                forward, s = residual_block(forward, size = 7, rate = r, block = 10, reuse = reuse)\n",
    "                zeros = tf.add(zeros, s)\n",
    "            return tf.layers.conv1d(\n",
    "                zeros,\n",
    "                block_size,\n",
    "                kernel_size = 1,\n",
    "                strides = 1,\n",
    "                padding = 'SAME',\n",
    "                activation = tf.nn.tanh,\n",
    "            )\n",
    "        \n",
    "        fw_logits = decoder(self.AFTER, False)\n",
    "        bw_logits = decoder(self.BEFORE, True)\n",
    "        self.attention = tf.matmul(\n",
    "            self.get_thought, tf.transpose(self.embeddings), name = 'attention'\n",
    "        )\n",
    "        self.loss = self.calculate_loss(fw_logits, self.AFTER) + self.calculate_loss(bw_logits, self.BEFORE)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.loss)\n",
    "    \n",
    "    def calculate_loss(self, outputs, labels):\n",
    "        mask = tf.cast(tf.sign(labels), tf.float32)\n",
    "        logits = self.output_layer(outputs)\n",
    "        return tf.contrib.seq2seq.sequence_loss(logits, labels, mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200004"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "with open('skip-wiki-dict.json') as fopen:\n",
    "    dictionary = json.load(fopen)\n",
    "len(dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rename(checkpoint_dir, replace_from, replace_to, add_prefix, dry_run=False):\n",
    "    checkpoint = tf.train.get_checkpoint_state(checkpoint_dir)\n",
    "    with tf.Session() as sess:\n",
    "        for var_name, _ in tf.contrib.framework.list_variables(checkpoint_dir):\n",
    "            var = tf.contrib.framework.load_variable(checkpoint_dir, var_name)\n",
    "            new_name = var_name\n",
    "            if None not in [replace_from, replace_to]:\n",
    "                new_name = new_name.replace(replace_from, replace_to)\n",
    "            if add_prefix:\n",
    "                new_name = add_prefix + new_name\n",
    "\n",
    "            if dry_run:\n",
    "                print('%s would be renamed to %s.' % (var_name, new_name))\n",
    "            else:\n",
    "                print('Renaming %s to %s.' % (var_name, new_name))\n",
    "                # Rename the variable\n",
    "                var = tf.Variable(var, name=new_name)\n",
    "\n",
    "        if not dry_run:\n",
    "            # Save the variables\n",
    "            saver = tf.train.Saver()\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            saver.save(sess, 'skip-rename/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rename('skip/model.ckpt','thought_scope_e1d42da4-5ae4-4898-b0f1-f52f687a4e28',\n",
    "#       'thought_scope',None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py:1711: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(len(dictionary), 64, 1e-3, 50)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from skip-wiki/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "saver=tf.train.Saver(tf.global_variables())\n",
    "saver.restore(sess, 'skip-wiki/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "def sequence(s, w2v_model, maxlen, vocabulary_size):\n",
    "    words = s.split()\n",
    "    np_array = np.zeros((maxlen),dtype=np.int32)\n",
    "    current_no = 0\n",
    "    for no, word in enumerate(words[:maxlen - 2]):\n",
    "        id_to_append = 1\n",
    "        if word in w2v_model:\n",
    "            word_id = w2v_model[word]\n",
    "            if word_id < vocabulary_size:\n",
    "                id_to_append = word_id\n",
    "        np_array[no] = id_to_append\n",
    "        current_no = no\n",
    "    np_array[current_no + 1] = 3\n",
    "    return np_array\n",
    "\n",
    "def generate_batch(sentences,batch_size,w2v_model,maxlen,vocabulary_size):\n",
    "    window_size = batch_size + 2\n",
    "    first_index = 1000\n",
    "    batch_sentences = sentences[first_index:first_index+window_size]\n",
    "    print(batch_sentences)\n",
    "    batch_sequences = np.array([sequence(sentence,w2v_model,maxlen,vocabulary_size) for sentence in batch_sentences])\n",
    "    window_shape = []\n",
    "    for i in range(batch_size):\n",
    "        window_shape.append(batch_sequences[i:i+3])\n",
    "    window_shape = np.array(window_shape)\n",
    "    return window_shape[:,0], window_shape[:,1], window_shape[:,2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('news-bm.json','r') as fopen:\n",
    "    sentences = json.loads(fopen.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['pahang diwakili pemangku raja pahang tengku abdullah sultan ahmad shah manakala kelantan diwakili pemangku raja kelantan dr', 'tengku muhammad faiz petra', 'pada hari kedua mesyuarat yang bermula kira pukul pagi itu raja-raja melayu diiringi menteri besar masing-masing manakala yang dipertua negeri pulau pinang sabah dan melaka diiringi ketua menteri masing-masing']\n"
     ]
    }
   ],
   "source": [
    "bw_input, current_input, fw_input = generate_batch(sentences,1,dictionary,50,len(dictionary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded = sess.run(model.get_thought,feed_dict={model.INPUT:fw_input})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.07066324,  0.13310698, -0.62426007, -0.4613824 , -0.17707539,\n",
       "        -0.3925364 ,  1.1155262 ,  1.1873002 ,  0.48969495,  0.81452906,\n",
       "        -0.1577659 , -0.17734857, -0.37914753, -0.7942437 ,  0.56107384,\n",
       "         0.29675886, -0.7340232 , -0.07755096,  0.29897642, -0.0737358 ,\n",
       "         0.6024291 ,  0.95485014, -0.95064414, -0.63884234,  0.03552189,\n",
       "        -0.40762448, -0.25227717, -0.24423571,  0.37850273, -0.11428429,\n",
       "        -0.8386208 , -0.2072649 , -0.9640392 , -0.63121736, -0.5339436 ,\n",
       "         0.96501446, -0.12163527,  0.31738836,  0.9421329 , -0.51436657,\n",
       "         0.6444553 , -0.2436821 , -0.4731561 , -0.00128211, -0.05046922,\n",
       "         0.5482205 ,  0.85903156,  0.681826  ,  0.02734087,  0.5048841 ,\n",
       "         0.08036114,  0.00166782,  0.5863657 ,  0.37902188, -0.14853519,\n",
       "         0.11486635,  0.03344561,  1.1854374 , -0.07733421, -0.8486209 ,\n",
       "         0.9942196 ,  0.9136265 , -0.10116772, -0.21602613]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if (\n",
    "            'Variable' in n.op\n",
    "            or n.name.find('Placeholder') >= 0\n",
    "            or 'add_1' in n.name\n",
    "            or 'attention' in n.name\n",
    "            or 'logits' in n.name\n",
    "        )\n",
    "        and 'Adam' not in n.name\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Placeholder_2',\n",
       " 'output_layer/kernel',\n",
       " 'output_layer/bias',\n",
       " 'Variable',\n",
       " 'conv1d/kernel',\n",
       " 'conv1d/bias',\n",
       " 'block_0_1/dense/kernel',\n",
       " 'block_0_1/dense/bias',\n",
       " 'block_0_1/dense/Tensordot/add_1',\n",
       " 'block_0_1/conv1d/kernel',\n",
       " 'block_0_1/conv1d/bias',\n",
       " 'block_0_1/conv1d_1/kernel',\n",
       " 'block_0_1/conv1d_1/bias',\n",
       " 'block_0_1/conv1d_2/kernel',\n",
       " 'block_0_1/conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_1/add_1',\n",
       " 'block_0_2/conv1d/kernel',\n",
       " 'block_0_2/conv1d/bias',\n",
       " 'block_0_2/conv1d_1/kernel',\n",
       " 'block_0_2/conv1d_1/bias',\n",
       " 'block_0_2/conv1d_2/kernel',\n",
       " 'block_0_2/conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_2/add_1',\n",
       " 'block_0_4/conv1d/kernel',\n",
       " 'block_0_4/conv1d/bias',\n",
       " 'block_0_4/conv1d_1/kernel',\n",
       " 'block_0_4/conv1d_1/bias',\n",
       " 'block_0_4/conv1d_2/kernel',\n",
       " 'block_0_4/conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_3/add_1',\n",
       " 'block_0_8/conv1d/kernel',\n",
       " 'block_0_8/conv1d/bias',\n",
       " 'block_0_8/conv1d_1/kernel',\n",
       " 'block_0_8/conv1d_1/bias',\n",
       " 'block_0_8/conv1d_2/kernel',\n",
       " 'block_0_8/conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_4/add_1',\n",
       " 'block_0_16/conv1d/kernel',\n",
       " 'block_0_16/conv1d/bias',\n",
       " 'block_0_16/conv1d_1/kernel',\n",
       " 'block_0_16/conv1d_1/bias',\n",
       " 'block_0_16/conv1d_2/kernel',\n",
       " 'block_0_16/conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_5/add_1',\n",
       " 'block_1_1/conv1d/kernel',\n",
       " 'block_1_1/conv1d/bias',\n",
       " 'block_1_1/conv1d_1/kernel',\n",
       " 'block_1_1/conv1d_1/bias',\n",
       " 'block_1_1/conv1d_2/kernel',\n",
       " 'block_1_1/conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_6/add_1',\n",
       " 'block_1_2/conv1d/kernel',\n",
       " 'block_1_2/conv1d/bias',\n",
       " 'block_1_2/conv1d_1/kernel',\n",
       " 'block_1_2/conv1d_1/bias',\n",
       " 'block_1_2/conv1d_2/kernel',\n",
       " 'block_1_2/conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_7/add_1',\n",
       " 'block_1_4/conv1d/kernel',\n",
       " 'block_1_4/conv1d/bias',\n",
       " 'block_1_4/conv1d_1/kernel',\n",
       " 'block_1_4/conv1d_1/bias',\n",
       " 'block_1_4/conv1d_2/kernel',\n",
       " 'block_1_4/conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_8/add_1',\n",
       " 'block_1_8/conv1d/kernel',\n",
       " 'block_1_8/conv1d/bias',\n",
       " 'block_1_8/conv1d_1/kernel',\n",
       " 'block_1_8/conv1d_1/bias',\n",
       " 'block_1_8/conv1d_2/kernel',\n",
       " 'block_1_8/conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_9/add_1',\n",
       " 'block_1_16/conv1d/kernel',\n",
       " 'block_1_16/conv1d/bias',\n",
       " 'block_1_16/conv1d_1/kernel',\n",
       " 'block_1_16/conv1d_1/bias',\n",
       " 'block_1_16/conv1d_2/kernel',\n",
       " 'block_1_16/conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_10/add_1',\n",
       " 'block_2_1/conv1d/kernel',\n",
       " 'block_2_1/conv1d/bias',\n",
       " 'block_2_1/conv1d_1/kernel',\n",
       " 'block_2_1/conv1d_1/bias',\n",
       " 'block_2_1/conv1d_2/kernel',\n",
       " 'block_2_1/conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_11/add_1',\n",
       " 'block_2_2/conv1d/kernel',\n",
       " 'block_2_2/conv1d/bias',\n",
       " 'block_2_2/conv1d_1/kernel',\n",
       " 'block_2_2/conv1d_1/bias',\n",
       " 'block_2_2/conv1d_2/kernel',\n",
       " 'block_2_2/conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_12/add_1',\n",
       " 'block_2_4/conv1d/kernel',\n",
       " 'block_2_4/conv1d/bias',\n",
       " 'block_2_4/conv1d_1/kernel',\n",
       " 'block_2_4/conv1d_1/bias',\n",
       " 'block_2_4/conv1d_2/kernel',\n",
       " 'block_2_4/conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_13/add_1',\n",
       " 'block_2_8/conv1d/kernel',\n",
       " 'block_2_8/conv1d/bias',\n",
       " 'block_2_8/conv1d_1/kernel',\n",
       " 'block_2_8/conv1d_1/bias',\n",
       " 'block_2_8/conv1d_2/kernel',\n",
       " 'block_2_8/conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_14/add_1',\n",
       " 'block_2_16/conv1d/kernel',\n",
       " 'block_2_16/conv1d/bias',\n",
       " 'block_2_16/conv1d_1/kernel',\n",
       " 'block_2_16/conv1d_1/bias',\n",
       " 'block_2_16/conv1d_2/kernel',\n",
       " 'block_2_16/conv1d_2/bias',\n",
       " 'conv1d_1/kernel',\n",
       " 'conv1d_1/bias',\n",
       " 'logits/reduction_indices',\n",
       " 'logits',\n",
       " 'conv1d_2/kernel',\n",
       " 'conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_15/add_1',\n",
       " 'block_10_8/conv1d/kernel',\n",
       " 'block_10_8/conv1d/bias',\n",
       " 'block_10_8/conv1d_1/kernel',\n",
       " 'block_10_8/conv1d_1/bias',\n",
       " 'block_10_8/conv1d_2/kernel',\n",
       " 'block_10_8/conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_16/add_1',\n",
       " 'block_10_16/conv1d/kernel',\n",
       " 'block_10_16/conv1d/bias',\n",
       " 'block_10_16/conv1d_1/kernel',\n",
       " 'block_10_16/conv1d_1/bias',\n",
       " 'block_10_16/conv1d_2/kernel',\n",
       " 'block_10_16/conv1d_2/bias',\n",
       " 'block_0_1/dense/Tensordot_17/add_1',\n",
       " 'block_10_24/conv1d/kernel',\n",
       " 'block_10_24/conv1d/bias',\n",
       " 'block_10_24/conv1d_1/kernel',\n",
       " 'block_10_24/conv1d_1/bias',\n",
       " 'block_10_24/conv1d_2/kernel',\n",
       " 'block_10_24/conv1d_2/bias',\n",
       " 'conv1d_3/kernel',\n",
       " 'conv1d_3/bias',\n",
       " 'conv1d_4/kernel',\n",
       " 'conv1d_4/bias',\n",
       " 'block_0_1/dense/Tensordot_18/add_1',\n",
       " 'block_0_1/dense/Tensordot_19/add_1',\n",
       " 'block_0_1/dense/Tensordot_20/add_1',\n",
       " 'conv1d_5/kernel',\n",
       " 'conv1d_5/bias',\n",
       " 'attention',\n",
       " 'output_layer/Tensordot/add_1',\n",
       " 'output_layer/Tensordot_1/add_1',\n",
       " 'beta1_power',\n",
       " 'beta2_power']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            \"directory: %s\" % model_dir)\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "    \n",
    "    absolute_model_dir = \"/\".join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + \"/frozen_model.pb\"\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph=tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(input_checkpoint + '.meta', clear_devices=clear_devices)\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(\",\")\n",
    "        ) \n",
    "        with tf.gfile.GFile(output_graph, \"wb\") as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print(\"%d ops in the final graph.\" % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from skip-wiki/model.ckpt\n",
      "INFO:tensorflow:Froze 127 variables.\n",
      "Converted 127 variables to const ops.\n",
      "2031 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('skip-wiki', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, \"rb\") as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "g=load_graph('skip-wiki/frozen_model.pb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py:1711: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    }
   ],
   "source": [
    "x = g.get_tensor_by_name('import/Placeholder_1:0')\n",
    "logits = g.get_tensor_by_name('import/logits:0')\n",
    "attention = g.get_tensor_by_name('import/attention:0')\n",
    "test_sess = tf.InteractiveSession(graph=g)\n",
    "out, att = test_sess.run([logits,attention], feed_dict={x:fw_input})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 200004)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "att.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "rev_dict = {v: k for k, v in dictionary.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38799\n",
      "jagaannya\n",
      "4035\n",
      "zulkifli\n",
      "101993\n",
      "ferdy\n",
      "11445\n",
      "hoe\n",
      "165827\n",
      "sharidake\n",
      "325\n",
      "televisyen\n",
      "1681\n",
      "kawan\n",
      "124186\n",
      "diimbau\n",
      "34683\n",
      "luteum\n",
      "636\n",
      "brunei\n"
     ]
    }
   ],
   "source": [
    "for i in att[0].argsort()[-10:][::-1]:\n",
    "    print(i)\n",
    "    print(rev_dict[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
