{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.contrib import seq2seq\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        vocabulary_size,\n",
    "        maxlen = 50,\n",
    "        output_size = 512,\n",
    "        learning_rate = 1e-3,\n",
    "        embedding_size = 256,\n",
    "        batch_size = 16,\n",
    "        max_grad_norm = 10,\n",
    "        **kwargs\n",
    "    ):\n",
    "        word_embeddings = tf.Variable(\n",
    "            tf.random_uniform(\n",
    "                [vocabulary_size, embedding_size], -np.sqrt(3), np.sqrt(3)\n",
    "            )\n",
    "        )\n",
    "        self.output_size = output_size\n",
    "        self.maxlen = maxlen\n",
    "        self.embeddings = word_embeddings\n",
    "        self.output_layer = tf.layers.Dense(vocabulary_size)\n",
    "        self.output_layer.build(output_size)\n",
    "\n",
    "        self.BEFORE = tf.placeholder(tf.int32, [None, maxlen])\n",
    "        self.INPUT = tf.placeholder(tf.int32, [None, maxlen])\n",
    "        self.AFTER = tf.placeholder(tf.int32, [None, maxlen])\n",
    "        self.batch_size = tf.shape(self.INPUT)[0]\n",
    "\n",
    "        self.get_thought = self.thought(self.INPUT)\n",
    "        self.attention = tf.matmul(\n",
    "            self.get_thought, tf.transpose(self.embeddings), name = 'attention'\n",
    "        )\n",
    "        self.fw_logits = self.decoder(self.get_thought, self.AFTER)\n",
    "        self.bw_logits = self.decoder(self.get_thought, self.BEFORE)\n",
    "        self.loss = self.calculate_loss(\n",
    "            self.fw_logits, self.AFTER\n",
    "        ) + self.calculate_loss(self.bw_logits, self.BEFORE)\n",
    "        tvars = tf.trainable_variables()\n",
    "        grads, _ = tf.clip_by_global_norm(\n",
    "            tf.gradients(self.loss, tvars), max_grad_norm\n",
    "        )\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate).apply_gradients(\n",
    "            zip(grads, tvars)\n",
    "        )\n",
    "\n",
    "    def get_embedding(self, inputs):\n",
    "        return tf.nn.embedding_lookup(self.embeddings, inputs)\n",
    "\n",
    "    def thought(self, inputs):\n",
    "        encoder_in = self.get_embedding(inputs)\n",
    "        fw_cell = tf.nn.rnn_cell.GRUCell(self.output_size)\n",
    "        bw_cell = tf.nn.rnn_cell.GRUCell(self.output_size)\n",
    "        sequence_length = tf.reduce_sum(tf.sign(inputs), axis = 1)\n",
    "        with tf.variable_scope(\n",
    "            'thought_scope', reuse = False\n",
    "        ):\n",
    "            rnn_output = tf.nn.bidirectional_dynamic_rnn(\n",
    "                fw_cell,\n",
    "                bw_cell,\n",
    "                encoder_in,\n",
    "                sequence_length = sequence_length,\n",
    "                dtype = tf.float32,\n",
    "            )[1]\n",
    "            return sum(rnn_output)\n",
    "\n",
    "    def decoder(self, thought, labels):\n",
    "        main = tf.strided_slice(labels, [0, 0], [self.batch_size, -1], [1, 1])\n",
    "        shifted_labels = tf.concat([tf.fill([self.batch_size, 1], 2), main], 1)\n",
    "        decoder_in = self.get_embedding(shifted_labels)\n",
    "        cell = tf.nn.rnn_cell.GRUCell(self.output_size)\n",
    "        max_seq_lengths = tf.fill([self.batch_size], self.maxlen)\n",
    "        helper = seq2seq.TrainingHelper(\n",
    "            decoder_in, max_seq_lengths, time_major = False\n",
    "        )\n",
    "        decoder = seq2seq.BasicDecoder(cell, helper, thought)\n",
    "        decoder_out = seq2seq.dynamic_decode(decoder)[0].rnn_output\n",
    "        return decoder_out\n",
    "\n",
    "    def calculate_loss(self, outputs, labels):\n",
    "        mask = tf.cast(tf.sign(labels), tf.float32)\n",
    "        logits = self.output_layer(outputs)\n",
    "        return seq2seq.sequence_loss(logits, labels, mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "54718"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "with open('skip-news-dict.json') as fopen:\n",
    "    dictionary = json.load(fopen)\n",
    "len(dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rename(checkpoint_dir, replace_from, replace_to, add_prefix, dry_run=False):\n",
    "    checkpoint = tf.train.get_checkpoint_state(checkpoint_dir)\n",
    "    with tf.Session() as sess:\n",
    "        for var_name, _ in tf.contrib.framework.list_variables(checkpoint_dir):\n",
    "            var = tf.contrib.framework.load_variable(checkpoint_dir, var_name)\n",
    "            new_name = var_name\n",
    "            if None not in [replace_from, replace_to]:\n",
    "                new_name = new_name.replace(replace_from, replace_to)\n",
    "            if add_prefix:\n",
    "                new_name = add_prefix + new_name\n",
    "\n",
    "            if dry_run:\n",
    "                print('%s would be renamed to %s.' % (var_name, new_name))\n",
    "            else:\n",
    "                print('Renaming %s to %s.' % (var_name, new_name))\n",
    "                # Rename the variable\n",
    "                var = tf.Variable(var, name=new_name)\n",
    "\n",
    "        if not dry_run:\n",
    "            # Save the variables\n",
    "            saver = tf.train.Saver()\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            saver.save(sess, 'skip-rename/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rename('skip/model.ckpt','thought_scope_e1d42da4-5ae4-4898-b0f1-f52f687a4e28',\n",
    "#       'thought_scope',None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(len(dictionary), embedding_size = 128, output_size = 128, batch_size=16,maxlen=100)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Variable:0',\n",
       " 'dense/kernel:0',\n",
       " 'dense/bias:0',\n",
       " 'thought_scope/bidirectional_rnn/fw/gru_cell/gates/kernel:0',\n",
       " 'thought_scope/bidirectional_rnn/fw/gru_cell/gates/bias:0',\n",
       " 'thought_scope/bidirectional_rnn/fw/gru_cell/candidate/kernel:0',\n",
       " 'thought_scope/bidirectional_rnn/fw/gru_cell/candidate/bias:0',\n",
       " 'thought_scope/bidirectional_rnn/bw/gru_cell/gates/kernel:0',\n",
       " 'thought_scope/bidirectional_rnn/bw/gru_cell/gates/bias:0',\n",
       " 'thought_scope/bidirectional_rnn/bw/gru_cell/candidate/kernel:0',\n",
       " 'thought_scope/bidirectional_rnn/bw/gru_cell/candidate/bias:0',\n",
       " 'decoder/gru_cell/gates/kernel:0',\n",
       " 'decoder/gru_cell/gates/bias:0',\n",
       " 'decoder/gru_cell/candidate/kernel:0',\n",
       " 'decoder/gru_cell/candidate/bias:0',\n",
       " 'decoder_1/gru_cell/gates/kernel:0',\n",
       " 'decoder_1/gru_cell/gates/bias:0',\n",
       " 'decoder_1/gru_cell/candidate/kernel:0',\n",
       " 'decoder_1/gru_cell/candidate/bias:0',\n",
       " 'beta1_power:0',\n",
       " 'beta2_power:0',\n",
       " 'Variable/Adam:0',\n",
       " 'Variable/Adam_1:0',\n",
       " 'dense/kernel/Adam:0',\n",
       " 'dense/kernel/Adam_1:0',\n",
       " 'dense/bias/Adam:0',\n",
       " 'dense/bias/Adam_1:0',\n",
       " 'thought_scope/bidirectional_rnn/fw/gru_cell/gates/kernel/Adam:0',\n",
       " 'thought_scope/bidirectional_rnn/fw/gru_cell/gates/kernel/Adam_1:0',\n",
       " 'thought_scope/bidirectional_rnn/fw/gru_cell/gates/bias/Adam:0',\n",
       " 'thought_scope/bidirectional_rnn/fw/gru_cell/gates/bias/Adam_1:0',\n",
       " 'thought_scope/bidirectional_rnn/fw/gru_cell/candidate/kernel/Adam:0',\n",
       " 'thought_scope/bidirectional_rnn/fw/gru_cell/candidate/kernel/Adam_1:0',\n",
       " 'thought_scope/bidirectional_rnn/fw/gru_cell/candidate/bias/Adam:0',\n",
       " 'thought_scope/bidirectional_rnn/fw/gru_cell/candidate/bias/Adam_1:0',\n",
       " 'thought_scope/bidirectional_rnn/bw/gru_cell/gates/kernel/Adam:0',\n",
       " 'thought_scope/bidirectional_rnn/bw/gru_cell/gates/kernel/Adam_1:0',\n",
       " 'thought_scope/bidirectional_rnn/bw/gru_cell/gates/bias/Adam:0',\n",
       " 'thought_scope/bidirectional_rnn/bw/gru_cell/gates/bias/Adam_1:0',\n",
       " 'thought_scope/bidirectional_rnn/bw/gru_cell/candidate/kernel/Adam:0',\n",
       " 'thought_scope/bidirectional_rnn/bw/gru_cell/candidate/kernel/Adam_1:0',\n",
       " 'thought_scope/bidirectional_rnn/bw/gru_cell/candidate/bias/Adam:0',\n",
       " 'thought_scope/bidirectional_rnn/bw/gru_cell/candidate/bias/Adam_1:0',\n",
       " 'decoder/gru_cell/gates/kernel/Adam:0',\n",
       " 'decoder/gru_cell/gates/kernel/Adam_1:0',\n",
       " 'decoder/gru_cell/gates/bias/Adam:0',\n",
       " 'decoder/gru_cell/gates/bias/Adam_1:0',\n",
       " 'decoder/gru_cell/candidate/kernel/Adam:0',\n",
       " 'decoder/gru_cell/candidate/kernel/Adam_1:0',\n",
       " 'decoder/gru_cell/candidate/bias/Adam:0',\n",
       " 'decoder/gru_cell/candidate/bias/Adam_1:0',\n",
       " 'decoder_1/gru_cell/gates/kernel/Adam:0',\n",
       " 'decoder_1/gru_cell/gates/kernel/Adam_1:0',\n",
       " 'decoder_1/gru_cell/gates/bias/Adam:0',\n",
       " 'decoder_1/gru_cell/gates/bias/Adam_1:0',\n",
       " 'decoder_1/gru_cell/candidate/kernel/Adam:0',\n",
       " 'decoder_1/gru_cell/candidate/kernel/Adam_1:0',\n",
       " 'decoder_1/gru_cell/candidate/bias/Adam:0',\n",
       " 'decoder_1/gru_cell/candidate/bias/Adam_1:0']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[i.name for i in tf.global_variables()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from skip/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "saver=tf.train.Saver(tf.global_variables())\n",
    "saver.restore(sess, 'skip/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "def sequence(s, w2v_model, maxlen, vocabulary_size):\n",
    "    words = s.split()\n",
    "    np_array = np.zeros((maxlen),dtype=np.int32)\n",
    "    current_no = 0\n",
    "    for no, word in enumerate(words[:maxlen - 2]):\n",
    "        id_to_append = 1\n",
    "        if word in w2v_model:\n",
    "            word_id = w2v_model[word]\n",
    "            if word_id < vocabulary_size:\n",
    "                id_to_append = word_id\n",
    "        np_array[no] = id_to_append\n",
    "        current_no = no\n",
    "    np_array[current_no + 1] = 3\n",
    "    return np_array\n",
    "\n",
    "def generate_batch(sentences,batch_size,w2v_model,maxlen,vocabulary_size):\n",
    "    window_size = batch_size + 2\n",
    "    first_index = 1000\n",
    "    batch_sentences = sentences[first_index:first_index+window_size]\n",
    "    print(batch_sentences)\n",
    "    batch_sequences = np.array([sequence(sentence,w2v_model,maxlen,vocabulary_size) for sentence in batch_sentences])\n",
    "    window_shape = []\n",
    "    for i in range(batch_size):\n",
    "        window_shape.append(batch_sequences[i:i+3])\n",
    "    window_shape = np.array(window_shape)\n",
    "    return window_shape[:,0], window_shape[:,1], window_shape[:,2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('news-bm.json','r') as fopen:\n",
    "    sentences = json.loads(fopen.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['pahang diwakili pemangku raja pahang tengku abdullah sultan ahmad shah manakala kelantan diwakili pemangku raja kelantan dr', 'tengku muhammad faiz petra', 'pada hari kedua mesyuarat yang bermula kira pukul pagi itu raja-raja melayu diiringi menteri besar masing-masing manakala yang dipertua negeri pulau pinang sabah dan melaka diiringi ketua menteri masing-masing']\n"
     ]
    }
   ],
   "source": [
    "bw_input, current_input, fw_input = generate_batch(sentences,1,dictionary,100,len(dictionary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded = sess.run(model.get_thought,feed_dict={model.INPUT:fw_input})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.0416520e-01, -5.5048943e-01, -8.6489022e-01, -4.7374249e-02,\n",
       "         1.1276997e+00,  1.8109307e+00,  6.9022512e-01,  1.3390839e-02,\n",
       "         2.2568166e-01, -1.2908951e+00,  1.8937750e+00, -6.6073686e-01,\n",
       "         8.8402975e-01, -1.9575896e+00, -1.3369490e+00,  8.7181759e-01,\n",
       "         6.0808134e-01, -1.3946321e+00,  1.2038462e-01,  1.2153907e+00,\n",
       "         5.5231041e-01, -1.6721604e+00, -1.9526482e-04, -6.4797735e-01,\n",
       "         1.9013047e-02,  1.6876624e+00, -1.7706637e+00,  3.0935839e-01,\n",
       "         2.3643266e-01, -7.0005804e-01, -7.6473856e-01, -6.4990938e-01,\n",
       "         8.5101128e-02,  1.9995425e+00, -1.3742411e+00,  1.4046657e+00,\n",
       "         1.2373401e+00,  1.3037590e+00,  5.5078387e-01, -1.6784103e+00,\n",
       "        -1.5637214e+00,  1.4834172e-01, -1.0372441e+00, -2.6549307e-01,\n",
       "        -1.8813536e+00,  1.2753011e-01,  1.6532394e+00, -5.8884758e-01,\n",
       "        -2.4680305e-01, -1.9865925e+00,  7.4487889e-01, -2.9214048e-01,\n",
       "         7.9541242e-01, -7.1536422e-01,  9.7346407e-01, -2.9780412e-01,\n",
       "        -1.4487034e+00,  1.0695006e+00,  7.1344101e-01, -1.7302066e-01,\n",
       "         1.3620573e-01,  1.3157678e-01,  4.6292901e-02, -6.6628301e-01,\n",
       "        -9.3853849e-01, -2.3844108e-02, -2.4575531e-02,  1.0214790e+00,\n",
       "        -1.6275005e+00,  1.0081427e+00,  1.0262668e-02,  1.8486687e+00,\n",
       "         1.1360471e+00, -8.4355950e-02, -2.7205276e-01, -3.5243776e-01,\n",
       "        -8.7074924e-01,  9.2197478e-01, -1.6891556e+00, -1.2980952e+00,\n",
       "        -5.3385198e-02, -6.4494354e-01,  6.6960633e-02,  4.6848938e-01,\n",
       "        -6.9672108e-01, -1.6785400e+00,  7.6200837e-01, -5.0406647e-01,\n",
       "        -1.4501936e+00,  1.3387250e+00, -5.6099737e-01, -2.6650232e-01,\n",
       "        -3.4384909e-01,  1.5968245e+00, -1.7252556e+00, -2.8877589e-01,\n",
       "         2.3671919e-01, -1.7661674e+00,  1.1558040e+00,  8.8561887e-01,\n",
       "         5.6536603e-01,  1.6616430e+00,  1.5410352e-01, -1.9581079e-02,\n",
       "        -1.4912158e+00,  1.4021204e+00,  9.7034663e-01,  1.5269648e+00,\n",
       "        -6.9160253e-02, -1.2739227e+00, -2.5241894e-01, -1.5882177e+00,\n",
       "        -1.1387055e+00, -1.7391834e+00,  1.9862680e+00,  8.7520087e-01,\n",
       "        -1.0236690e+00,  9.9145275e-01,  1.8478736e-01, -5.5831087e-01,\n",
       "        -8.1992823e-01,  6.1038101e-01,  4.4993043e-02,  1.4730409e+00,\n",
       "         3.2682568e-01,  1.8637949e-01,  1.8340302e-01, -4.0022135e-01]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if (\n",
    "            'Variable' in n.op\n",
    "            or n.name.find('Placeholder') >= 0\n",
    "            or 'add_1' in n.name\n",
    "            or 'attention' in n.name\n",
    "        )\n",
    "        and 'Adam' not in n.name\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Variable',\n",
       " 'dense/kernel',\n",
       " 'dense/bias',\n",
       " 'Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Placeholder_2',\n",
       " 'thought_scope/bidirectional_rnn/fw/gru_cell/gates/kernel',\n",
       " 'thought_scope/bidirectional_rnn/fw/gru_cell/gates/bias',\n",
       " 'thought_scope/bidirectional_rnn/fw/gru_cell/candidate/kernel',\n",
       " 'thought_scope/bidirectional_rnn/fw/gru_cell/candidate/bias',\n",
       " 'thought_scope/bidirectional_rnn/fw/fw/while/add_1/y',\n",
       " 'thought_scope/bidirectional_rnn/fw/fw/while/add_1',\n",
       " 'thought_scope/bidirectional_rnn/bw/gru_cell/gates/kernel',\n",
       " 'thought_scope/bidirectional_rnn/bw/gru_cell/gates/bias',\n",
       " 'thought_scope/bidirectional_rnn/bw/gru_cell/candidate/kernel',\n",
       " 'thought_scope/bidirectional_rnn/bw/gru_cell/candidate/bias',\n",
       " 'thought_scope/bidirectional_rnn/bw/bw/while/add_1/y',\n",
       " 'thought_scope/bidirectional_rnn/bw/bw/while/add_1',\n",
       " 'thought_scope/add_1',\n",
       " 'attention',\n",
       " 'decoder/gru_cell/gates/kernel',\n",
       " 'decoder/gru_cell/gates/bias',\n",
       " 'decoder/gru_cell/candidate/kernel',\n",
       " 'decoder/gru_cell/candidate/bias',\n",
       " 'decoder/while/add_1/y',\n",
       " 'decoder/while/add_1',\n",
       " 'decoder_1/gru_cell/gates/kernel',\n",
       " 'decoder_1/gru_cell/gates/bias',\n",
       " 'decoder_1/gru_cell/candidate/kernel',\n",
       " 'decoder_1/gru_cell/candidate/bias',\n",
       " 'decoder_1/while/add_1/y',\n",
       " 'decoder_1/while/add_1',\n",
       " 'gradients/thought_scope/add_1_grad/Shape',\n",
       " 'gradients/thought_scope/add_1_grad/Shape_1',\n",
       " 'gradients/thought_scope/add_1_grad/BroadcastGradientArgs',\n",
       " 'gradients/thought_scope/add_1_grad/Sum',\n",
       " 'gradients/thought_scope/add_1_grad/Reshape',\n",
       " 'gradients/thought_scope/add_1_grad/Sum_1',\n",
       " 'gradients/thought_scope/add_1_grad/Reshape_1',\n",
       " 'beta1_power',\n",
       " 'beta2_power']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            \"directory: %s\" % model_dir)\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "    \n",
    "    absolute_model_dir = \"/\".join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + \"/frozen_model.pb\"\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph=tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(input_checkpoint + '.meta', clear_devices=clear_devices)\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(\",\")\n",
    "        ) \n",
    "        with tf.gfile.GFile(output_graph, \"wb\") as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print(\"%d ops in the final graph.\" % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from skip/model.ckpt\n",
      "INFO:tensorflow:Froze 21 variables.\n",
      "INFO:tensorflow:Converted 21 variables to const ops.\n",
      "1224 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('skip', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, \"rb\") as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "g=load_graph('skip/frozen_model.pb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py:1702: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    }
   ],
   "source": [
    "x = g.get_tensor_by_name('import/Placeholder_1:0')\n",
    "logits = g.get_tensor_by_name('import/thought_scope/add_1:0')\n",
    "attention = g.get_tensor_by_name('import/attention:0')\n",
    "test_sess = tf.InteractiveSession(graph=g)\n",
    "out, att = test_sess.run([logits,attention], feed_dict={x:fw_input})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 54718)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "att.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "rev_dict = {v: k for k, v in dictionary.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49104\n",
      "menjebaknya\n",
      "54\n",
      "seperti\n",
      "5951\n",
      "gunanya\n",
      "41221\n",
      "hawar\n",
      "6333\n",
      "ganjaran\n",
      "27612\n",
      "dayangku\n",
      "33504\n",
      "pijak\n",
      "44119\n",
      "parol\n",
      "43996\n",
      "poupart\n",
      "22753\n",
      "scb\n"
     ]
    }
   ],
   "source": [
    "for i in att[0].argsort()[-10:][::-1]:\n",
    "    print(i)\n",
    "    print(rev_dict[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
