{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import re\n",
    "import sentencepiece as spm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from prepro_utils import preprocess_text, encode_ids, encode_pieces\n",
    "\n",
    "sp_model = spm.SentencePieceProcessor()\n",
    "sp_model.Load('sp10m.cased.bert.model')\n",
    "\n",
    "with open('sp10m.cased.bert.vocab') as fopen:\n",
    "    v = fopen.read().split('\\n')[:-1]\n",
    "v = [i.split('\\t') for i in v]\n",
    "v = {i[0]: i[1] for i in v}\n",
    "\n",
    "class Tokenizer:\n",
    "    def __init__(self, v):\n",
    "        self.vocab = v\n",
    "        pass\n",
    "    \n",
    "    def tokenize(self, string):\n",
    "        return encode_pieces(sp_model, string, return_unicode=False, sample=False)\n",
    "    \n",
    "    def convert_tokens_to_ids(self, tokens):\n",
    "        return [sp_model.PieceToId(piece) for piece in tokens]\n",
    "    \n",
    "    def convert_ids_to_tokens(self, ids):\n",
    "        return [sp_model.IdToPiece(i) for i in ids]\n",
    "    \n",
    "tokenizer = Tokenizer(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/optimization.py:87: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import bert\n",
    "from bert import run_classifier\n",
    "from bert import optimization\n",
    "from bert import tokenization\n",
    "from bert import modeling\n",
    "import numpy as np\n",
    "import json\n",
    "import tensorflow as tf\n",
    "import itertools\n",
    "from unidecode import unidecode\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_INIT_CHKPNT = 'bert-base-v3/model.ckpt'\n",
    "BERT_CONFIG = 'bert-base-v3/config.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['anger', 'fear', 'happy', 'love', 'sadness', 'surprise'])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "import random\n",
    "\n",
    "emotion_label = ['anger', 'fear', 'happy', 'love', 'sadness', 'surprise']\n",
    "\n",
    "with open('../sentiment/emotion-twitter-lexicon.json') as fopen:\n",
    "    emotion = json.load(fopen)\n",
    "    \n",
    "emotion.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "anger 30000\n",
      "fear 20316\n",
      "happy 30000\n",
      "love 20783\n",
      "sadness 26468\n",
      "surprise 13107\n"
     ]
    }
   ],
   "source": [
    "texts, labels = [], []\n",
    "\n",
    "for k, v in emotion.items():\n",
    "    if len(v) > 30000:\n",
    "        emotion[k] = random.sample(v, 30000)\n",
    "    print(k, len(emotion[k]))\n",
    "    texts.extend(emotion[k])\n",
    "    labels.extend([emotion_label.index(k)] * len(emotion[k]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[50316, 50317, 50318, 50319, 50320]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i_s = [no for no, i in enumerate(labels) if i == 2]\n",
    "i_s[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Takut kehilangan kau',\n",
       " 'nak tidur kejap petang pun takut sampai set alarm every 20min',\n",
       " 'Netizenz: Pul, buatlah lagu untuk P*ndan tak takut Tuhan tu.\\n\\nSeorang Aepul: Say no more, fam. https://t.co/amycMwQW8d',\n",
       " 'Enak banget pegang hp ga dipakein case, tapi aku takut tiba tiba tangan aku ga sadar klo lagi megang hp',\n",
       " '@Kunto_Drummer Lihat petanya ngeri mas, Kalimantan Barat dari data penyebaran tidak sebanyak itu, Gubernur tegas, s https://t.co/mBKfgCbgE2',\n",
       " 'Sial seram bodo org sebelah ni gelak',\n",
       " '@delicylv Pas baca ini jadi takut pilih gerbong yg sepi. Padahal kalau naik kereta biasanya pilih yg malam, dan ger https://t.co/pBRTrdhqJl',\n",
       " 'Kangen banget jam segini pergi ke kopmer, perjalanannya ituloh ngeri ngeri sedap ',\n",
       " 'Hujan masih air\\nYa emang hujan itu air\\nCoba aja bayangin kalau hujan batu\\nKan ngeri lor',\n",
       " '@radyaab @la_neey wehh BEM KM sii ngeri ngerii',\n",
       " 'best, tapi sumpah seram nak mampus',\n",
       " '@iamyourhonestly @jawafess jgn buat aku takut kak ::',\n",
       " 'Orang-orang udah pada nongkrong lagi. Padahal prediksi dokter dirga, puncak pandemi di indonesia akan terlihat dalam empat pekan ke depan. Apa nggak ngeri tuh masih aja bandel.',\n",
       " 'Perasaan cemburu dan takut kehilangan mesti lagi cepat rasa bila kita dah semakin sayang dekat someone tu.',\n",
       " 'Perasaan cemburu dan takut kehilangan mesti lagi cepat rasa bila kita dah semakin sayang dekat someone tu.',\n",
       " 'Ngeri ke gep kawan rl kalo pake ava asli hikss',\n",
       " 'Ada beberapa hal yang bikin aku takut dan aku gak mau lagi,\\nada beberapa kalimat yang selalu ada di pikiran aku, \\ns https://t.co/sDvjejNDdk',\n",
       " 'Orang-orang udah pada nongkrong lagi. Padahal prediksi dokter dirga, puncak pandemi di indonesia akan terlihat dalam empat pekan ke depan. Apa nggak ngeri tuh masih aja bandel.',\n",
       " '@bumblessoul Aku takut',\n",
       " '*THREAD*\\nOkay saya nak main psikologi sedikit harini so harap mutuals saya sporting la.\\n\\nPintu manakah yang korang takut nak masuk? https://t.co/Oyeo8pL6RC',\n",
       " 'Mau di-screenshot kayak gimana juga, janji tinggal janji. Yang katanya berjuang bareng-bareng, pas sukses ninggalin. Yang katanya takut kehilangan ujung-ujungnya bisa menghilang. Yang katanya aku tak bisa hidup tanpamu buktinya masih hidup sehat walafiat',\n",
       " 'Perasaan cemburu dan takut kehilangan mesti lagi cepat rasa bila kita dah semakin sayang dekat someone tu.',\n",
       " 'Mau di-screenshot kayak gimana juga, janji tinggal janji. Yang katanya berjuang bareng-bareng, pas sukses ninggalin. Yang katanya takut kehilangan ujung-ujungnya bisa menghilang. Yang katanya aku tak bisa hidup tanpamu buktinya masih hidup sehat walafiat',\n",
       " 'Kenapa aku takut sgt ni nk kelas 435 ',\n",
       " 'Aku Takut Kucing Aku Duduk Atas Dia ',\n",
       " '@Bimaadii_ Aku takut kamu baperin bim, aku lemahan',\n",
       " '@sellyagusriani Mamiii parah bgttt :( aku jadi takut nii, maknya aku langsung block :(',\n",
       " \"@C_ArielJKT48 Ip @Eve_JKT48 nitip salam ya buat kakaknya. Tolong bilangin Ci Eril jangan lucuk2...... aku takut......... \\n\\nsynk.. &gt;_&lt;'\",\n",
       " 'Ye doh. Benda macam ni lah aku takut. Masa sebelum kahwin memanglah tak nampak. Dah kahwin, ko tahulah cemana... mi https://t.co/985jCcSTiY',\n",
       " 'Yang aku takutin dari jarak itu, aku takut kamu di sana diem-diem jalan sama yang lain.',\n",
       " '@Lovedy0597_ Takut bgt.... takuttttt..... mutualku aku takut siw',\n",
       " '*THREAD*\\nOkay saya nak main psikologi sedikit harini so harap mutuals saya sporting la.\\n\\nPintu manakah yang korang takut nak masuk? https://t.co/Oyeo8pL6RC',\n",
       " '2x anjic direp rep  ya Allah aku takut :((',\n",
       " 'tanyarl  aku takut banget ga bisa ngebahagiain ortu aku :(',\n",
       " '\"Jangan penjarakan ucapanmu\\njika kau menjadi budak\\npada ketakutan\\nkarna kami akan\\nmemperpanjang ketakutan\"\\n\\nPernah suara dibungkam\\njadi anak ayam mau disangkar\\ntapi tak bisa mereka\\nmenghapus sajak-sajakku. https://t.co/685xB2bs7g',\n",
       " 'Kadang kamu sulit ungkapkan perasaanmu tentang seseorang, karena kamu takut kehilangan apa yg telah kamu dan dia miliki saat ini.',\n",
       " 'Perasaan cemburu dan takut kehilangan mesti lagi cepat rasa bila kita dah semakin sayang dekat someone tu.',\n",
       " 'Perasaan cemburu dan takut kehilangan mesti lagi cepat rasa bila kita dah semakin sayang dekat someone tu.',\n",
       " '\"Tuhkan! Mas @moektito aja sudah ikut gerakan #MaskerKainUntukSemua ! Kok kamu belum sih? Masih ragu dan bertanya-tanya kegunaannya? Bisa langsung cek https://t.co/lQBGujWqyD ! Jadi, tunjukkan masker kainmu ya, #SobatKUKM . Karena maskerku melindungimu, dan maskermu melindungiku. https://t.co/iRy8d23ywS',\n",
       " '@Telkomsel Ngeri x emang makan paketnya ah...\\nGila emang.....\\nParah....\\nCepat bangat habisnya\\nKaya makan kacang gor https://t.co/rk2mq40RmZ',\n",
       " '@RPWAutoBase @nctjahyun @huangrwenjwun @najaemln___ @nevanfaey @doyoungenct @taeyongdl @parkjisunc @zchenlx @Jxeejeno @yangyamg_x2 @CHITTA10_ @jonhnnyjsuh @Taeidl @siceung @yukheyc gua mau desc hmm\\nnyantuy gdm paling penuh suasana men, seneng, galau, sedih, ambyar, melow, julid, mabok dangdut. gdm yg selalu gua cek, rumah bgt men, mau blg syg tapi lo pd ngeri kegeeran, rame terus yaa',\n",
       " 'Banyak yang lebih takut kehilangan sekitar dari pada kehilangan diri sendiri saat berada di sekitar\\n\\nrasa.ku~ @ Si https://t.co/HHaouysP6B',\n",
       " '@mxtchagurl tapi kiat ndak ya kainnya, aku takut nanti robek :(',\n",
       " 'Obsesi atau apa itu?? Iiihh parah sih ini manusia, kok jadi ngeri sendiri w',\n",
       " '@aphrodei AKU TAKUT ALLAH',\n",
       " '@NyiGiri @ustadzmaaher_ stad tukang misuh misuh\\ngilo aku lihat mata nya\\nngeri aku sma mulut nya yg ngalahin tinggi https://t.co/KIRUIqKndV',\n",
       " '@amirannadiah @aimanadib87 Haah do lupa lak aku takut ngn kau je ',\n",
       " 'Oh god... I really need to stop watching horror/thriller movies... with my sister  aku paksa dia tgk sbb takut nak https://t.co/1yLP8dwDJ6',\n",
       " '@fiiraa22 aku takut latah..',\n",
       " '@ShafiraTR @deviagrni ngeri besoknya di bales di batu. Tulang bungkus kulit tbtb nyampe aja di pipi terbangnya gampang bgt lagi',\n",
       " '@hiunzae Apa kau ketakutan? Tenang saja disini ada aku yang akan selalu melindungimu..',\n",
       " 'Kriminalitas kembali terjadi.\\nKali ini siang2 Menjambret Ibu dan anak. \\n\\nSelamat sore pak @jokowi Ketakutan Masyarakat skrg jadi Double pak. Sudah kita diintai Corona Pembunuh, skrg diancam Penjahat pulak. \\n\\nTolong kandangi lagi mereka pak.\\n https://t.co/yqhwqw201D',\n",
       " 'Aku cemburu tandanya aku takut kehilangan kamu.',\n",
       " 'Yang aku takutkan bukan kau meninggalkanku karena ada yang lebih baik dariku, tapi aku takut kau pergi karena tak lagi menemukan alasan untuk tinggal.',\n",
       " 'Sumpah aku takut dengan laki zaman sekarangni :)',\n",
       " 'Aku diam bukan berarti aku gak perduli, tapi aku takut kamu terganggu dengan tingkahku\\nDan aku menjaga itu\\n\\n@GuyonWaton @aswkabeh1 @Kulinobareng_ @NasibmuPiyeee @kulinodewe_ @Jowohits @YowessBUBAR https://t.co/GDqCXG27iF',\n",
       " 'aku takut nanti bila dah sambung belajar mata aku terkejut je sebab jadual tidur dah serabut',\n",
       " 'Kriminalitas kembali terjadi.\\nKali ini siang2 Menjambret Ibu dan anak. \\n\\nSelamat sore pak @jokowi Ketakutan Masyarakat skrg jadi Double pak. Sudah kita diintai Corona Pembunuh, skrg diancam Penjahat pulak. \\n\\nTolong kandangi lagi mereka pak.\\n https://t.co/yqhwqw201D',\n",
       " '@subtanyarl Karena keadaan pandemi seperti sekarang aku takut nantinya orang akan terbiasa dan biasa aja. I mean ya https://t.co/k7fLPPRpum',\n",
       " '@minnyies AKU MAU SPILL TAPI AKU TAKUT DIAMUK KAMU HUEKS.',\n",
       " 'Lipas terbang ajak aku fight. Tapi aku takut',\n",
       " 'KORANG YG BAIK HATI TU TOLONG REPORT ACC IG NI !! ADA MANUSIA YG TAK PERIKEMANUSIAAN YG HACK IG NI !! DLM ACC TU ADA HAL PERIBADI AQ !! AQ TAKUT HACKER TU BUAT YG BUKAN II .. SO MINTAK TOLONG SANGAT II REPORT ACC NI  https://t.co/tsxopZOJNe',\n",
       " 'Aku sorang je ke takut nak cabut gigi masa kecik',\n",
       " '@Lenny_diary Aku juga sering kek gini karena ngerasa gatell yaampun jadi takut',\n",
       " 'okay since dah ramai yang rt aku takut ada yang dah menunggu pula. Mungkin ada yang dah pernah nampak gambar ni which is pernah jadi headline surat khabar Berita Harian pada tahun 1995 dulu https://t.co/c2CKj7hLyi',\n",
       " 'Ngeri wallah',\n",
       " '@AzriAsnawi Sapa ada lagi eh time tu.. Tapi seram babi aku still ingat do',\n",
       " '/rlt/ gaes tolongin aku buat nahan diri biar ga suka sama temen sekelasku   soalnya aku udah ada cowo. Aku takut https://t.co/SE75TJ5FaD',\n",
       " 'okay since dah ramai yang rt aku takut ada yang dah menunggu pula. Mungkin ada yang dah pernah nampak gambar ni which is pernah jadi headline surat khabar Berita Harian pada tahun 1995 dulu https://t.co/c2CKj7hLyi',\n",
       " 'Aku ada rasa nak betulkan hubungan ni tapi dalam masa yang sama, aku takut aku terluka lagi. Dengan sifat aku yang sensitive ni, aku tak yakin yang aku mampu hadapi sakit tu lagi.',\n",
       " 'jagalah ketakwaan pada Allah swt,,,Apa arti tawakal jika kita tidak bertakwa,\\n\\njangan ketakutan dg ujian,cobaan,mus https://t.co/wdvW31Wte5',\n",
       " 'Cemburu karena sayang, khawatir karena takut kehilangan. Semua boleh dilakukan, asal jangan berlebihan.\"',\n",
       " \"Ada ketakutan saya selama ini. Di kala banyak orang meng'glorofikasi' anak muda yang bagus itu harus pengusaha, ana https://t.co/GuDhr8zN0m\",\n",
       " 'Dan sungguh akan Kami berikan cobaan kepadamu, dengan sedikit ketakutan, kelaparan, kekurangan harta, jiwa dan buah-buahan. Dan berikanlah berita gembira kepada orang-orang yang sabar. (2:155)',\n",
       " 'okay since dah ramai yang rt aku takut ada yang dah menunggu pula. Mungkin ada yang dah pernah nampak gambar ni which is pernah jadi headline surat khabar Berita Harian pada tahun 1995 dulu https://t.co/c2CKj7hLyi',\n",
       " 'Dan sungguh akan Kami berikan cobaan kepadamu, dengan sedikit ketakutan, kelaparan, kekurangan harta, jiwa dan buah-buahan. Dan berikanlah berita gembira kepada orang-orang yang sabar. (2:155)',\n",
       " 'Dan sungguh akan Kami berikan cobaan kepadamu, dengan sedikit ketakutan, kelaparan, kekurangan harta, jiwa dan buah-buahan. Dan berikanlah berita gembira kepada orang-orang yang sabar. (2:155)',\n",
       " 'Mmg tk laa test covid. Ngeri cukup',\n",
       " 'Tazdkirah\\nManusia yg tdk beriman sungguh amat ambisius tapi hilang hatinya manakala dalam ketakutan seolah esok tak ada hari cerah. Manakala musibah menimpa banyak berkeluh kesah. Manakala mendapat anugrah kenikmatan sungguh amat sulit berderma. https://t.co/NSFZR8548E',\n",
       " 'Dan sungguh akan Kami berikan cobaan kepadamu, dengan sedikit ketakutan, kelaparan, kekurangan harta, jiwa dan buah-buahan. Dan berikanlah berita gembira kepada orang-orang yang sabar. (2:155)',\n",
       " '\"Takut Kehilangan\"\\n\\nSebuah alasan yang diberlaku logiskan oleh orang yang tengah larut dalam cinta.',\n",
       " 'Mau di-screenshot kayak gimana juga, janji tinggal janji. Yang katanya berjuang bareng-bareng, pas sukses ninggalin. Yang katanya takut kehilangan ujung-ujungnya bisa menghilang. Yang katanya aku tak bisa hidup tanpamu buktinya masih hidup sehat walafiat',\n",
       " '@JeromePolin lebih ngeri kalau centang 2 biru tapi di abaikan',\n",
       " 'Dan sungguh akan Kami berikan cobaan kepadamu, dengan sedikit ketakutan, kelaparan, kekurangan harta, jiwa dan buah-buahan. Dan berikanlah berita gembira kepada orang-orang yang sabar. (2:155)',\n",
       " 'Gw jadi takut ibu gw kenapa kenapa',\n",
       " 'Aku takut lg nk bkk doc msm lxw assessments baru siap bg  hm penat bincg ritu wlaupon pendek 3x gak ah tukar https://t.co/zf35uBqK9W',\n",
       " 'Banyak nih kayak gini ngaku2 pejabat biar orang menengah dibikin down ketakutan, tapi kalo kita bener mah ngingetin https://t.co/Ub74zW8WDk',\n",
       " '@collegemenfess Aku kuliah pendidikan tp masih ragu mau jadi guru apa engga... Hm',\n",
       " 'Flis ak nga punya temen nonton horror, ak dan adekku cupu',\n",
       " 'Orang-orang udah pada nongkrong lagi. Padahal prediksi dokter dirga, puncak pandemi di indonesia akan terlihat dalam empat pekan ke depan. Apa nggak ngeri tuh masih aja bandel.',\n",
       " '@ENAEVENTT agak ngeri\\nliat liat ke batu pinggiran dan atas',\n",
       " 'Pernah. Aku takut nak berharap. Takut nak percaya. Takut nk mula semula. Sbb aku takut ditinggalkn. Takut dilupakan.',\n",
       " 'Mau di-screenshot kayak gimana juga, janji tinggal janji. Yang katanya berjuang bareng-bareng, pas sukses ninggalin. Yang katanya takut kehilangan ujung-ujungnya bisa menghilang. Yang katanya aku tak bisa hidup tanpamu buktinya masih hidup sehat walafiat',\n",
       " '@icegulali @hellxtear @tautaululus @kikiselalubenar @BERBURUSALE_ Tapi aku takut kalo salah harga..:))',\n",
       " 'Katanya kalo mau viral suruh minta bantuan magic, tapi aku takut syirik. Jadi aku woro-woro aja kalau aku dan ibuk jualan MASAKAN IBUK \\nIG-nya @maemasakanibuk, spesial ayam ungkep kuning Rp30.000,00 / 2 potong paha tepong. Ready Jogja bisa PO via DM  https://t.co/2P3xv6jeuL',\n",
       " 'Aku ni nak membeli seluar mengandung. Tapi aku takut beli online. Tapi nak pegi kedai mana ada kedai baju yg bukak. https://t.co/Z3qnCPVZgM',\n",
       " 'Yang aku takut, aku membenci dosa orang lain, namun saat aku berbuat dosa aku enggan membencinya...',\n",
       " '@wanmox tapi mahal ya allah yahudi pun takut bila tgn rege!',\n",
       " '@estoneecha aku jadi takut gimana coba udah kek dewasa bangett',\n",
       " 'Dan sungguh akan Kami berikan cobaan kepadamu, dengan sedikit ketakutan, kelaparan, kekurangan harta, jiwa dan buah-buahan. Dan berikanlah berita gembira kepada orang-orang yang sabar. (2:155)']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[48656:48656 + 100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rules import normalized_chars\n",
    "import random\n",
    "\n",
    "laughing = {\n",
    "    'huhu',\n",
    "    'haha',\n",
    "    'gagaga',\n",
    "    'hihi',\n",
    "    'wkawka',\n",
    "    'wkwk',\n",
    "    'kiki',\n",
    "    'keke',\n",
    "    'huehue',\n",
    "    'hshs',\n",
    "    'hoho',\n",
    "    'hewhew',\n",
    "    'uwu',\n",
    "    'sksk',\n",
    "    'ksks',\n",
    "    'gituu',\n",
    "    'gitu',\n",
    "    'mmeeooww',\n",
    "    'meow',\n",
    "    'alhamdulillah',\n",
    "    'muah',\n",
    "    'mmuahh',\n",
    "    'hehe',\n",
    "    'salamramadhan',\n",
    "    'happywomensday',\n",
    "    'jahagaha',\n",
    "    'ahakss',\n",
    "    'ahksk'\n",
    "}\n",
    "\n",
    "def make_cleaning(s, c_dict):\n",
    "    s = s.translate(c_dict)\n",
    "    return s\n",
    "\n",
    "def cleaning(string):\n",
    "    \"\"\"\n",
    "    use by any transformer model before tokenization\n",
    "    \"\"\"\n",
    "    string = unidecode(string)\n",
    "    \n",
    "    string = ' '.join(\n",
    "        [make_cleaning(w, normalized_chars) for w in string.split()]\n",
    "    )\n",
    "    string = re.sub('\\(dot\\)', '.', string)\n",
    "    string = (\n",
    "        re.sub(re.findall(r'\\<a(.*?)\\>', string)[0], '', string)\n",
    "        if (len(re.findall(r'\\<a (.*?)\\>', string)) > 0)\n",
    "        and ('href' in re.findall(r'\\<a (.*?)\\>', string)[0])\n",
    "        else string\n",
    "    )\n",
    "    string = re.sub(\n",
    "        r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', ' ', string\n",
    "    )\n",
    "    \n",
    "    chars = '.,/'\n",
    "    for c in chars:\n",
    "        string = string.replace(c, f' {c} ')\n",
    "        \n",
    "    string = re.sub(r'[ ]+', ' ', string).strip().split()\n",
    "    string = [w for w in string if w[0] != '@']\n",
    "    x = []\n",
    "    for word in string:\n",
    "        word = word.lower()\n",
    "        if any([laugh in word for laugh in laughing]):\n",
    "            if random.random() >= 0.5:\n",
    "                x.append(word)\n",
    "        else:\n",
    "            x.append(word)\n",
    "    string = [w.title() if w[0].isupper() else w for w in x]\n",
    "    return ' '.join(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 140674/140674 [00:12<00:00, 11682.82it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "for i in tqdm(range(len(texts))):\n",
    "    texts[i] = cleaning(texts[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 140674/140674 [00:00<00:00, 1230576.68it/s]\n"
     ]
    }
   ],
   "source": [
    "actual_t, actual_l = [], []\n",
    "\n",
    "for i in tqdm(range(len(texts))):\n",
    "    if len(texts[i]) > 2:\n",
    "        actual_t.append(texts[i])\n",
    "        actual_l.append(labels[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 140674/140674 [00:16<00:00, 8533.44it/s] \n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "input_ids, input_masks = [], []\n",
    "\n",
    "for text in tqdm(actual_t):\n",
    "    tokens_a = tokenizer.tokenize(text)\n",
    "    tokens = [\"[CLS]\"] + tokens_a + [\"[SEP]\"]\n",
    "    input_id = tokenizer.convert_tokens_to_ids(tokens)\n",
    "    input_mask = [1] * len(input_id)\n",
    "    \n",
    "    input_ids.append(input_id)\n",
    "    input_masks.append(input_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:93: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch = 3\n",
    "batch_size = 60\n",
    "warmup_proportion = 0.1\n",
    "num_train_steps = int(len(texts) / batch_size * epoch)\n",
    "num_warmup_steps = int(num_train_steps * warmup_proportion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "768"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bert_config.hidden_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_initializer(initializer_range=0.02):\n",
    "    return tf.truncated_normal_initializer(stddev=initializer_range)\n",
    "\n",
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        dimension_output,\n",
    "        learning_rate = 2e-5,\n",
    "        training = True,\n",
    "    ):\n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.MASK = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None])\n",
    "        \n",
    "        model = modeling.BertModel(\n",
    "            config=bert_config,\n",
    "            is_training=training,\n",
    "            input_ids=self.X,\n",
    "            input_mask=self.MASK,\n",
    "            use_one_hot_embeddings=False)\n",
    "        \n",
    "        output_layer = model.get_sequence_output()\n",
    "        output_layer = tf.layers.dense(\n",
    "            output_layer,\n",
    "            bert_config.hidden_size,\n",
    "            activation=tf.tanh,\n",
    "            kernel_initializer=create_initializer())\n",
    "        self.logits_seq = tf.layers.dense(output_layer, dimension_output,\n",
    "                                         kernel_initializer=create_initializer())\n",
    "        self.logits_seq = tf.identity(self.logits_seq, name = 'logits_seq')\n",
    "        self.logits = self.logits_seq[:, 0]\n",
    "        self.logits = tf.identity(self.logits, name = 'logits')\n",
    "        \n",
    "        self.cost = tf.reduce_mean(\n",
    "            tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "                logits = self.logits, labels = self.Y\n",
    "            )\n",
    "        )\n",
    "        \n",
    "        self.optimizer = optimization.create_optimizer(self.cost, learning_rate, \n",
    "                                                       num_train_steps, num_warmup_steps, False)\n",
    "        correct_pred = tf.equal(\n",
    "            tf.argmax(self.logits, 1, output_type = tf.int32), self.Y\n",
    "        )\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:171: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:409: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:490: The name tf.assert_less_equal is deprecated. Please use tf.compat.v1.assert_less_equal instead.\n",
      "\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:358: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/optimization.py:27: The name tf.train.get_or_create_global_step is deprecated. Please use tf.compat.v1.train.get_or_create_global_step instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/optimization.py:32: The name tf.train.polynomial_decay is deprecated. Please use tf.compat.v1.train.polynomial_decay instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/optimization.py:70: The name tf.trainable_variables is deprecated. Please use tf.compat.v1.trainable_variables instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/math_grad.py:1375: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "INFO:tensorflow:Restoring parameters from bert-base-v3/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "dimension_output = 6\n",
    "learning_rate = 2e-5\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(\n",
    "    dimension_output,\n",
    "    learning_rate\n",
    ")\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = 'bert')\n",
    "saver = tf.train.Saver(var_list = var_lists)\n",
    "saver.restore(sess, BERT_INIT_CHKPNT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_input_ids, test_input_ids, train_Y, test_Y, train_mask, test_mask = train_test_split(\n",
    "    input_ids, actual_l, input_masks, test_size = 0.2\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad_sequences = tf.keras.preprocessing.sequence.pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 1876/1876 [10:52<00:00,  2.88it/s, accuracy=1, cost=0.00116]   \n",
      "test minibatch loop: 100%|██████████| 469/469 [00:55<00:00,  8.43it/s, accuracy=1, cost=0.00154]   \n",
      "train minibatch loop:   0%|          | 0/1876 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, training loss: 0.116339, training acc: 0.964241, valid loss: 0.019386, valid acc: 0.995274\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 1876/1876 [10:47<00:00,  2.90it/s, accuracy=1, cost=0.00045]   \n",
      "test minibatch loop: 100%|██████████| 469/469 [00:55<00:00,  8.51it/s, accuracy=1, cost=0.000458]  \n",
      "train minibatch loop:   0%|          | 0/1876 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, training loss: 0.010469, training acc: 0.997646, valid loss: 0.013054, valid acc: 0.997015\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 1876/1876 [10:46<00:00,  2.90it/s, accuracy=1, cost=0.000191]  \n",
      "test minibatch loop: 100%|██████████| 469/469 [00:55<00:00,  8.51it/s, accuracy=1, cost=0.000176]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2, training loss: 0.003887, training acc: 0.999200, valid loss: 0.009946, valid acc: 0.998081\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import time\n",
    "\n",
    "for EPOCH in range(epoch):\n",
    "\n",
    "    train_acc, train_loss, test_acc, test_loss = [], [], [], []\n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_input_ids), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_input_ids))\n",
    "        batch_x = train_input_ids[i: index]\n",
    "        batch_x = pad_sequences(batch_x, padding='post')\n",
    "        batch_mask = train_mask[i: index]\n",
    "        batch_mask = pad_sequences(batch_mask, padding='post')\n",
    "        batch_y = train_Y[i: index]\n",
    "        acc, cost, _ = sess.run(\n",
    "            [model.accuracy, model.cost, model.optimizer],\n",
    "            feed_dict = {\n",
    "                model.Y: batch_y,\n",
    "                model.X: batch_x,\n",
    "                model.MASK: batch_mask\n",
    "            },\n",
    "        )\n",
    "        train_loss.append(cost)\n",
    "        train_acc.append(acc)\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "        \n",
    "    pbar = tqdm(range(0, len(test_input_ids), batch_size), desc = 'test minibatch loop')\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_input_ids))\n",
    "        batch_x = test_input_ids[i: index]\n",
    "        batch_x = pad_sequences(batch_x, padding='post')\n",
    "        batch_mask = test_mask[i: index]\n",
    "        batch_mask = pad_sequences(batch_mask, padding='post')\n",
    "        batch_y = test_Y[i: index]\n",
    "        acc, cost = sess.run(\n",
    "            [model.accuracy, model.cost],\n",
    "            feed_dict = {\n",
    "                model.Y: batch_y,\n",
    "                model.X: batch_x,\n",
    "                model.MASK: batch_mask\n",
    "            },\n",
    "        )\n",
    "        test_loss.append(cost)\n",
    "        test_acc.append(acc)\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "        \n",
    "    train_loss = np.mean(train_loss)\n",
    "    train_acc = np.mean(train_acc)\n",
    "    test_loss = np.mean(test_loss)\n",
    "    test_acc = np.mean(test_acc)\n",
    "    \n",
    "    print(\n",
    "        'epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'\n",
    "        % (EPOCH, train_loss, train_acc, test_loss, test_acc)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'bert-base-emotion/model.ckpt'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'bert-base-emotion/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py:1750: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from bert-base-emotion/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "dimension_output = 6\n",
    "learning_rate = 2e-5\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(\n",
    "    dimension_output,\n",
    "    learning_rate,\n",
    "    training = False\n",
    ")\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.restore(sess, 'bert-base-emotion/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Placeholder_2',\n",
       " 'bert/embeddings/word_embeddings',\n",
       " 'bert/embeddings/token_type_embeddings',\n",
       " 'bert/embeddings/position_embeddings',\n",
       " 'bert/embeddings/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_0/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/query/bias',\n",
       " 'bert/encoder/layer_0/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/key/bias',\n",
       " 'bert/encoder/layer_0/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/value/bias',\n",
       " 'bert/encoder/layer_0/attention/self/Softmax',\n",
       " 'bert/encoder/layer_0/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_0/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_0/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_0/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_0/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_0/output/dense/kernel',\n",
       " 'bert/encoder/layer_0/output/dense/bias',\n",
       " 'bert/encoder/layer_0/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_1/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/query/bias',\n",
       " 'bert/encoder/layer_1/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/key/bias',\n",
       " 'bert/encoder/layer_1/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/value/bias',\n",
       " 'bert/encoder/layer_1/attention/self/Softmax',\n",
       " 'bert/encoder/layer_1/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_1/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_1/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_1/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_1/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_1/output/dense/kernel',\n",
       " 'bert/encoder/layer_1/output/dense/bias',\n",
       " 'bert/encoder/layer_1/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_2/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/query/bias',\n",
       " 'bert/encoder/layer_2/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/key/bias',\n",
       " 'bert/encoder/layer_2/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/value/bias',\n",
       " 'bert/encoder/layer_2/attention/self/Softmax',\n",
       " 'bert/encoder/layer_2/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_2/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_2/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_2/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_2/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_2/output/dense/kernel',\n",
       " 'bert/encoder/layer_2/output/dense/bias',\n",
       " 'bert/encoder/layer_2/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_3/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/query/bias',\n",
       " 'bert/encoder/layer_3/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/key/bias',\n",
       " 'bert/encoder/layer_3/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/value/bias',\n",
       " 'bert/encoder/layer_3/attention/self/Softmax',\n",
       " 'bert/encoder/layer_3/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_3/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_3/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_3/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_3/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_3/output/dense/kernel',\n",
       " 'bert/encoder/layer_3/output/dense/bias',\n",
       " 'bert/encoder/layer_3/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_4/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_4/attention/self/query/bias',\n",
       " 'bert/encoder/layer_4/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_4/attention/self/key/bias',\n",
       " 'bert/encoder/layer_4/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_4/attention/self/value/bias',\n",
       " 'bert/encoder/layer_4/attention/self/Softmax',\n",
       " 'bert/encoder/layer_4/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_4/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_4/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_4/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_4/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_4/output/dense/kernel',\n",
       " 'bert/encoder/layer_4/output/dense/bias',\n",
       " 'bert/encoder/layer_4/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_5/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_5/attention/self/query/bias',\n",
       " 'bert/encoder/layer_5/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_5/attention/self/key/bias',\n",
       " 'bert/encoder/layer_5/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_5/attention/self/value/bias',\n",
       " 'bert/encoder/layer_5/attention/self/Softmax',\n",
       " 'bert/encoder/layer_5/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_5/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_5/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_5/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_5/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_5/output/dense/kernel',\n",
       " 'bert/encoder/layer_5/output/dense/bias',\n",
       " 'bert/encoder/layer_5/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_6/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_6/attention/self/query/bias',\n",
       " 'bert/encoder/layer_6/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_6/attention/self/key/bias',\n",
       " 'bert/encoder/layer_6/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_6/attention/self/value/bias',\n",
       " 'bert/encoder/layer_6/attention/self/Softmax',\n",
       " 'bert/encoder/layer_6/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_6/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_6/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_6/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_6/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_6/output/dense/kernel',\n",
       " 'bert/encoder/layer_6/output/dense/bias',\n",
       " 'bert/encoder/layer_6/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_7/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_7/attention/self/query/bias',\n",
       " 'bert/encoder/layer_7/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_7/attention/self/key/bias',\n",
       " 'bert/encoder/layer_7/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_7/attention/self/value/bias',\n",
       " 'bert/encoder/layer_7/attention/self/Softmax',\n",
       " 'bert/encoder/layer_7/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_7/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_7/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_7/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_7/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_7/output/dense/kernel',\n",
       " 'bert/encoder/layer_7/output/dense/bias',\n",
       " 'bert/encoder/layer_7/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_8/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_8/attention/self/query/bias',\n",
       " 'bert/encoder/layer_8/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_8/attention/self/key/bias',\n",
       " 'bert/encoder/layer_8/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_8/attention/self/value/bias',\n",
       " 'bert/encoder/layer_8/attention/self/Softmax',\n",
       " 'bert/encoder/layer_8/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_8/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_8/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_8/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_8/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_8/output/dense/kernel',\n",
       " 'bert/encoder/layer_8/output/dense/bias',\n",
       " 'bert/encoder/layer_8/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_9/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_9/attention/self/query/bias',\n",
       " 'bert/encoder/layer_9/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_9/attention/self/key/bias',\n",
       " 'bert/encoder/layer_9/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_9/attention/self/value/bias',\n",
       " 'bert/encoder/layer_9/attention/self/Softmax',\n",
       " 'bert/encoder/layer_9/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_9/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_9/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_9/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_9/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_9/output/dense/kernel',\n",
       " 'bert/encoder/layer_9/output/dense/bias',\n",
       " 'bert/encoder/layer_9/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_10/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_10/attention/self/query/bias',\n",
       " 'bert/encoder/layer_10/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_10/attention/self/key/bias',\n",
       " 'bert/encoder/layer_10/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_10/attention/self/value/bias',\n",
       " 'bert/encoder/layer_10/attention/self/Softmax',\n",
       " 'bert/encoder/layer_10/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_10/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_10/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_10/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_10/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_10/output/dense/kernel',\n",
       " 'bert/encoder/layer_10/output/dense/bias',\n",
       " 'bert/encoder/layer_10/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_11/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_11/attention/self/query/bias',\n",
       " 'bert/encoder/layer_11/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_11/attention/self/key/bias',\n",
       " 'bert/encoder/layer_11/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_11/attention/self/value/bias',\n",
       " 'bert/encoder/layer_11/attention/self/Softmax',\n",
       " 'bert/encoder/layer_11/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_11/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_11/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_11/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_11/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_11/output/dense/kernel',\n",
       " 'bert/encoder/layer_11/output/dense/bias',\n",
       " 'bert/encoder/layer_11/output/LayerNorm/gamma',\n",
       " 'bert/pooler/dense/kernel',\n",
       " 'bert/pooler/dense/bias',\n",
       " 'dense/kernel',\n",
       " 'dense/bias',\n",
       " 'dense_1/kernel',\n",
       " 'dense_1/bias',\n",
       " 'logits_seq',\n",
       " 'logits',\n",
       " 'gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_11/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_11/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_11/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_10/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_10/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_10/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_9/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_9/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_9/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_8/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_8/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_8/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_7/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_7/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_7/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_6/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_6/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_6/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_5/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_5/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_5/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_4/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_4/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_4/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul_1']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm as tqdm_base\n",
    "def tqdm(*args, **kwargs):\n",
    "    if hasattr(tqdm_base, '_instances'):\n",
    "        for instance in list(tqdm_base._instances):\n",
    "            tqdm_base._decr_instances(instance)\n",
    "    return tqdm_base(*args, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "validation minibatch loop: 100%|██████████| 469/469 [00:49<00:00,  9.47it/s]\n"
     ]
    }
   ],
   "source": [
    "real_Y, predict_Y = [], []\n",
    "\n",
    "pbar = tqdm(\n",
    "    range(0, len(test_input_ids), batch_size), desc = 'validation minibatch loop'\n",
    ")\n",
    "for i in pbar:\n",
    "    index = min(i + batch_size, len(test_input_ids))\n",
    "    batch_x = test_input_ids[i: index]\n",
    "    batch_x = pad_sequences(batch_x, padding='post')\n",
    "    batch_y = test_Y[i: index]\n",
    "    batch_mask = test_mask[i: index]\n",
    "    batch_mask = pad_sequences(batch_mask, padding='post')\n",
    "    \n",
    "    predict_Y += np.argmax(sess.run(model.logits,\n",
    "            feed_dict = {\n",
    "            model.X: batch_x,\n",
    "            model.MASK: batch_mask\n",
    "            },\n",
    "    ), 1, ).tolist()\n",
    "    real_Y += batch_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       anger    0.99712   0.99763   0.99737      5895\n",
      "        fear    0.99687   0.99759   0.99723      4150\n",
      "       happy    0.99900   0.99900   0.99900      6017\n",
      "        love    0.99855   0.99615   0.99735      4154\n",
      "     sadness    0.99793   0.99906   0.99849      5307\n",
      "    surprise    0.99770   0.99694   0.99732      2612\n",
      "\n",
      "    accuracy                        0.99790     28135\n",
      "   macro avg    0.99786   0.99773   0.99779     28135\n",
      "weighted avg    0.99790   0.99790   0.99790     28135\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "print(\n",
    "    metrics.classification_report(\n",
    "        real_Y, predict_Y, target_names = ['anger', 'fear', 'happy', 'love', 'sadness', 'surprise'],\n",
    "        digits = 5\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from bert-base-emotion/model.ckpt\n",
      "WARNING:tensorflow:From <ipython-input-29-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 203 variables.\n",
      "INFO:tensorflow:Converted 203 variables to const ops.\n",
      "6952 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('bert-base-emotion', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py:1750: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[3.6769827e-05, 4.6792440e-05, 3.0530129e-05, 6.9144720e-05,\n",
       "        6.4117812e-05, 9.9975270e-01]], dtype=float32)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g = load_graph('bert-base-emotion/frozen_model.pb')\n",
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "mask = g.get_tensor_by_name('import/Placeholder_1:0')\n",
    "logits = g.get_tensor_by_name('import/logits:0')\n",
    "test_sess = tf.InteractiveSession(graph = g)\n",
    "result = test_sess.run(tf.nn.softmax(logits), feed_dict = {x: [input_id], mask: [input_mask]})\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "\n",
    "bucketName = 'huseinhouse-storage'\n",
    "Key = 'bert-base-emotion/frozen_model.pb'\n",
    "outPutname = \"v34/emotion/bert-base-emotion.pb\"\n",
    "\n",
    "s3 = boto3.client('s3')\n",
    "\n",
    "s3.upload_file(Key,bucketName,outPutname)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
