{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/alxnet/model_utils.py:334: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import xlnet\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tqdm import tqdm\n",
    "import model_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sentencepiece as spm\n",
    "from prepro_utils import preprocess_text, encode_ids\n",
    "\n",
    "sp_model = spm.SentencePieceProcessor()\n",
    "sp_model.Load('alxlnet-base/sp10m.cased.v9.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rules import normalized_chars\n",
    "import random\n",
    "import re\n",
    "from unidecode import unidecode\n",
    "\n",
    "laughing = {\n",
    "    'huhu',\n",
    "    'haha',\n",
    "    'gagaga',\n",
    "    'hihi',\n",
    "    'wkawka',\n",
    "    'wkwk',\n",
    "    'kiki',\n",
    "    'keke',\n",
    "    'huehue',\n",
    "    'hshs',\n",
    "    'hoho',\n",
    "    'hewhew',\n",
    "    'uwu',\n",
    "    'sksk',\n",
    "    'ksks',\n",
    "    'gituu',\n",
    "    'gitu',\n",
    "    'mmeeooww',\n",
    "    'meow',\n",
    "    'alhamdulillah',\n",
    "    'muah',\n",
    "    'mmuahh',\n",
    "    'hehe',\n",
    "    'salamramadhan',\n",
    "    'happywomensday',\n",
    "    'jahagaha',\n",
    "    'ahakss',\n",
    "    'ahksk'\n",
    "}\n",
    "\n",
    "def make_cleaning(s, c_dict):\n",
    "    s = s.translate(c_dict)\n",
    "    return s\n",
    "\n",
    "def cleaning(string):\n",
    "    \"\"\"\n",
    "    use by any transformer model before tokenization\n",
    "    \"\"\"\n",
    "    string = unidecode(string)\n",
    "    \n",
    "    string = ' '.join(\n",
    "        [make_cleaning(w, normalized_chars) for w in string.split()]\n",
    "    )\n",
    "    string = re.sub('\\(dot\\)', '.', string)\n",
    "    string = (\n",
    "        re.sub(re.findall(r'\\<a(.*?)\\>', string)[0], '', string)\n",
    "        if (len(re.findall(r'\\<a (.*?)\\>', string)) > 0)\n",
    "        and ('href' in re.findall(r'\\<a (.*?)\\>', string)[0])\n",
    "        else string\n",
    "    )\n",
    "    string = re.sub(\n",
    "        r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', ' ', string\n",
    "    )\n",
    "    \n",
    "    chars = '.,/'\n",
    "    for c in chars:\n",
    "        string = string.replace(c, f' {c} ')\n",
    "        \n",
    "    string = re.sub(r'[ ]+', ' ', string).strip().split()\n",
    "    string = [w for w in string if w[0] != '@']\n",
    "    x = []\n",
    "    for word in string:\n",
    "        word = word.lower()\n",
    "        if any([laugh in word for laugh in laughing]):\n",
    "            if random.random() >= 0.5:\n",
    "                x.append(word)\n",
    "        else:\n",
    "            x.append(word)\n",
    "    string = [w.title() if w[0].isupper() else w for w in x]\n",
    "    return ' '.join(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['severe toxic',\n",
       " 'obscene',\n",
       " 'identity attack',\n",
       " 'insult',\n",
       " 'threat',\n",
       " 'asian',\n",
       " 'atheist',\n",
       " 'bisexual',\n",
       " 'black',\n",
       " 'buddhist',\n",
       " 'christian',\n",
       " 'female',\n",
       " 'heterosexual',\n",
       " 'indian',\n",
       " 'homosexual, gay or lesbian',\n",
       " 'intellectual or learning disability',\n",
       " 'jewish',\n",
       " 'latino',\n",
       " 'male',\n",
       " 'muslim',\n",
       " 'other disability',\n",
       " 'other gender',\n",
       " 'other race or ethnicity',\n",
       " 'other religion',\n",
       " 'other sexual orientation',\n",
       " 'physical disability',\n",
       " 'psychiatric or mental illness',\n",
       " 'transgender',\n",
       " 'white',\n",
       " 'malay',\n",
       " 'chinese']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = \"\"\"\n",
    "1. severe toxic\n",
    "2. obscene\n",
    "3. identity attack\n",
    "4. insult\n",
    "5. threat\n",
    "6. asian\n",
    "7. atheist\n",
    "8. bisexual\n",
    "9. black\n",
    "10. buddhist\n",
    "11. christian\n",
    "12. female\n",
    "13. heterosexual\n",
    "14. indian\n",
    "15. homosexual, gay or lesbian\n",
    "16. intellectual or learning disability\n",
    "17. jewish\n",
    "18. latino\n",
    "19. male\n",
    "20. muslim\n",
    "21. other disability\n",
    "22. other gender\n",
    "23. other race or ethnicity\n",
    "24. other religion\n",
    "25. other sexual orientation\n",
    "26. physical disability\n",
    "27. psychiatric or mental illness\n",
    "28. transgender\n",
    "29. white\n",
    "30. malay\n",
    "31. chinese\n",
    "\"\"\"\n",
    "labels = [l.split('. ')[1].strip() for l in labels.split('\\n') if len(l)]\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../toxicity/translated-1750000.json',\n",
       " '../toxicity/translated-1450000.json',\n",
       " '../toxicity/translated-700000.json',\n",
       " '../toxicity/translated-350000.json',\n",
       " '../toxicity/translated-600000.json',\n",
       " '../toxicity/translated-900000.json',\n",
       " '../toxicity/translated-1000000.json',\n",
       " '../toxicity/translated-1100000.json',\n",
       " '../toxicity/translated-550000.json',\n",
       " '../toxicity/translated-150000.json',\n",
       " '../toxicity/translated-500000.json',\n",
       " '../toxicity/translated-1500000.json',\n",
       " '../toxicity/translated-1150000.json',\n",
       " '../toxicity/translated-750000.json',\n",
       " '../toxicity/translated-850000.json',\n",
       " '../toxicity/translated-1650000.json',\n",
       " '../toxicity/translated-300000.json',\n",
       " '../toxicity/translated-650000.json',\n",
       " '../toxicity/translated-950000.json',\n",
       " '../toxicity/translated-250000.json',\n",
       " '../toxicity/translated-1600000.json',\n",
       " '../toxicity/translated-0.json',\n",
       " '../toxicity/translated-1550000.json',\n",
       " '../toxicity/translated-1800000.json',\n",
       " '../toxicity/translated-450000.json',\n",
       " '../toxicity/translated-50000.json',\n",
       " '../toxicity/translated-1050000.json',\n",
       " '../toxicity/translated-1200000.json',\n",
       " '../toxicity/translated-1700000.json',\n",
       " '../toxicity/translated-400000.json']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import glob\n",
    "\n",
    "files = glob.glob('../toxicity/translated*')\n",
    "files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../toxicity/translated-1750000.json\n",
      "../toxicity/translated-1450000.json\n",
      "../toxicity/translated-700000.json\n",
      "../toxicity/translated-350000.json\n",
      "../toxicity/translated-600000.json\n",
      "../toxicity/translated-900000.json\n",
      "../toxicity/translated-1000000.json\n",
      "../toxicity/translated-1100000.json\n",
      "../toxicity/translated-550000.json\n",
      "../toxicity/translated-150000.json\n",
      "../toxicity/translated-500000.json\n",
      "../toxicity/translated-1500000.json\n",
      "../toxicity/translated-1150000.json\n",
      "../toxicity/translated-750000.json\n",
      "../toxicity/translated-850000.json\n",
      "../toxicity/translated-1650000.json\n",
      "../toxicity/translated-300000.json\n",
      "../toxicity/translated-650000.json\n",
      "../toxicity/translated-950000.json\n",
      "../toxicity/translated-250000.json\n",
      "../toxicity/translated-1600000.json\n",
      "../toxicity/translated-0.json\n",
      "../toxicity/translated-1550000.json\n",
      "../toxicity/translated-1800000.json\n",
      "../toxicity/translated-450000.json\n",
      "../toxicity/translated-50000.json\n",
      "../toxicity/translated-1050000.json\n",
      "../toxicity/translated-1200000.json\n",
      "../toxicity/translated-1700000.json\n",
      "../toxicity/translated-400000.json\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1401054"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "X, Y = [], []\n",
    "\n",
    "for file in files:\n",
    "    print(file)\n",
    "    with open(file) as fopen:\n",
    "        f = json.load(fopen)\n",
    "        for row in f:\n",
    "            if len(row[1]) == 29:\n",
    "                X.append(row[0])\n",
    "                Y.append(row[1] + [0, 0])\n",
    "        \n",
    "    \n",
    "len(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "rejected_labels = ['black', 'white', 'jewish', 'latino']\n",
    "[labels.index(l) for l in rejected_labels]\n",
    "labels = [l for l in labels if l not in rejected_labels]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "ydf = pd.DataFrame(np.array(Y))\n",
    "ydf = ydf.loc[(ydf[8] == 0) & (ydf[28] == 0) & (ydf[16] == 0) & (ydf[17] == 0)]\n",
    "ydf = ydf.drop([8, 28, 16, 17], axis = 1)\n",
    "ix = ydf.index.tolist()\n",
    "Y = ydf.values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1361040, 1361040)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = [X[i] for i in ix]\n",
    "len(X), len(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping = {'severe_toxic': 'severe toxic', 'identity_hate': 'identity attack',\n",
    "          'toxic': 'severe toxic', 'melayu': 'malay', 'cina': 'chinese', 'india': 'indian'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_onehot(tags, depth = len(labels)):\n",
    "    onehot = [0] * depth\n",
    "    for tag in tags:\n",
    "        onehot[labels.index(tag)] = 1\n",
    "    return onehot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "melayu 84851\n",
      "cina 43956\n",
      "india 20208\n"
     ]
    }
   ],
   "source": [
    "with open('../toxicity/kaum.json') as fopen:\n",
    "    kaum = json.load(fopen)\n",
    "    \n",
    "for k, v in kaum.items():\n",
    "    print(k, len(v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../toxicity/weak-learning-toxicity.json') as fopen:\n",
    "    scores = json.load(fopen)\n",
    "    \n",
    "for k, v in scores.items():\n",
    "    for no in range(len(v)):\n",
    "        tags = []\n",
    "        for l, v_ in v[no].items():\n",
    "            if round(v_) == 1:\n",
    "                tags.append(mapping.get(l, l))\n",
    "        tags.append(mapping[k])\n",
    "        Y.append(generate_onehot(tags))\n",
    "        X.append(kaum[k][no])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1510055/1510055 [04:46<00:00, 5279.91it/s] \n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "for i in tqdm(range(len(X))):\n",
    "    X[i] = cleaning(X[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1510055/1510055 [00:01<00:00, 1034178.33it/s]\n"
     ]
    }
   ],
   "source": [
    "actual_t, actual_l = [], []\n",
    "\n",
    "for i in tqdm(range(len(X))):\n",
    "    if len(X[i]) > 2:\n",
    "        actual_t.append(X[i])\n",
    "        actual_l.append(Y[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from prepro_utils import preprocess_text, encode_ids\n",
    "\n",
    "def tokenize_fn(text):\n",
    "    text = preprocess_text(text, lower= False)\n",
    "    return encode_ids(sp_model, text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1502328/1502328 [10:15<00:00, 2439.38it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "SEG_ID_A   = 0\n",
    "SEG_ID_B   = 1\n",
    "SEG_ID_CLS = 2\n",
    "SEG_ID_SEP = 3\n",
    "SEG_ID_PAD = 4\n",
    "MAX_SEQ_LENGTH = 150\n",
    "\n",
    "special_symbols = {\n",
    "    \"<unk>\"  : 0,\n",
    "    \"<s>\"    : 1,\n",
    "    \"</s>\"   : 2,\n",
    "    \"<cls>\"  : 3,\n",
    "    \"<sep>\"  : 4,\n",
    "    \"<pad>\"  : 5,\n",
    "    \"<mask>\" : 6,\n",
    "    \"<eod>\"  : 7,\n",
    "    \"<eop>\"  : 8,\n",
    "}\n",
    "\n",
    "VOCAB_SIZE = 32000\n",
    "UNK_ID = special_symbols[\"<unk>\"]\n",
    "CLS_ID = special_symbols[\"<cls>\"]\n",
    "SEP_ID = special_symbols[\"<sep>\"]\n",
    "MASK_ID = special_symbols[\"<mask>\"]\n",
    "EOD_ID = special_symbols[\"<eod>\"]\n",
    "\n",
    "input_ids, input_masks, segment_ids = [], [], []\n",
    "\n",
    "for text in tqdm(actual_t):\n",
    "    tokens_a = tokenize_fn(text)\n",
    "    if len(tokens_a) > MAX_SEQ_LENGTH - 2:\n",
    "        tokens_a = tokens_a[:(MAX_SEQ_LENGTH - 2)]\n",
    "        \n",
    "    tokens = []\n",
    "    segment_id = []\n",
    "    for token in tokens_a:\n",
    "        tokens.append(token)\n",
    "        segment_id.append(SEG_ID_A)\n",
    "    tokens.append(SEP_ID)\n",
    "    segment_id.append(SEG_ID_A)\n",
    "    tokens.append(CLS_ID)\n",
    "    segment_id.append(SEG_ID_CLS)\n",
    "    \n",
    "    input_id = tokens\n",
    "    input_mask = [0] * len(input_id)\n",
    "    if len(input_id) < MAX_SEQ_LENGTH:\n",
    "        delta_len = MAX_SEQ_LENGTH - len(input_id)\n",
    "        input_id = [0] * delta_len + input_id\n",
    "        input_mask = [1] * delta_len + input_mask\n",
    "        segment_id = [SEG_ID_PAD] * delta_len + segment_id\n",
    "    \n",
    "    input_ids.append(input_id)\n",
    "    input_masks.append(input_mask)\n",
    "    segment_ids.append(segment_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "kwargs = dict(\n",
    "      is_training=True,\n",
    "      use_tpu=False,\n",
    "      use_bfloat16=False,\n",
    "      dropout=0.1,\n",
    "      dropatt=0.1,\n",
    "      init='normal',\n",
    "      init_range=0.1,\n",
    "      init_std=0.05,\n",
    "      clamp_len=-1)\n",
    "\n",
    "xlnet_parameters = xlnet.RunConfig(**kwargs)\n",
    "xlnet_config = xlnet.XLNetConfig(json_path='alxlnet-base/config.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300465 30046\n"
     ]
    }
   ],
   "source": [
    "epoch = 10\n",
    "batch_size = 50\n",
    "warmup_proportion = 0.1\n",
    "num_train_steps = int(len(actual_t) / batch_size * epoch)\n",
    "num_warmup_steps = int(num_train_steps * warmup_proportion)\n",
    "print(num_train_steps, num_warmup_steps)\n",
    "learning_rate = 2e-5\n",
    "\n",
    "training_parameters = dict(\n",
    "      decay_method = 'poly',\n",
    "      train_steps = num_train_steps,\n",
    "      learning_rate = learning_rate,\n",
    "      warmup_steps = num_warmup_steps,\n",
    "      min_lr_ratio = 0.0,\n",
    "      weight_decay = 0.00,\n",
    "      adam_epsilon = 1e-8,\n",
    "      num_core_per_host = 1,\n",
    "      lr_layer_decay_rate = 1,\n",
    "      use_tpu=False,\n",
    "      use_bfloat16=False,\n",
    "      dropout=0.0,\n",
    "      dropatt=0.0,\n",
    "      init='normal',\n",
    "      init_range=0.1,\n",
    "      init_std=0.05,\n",
    "      clip = 1.0,\n",
    "      clamp_len=-1,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Parameter:\n",
    "    def __init__(self, decay_method, warmup_steps, weight_decay, adam_epsilon, \n",
    "                num_core_per_host, lr_layer_decay_rate, use_tpu, learning_rate, train_steps,\n",
    "                min_lr_ratio, clip, **kwargs):\n",
    "        self.decay_method = decay_method\n",
    "        self.warmup_steps = warmup_steps\n",
    "        self.weight_decay = weight_decay\n",
    "        self.adam_epsilon = adam_epsilon\n",
    "        self.num_core_per_host = num_core_per_host\n",
    "        self.lr_layer_decay_rate = lr_layer_decay_rate\n",
    "        self.use_tpu = use_tpu\n",
    "        self.learning_rate = learning_rate\n",
    "        self.train_steps = train_steps\n",
    "        self.min_lr_ratio = min_lr_ratio\n",
    "        self.clip = clip\n",
    "        \n",
    "training_parameters = Parameter(**training_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        dimension_output,\n",
    "        learning_rate = 2e-5,\n",
    "    ):\n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.segment_ids = tf.placeholder(tf.int32, [None, None])\n",
    "        self.input_masks = tf.placeholder(tf.float32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.float32, [None, dimension_output])\n",
    "        \n",
    "        xlnet_model = xlnet.XLNetModel(\n",
    "            xlnet_config=xlnet_config,\n",
    "            run_config=xlnet_parameters,\n",
    "            input_ids=tf.transpose(self.X, [1, 0]),\n",
    "            seg_ids=tf.transpose(self.segment_ids, [1, 0]),\n",
    "            input_mask=tf.transpose(self.input_masks, [1, 0]))\n",
    "        \n",
    "        output_layer = xlnet_model.get_sequence_output()\n",
    "        output_layer = tf.transpose(output_layer, [1, 0, 2])\n",
    "        \n",
    "        self.logits_seq = tf.layers.dense(output_layer, dimension_output)\n",
    "        self.logits_seq = tf.identity(self.logits_seq, name = 'logits_seq')\n",
    "        self.logits = self.logits_seq[:, 0]\n",
    "        self.logits = tf.identity(self.logits, name = 'logits')\n",
    "        \n",
    "        self.cost = tf.reduce_mean(\n",
    "            tf.nn.sigmoid_cross_entropy_with_logits(\n",
    "                logits = self.logits, labels = self.Y\n",
    "            )\n",
    "        )\n",
    "        \n",
    "        self.optimizer, self.learning_rate, _ = model_utils.get_train_op(training_parameters, self.cost)\n",
    "        \n",
    "        correct_prediction = tf.equal(tf.round(tf.nn.sigmoid(self.logits)), tf.round(self.Y))\n",
    "        all_labels_true = tf.reduce_min(tf.cast(correct_prediction, tf.float32), 1)\n",
    "        self.accuracy = tf.reduce_mean(all_labels_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/alxnet/xlnet.py:253: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/alxnet/xlnet.py:253: The name tf.AUTO_REUSE is deprecated. Please use tf.compat.v1.AUTO_REUSE instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/alxnet/custom_modeling.py:696: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
      "\n",
      "INFO:tensorflow:memory input None\n",
      "INFO:tensorflow:Use float type <dtype: 'float32'>\n",
      "WARNING:tensorflow:From /home/husein/alxnet/custom_modeling.py:703: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/alxnet/custom_modeling.py:808: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dropout instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:271: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/alxnet/custom_modeling.py:109: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/nn_impl.py:183: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/husein/alxnet/model_utils.py:105: The name tf.train.get_or_create_global_step is deprecated. Please use tf.compat.v1.train.get_or_create_global_step instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/alxnet/model_utils.py:119: The name tf.train.polynomial_decay is deprecated. Please use tf.compat.v1.train.polynomial_decay instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/alxnet/model_utils.py:150: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "dimension_output = len(labels)\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(\n",
    "    dimension_output,\n",
    "    learning_rate\n",
    ")\n",
    "\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import re\n",
    "\n",
    "def get_assignment_map_from_checkpoint(tvars, init_checkpoint):\n",
    "    \"\"\"Compute the union of the current variables and checkpoint variables.\"\"\"\n",
    "    assignment_map = {}\n",
    "    initialized_variable_names = {}\n",
    "\n",
    "    name_to_variable = collections.OrderedDict()\n",
    "    for var in tvars:\n",
    "        name = var.name\n",
    "        m = re.match('^(.*):\\\\d+$', name)\n",
    "        if m is not None:\n",
    "            name = m.group(1)\n",
    "        name_to_variable[name] = var\n",
    "\n",
    "    init_vars = tf.train.list_variables(init_checkpoint)\n",
    "\n",
    "    assignment_map = collections.OrderedDict()\n",
    "    for x in init_vars:\n",
    "        (name, var) = (x[0], x[1])\n",
    "        if name not in name_to_variable:\n",
    "            continue\n",
    "        assignment_map[name] = name_to_variable[name]\n",
    "        initialized_variable_names[name] = 1\n",
    "        initialized_variable_names[name + ':0'] = 1\n",
    "\n",
    "    return (assignment_map, initialized_variable_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "tvars = tf.trainable_variables()\n",
    "checkpoint = 'alxlnet-base/model.ckpt'\n",
    "assignment_map, initialized_variable_names = get_assignment_map_from_checkpoint(tvars, \n",
    "                                                                                checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from alxlnet-base/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver(var_list = assignment_map)\n",
    "saver.restore(sess, checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_X, test_X, train_masks, test_masks, train_segments, test_segments, train_Y, test_Y = train_test_split(\n",
    "    input_ids, input_masks, segment_ids, actual_l, test_size = 0.2\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.0, 0.9872991]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pad_sequences = tf.keras.preprocessing.sequence.pad_sequences\n",
    "\n",
    "i = 0\n",
    "index = 4\n",
    "batch_x = train_X[i : index]\n",
    "batch_y = train_Y[i : index]\n",
    "batch_masks = train_masks[i : index]\n",
    "batch_segments = train_segments[i : index]\n",
    "\n",
    "sess.run(\n",
    "    [model.accuracy, model.cost],\n",
    "    feed_dict = {\n",
    "        model.X: batch_x,\n",
    "        model.Y: batch_y,\n",
    "        model.segment_ids: batch_segments,\n",
    "        model.input_masks: batch_masks,\n",
    "    },\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 24038/24038 [5:23:40<00:00,  1.24it/s, accuracy=0.917, cost=0.0264]  \n",
      "test minibatch loop: 100%|██████████| 6010/6010 [28:52<00:00,  3.47it/s, accuracy=0.812, cost=0.0207]\n",
      "train minibatch loop:   0%|          | 0/24038 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.000000, current acc: 0.910005\n",
      "time taken: 21153.76011800766\n",
      "epoch: 0, training loss: 0.042540, training acc: 0.867588, valid loss: 0.022529, valid acc: 0.910005\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 24038/24038 [5:23:52<00:00,  1.24it/s, accuracy=1, cost=0.0225]      \n",
      "test minibatch loop: 100%|██████████| 6010/6010 [28:51<00:00,  3.47it/s, accuracy=0.875, cost=0.0217]\n",
      "train minibatch loop:   0%|          | 0/24038 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, pass acc: 0.910005, current acc: 0.916076\n",
      "time taken: 21164.35333251953\n",
      "epoch: 1, training loss: 0.021906, training acc: 0.912364, valid loss: 0.021220, valid acc: 0.916076\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 24038/24038 [5:24:22<00:00,  1.24it/s, accuracy=1, cost=0.0222]      \n",
      "test minibatch loop: 100%|██████████| 6010/6010 [29:06<00:00,  3.44it/s, accuracy=0.875, cost=0.0221]\n",
      "train minibatch loop:   0%|          | 0/24038 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2, pass acc: 0.916076, current acc: 0.917317\n",
      "time taken: 21209.230727910995\n",
      "epoch: 2, training loss: 0.020763, training acc: 0.918800, valid loss: 0.020846, valid acc: 0.917317\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 24038/24038 [5:24:21<00:00,  1.24it/s, accuracy=1, cost=0.0206]      \n",
      "test minibatch loop: 100%|██████████| 6010/6010 [28:46<00:00,  3.48it/s, accuracy=0.875, cost=0.0226]\n",
      "train minibatch loop:   0%|          | 0/24038 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3, pass acc: 0.917317, current acc: 0.918731\n",
      "time taken: 21187.296753406525\n",
      "epoch: 3, training loss: 0.020114, training acc: 0.923085, valid loss: 0.020673, valid acc: 0.918731\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 24038/24038 [5:24:31<00:00,  1.23it/s, accuracy=1, cost=0.021]       \n",
      "test minibatch loop: 100%|██████████| 6010/6010 [28:47<00:00,  3.48it/s, accuracy=0.812, cost=0.0209]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 21198.428645133972\n",
      "epoch: 4, training loss: 0.019618, training acc: 0.926257, valid loss: 0.020746, valid acc: 0.918218\n",
      "\n",
      "break epoch:5\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import time\n",
    "\n",
    "\n",
    "EARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 1, 0, 0, 0\n",
    "\n",
    "while True:\n",
    "    lasttime = time.time()\n",
    "    if CURRENT_CHECKPOINT == EARLY_STOPPING:\n",
    "        print('break epoch:%d\\n' % (EPOCH))\n",
    "        break\n",
    "\n",
    "    train_acc, train_loss, test_acc, test_loss = [], [], [], []\n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i : index]\n",
    "        batch_y = train_Y[i : index]\n",
    "        batch_masks = train_masks[i : index]\n",
    "        batch_segments = train_segments[i : index]\n",
    "        acc, cost, _ = sess.run(\n",
    "            [model.accuracy, model.cost, model.optimizer],\n",
    "            feed_dict = {\n",
    "                model.X: batch_x,\n",
    "                model.Y: batch_y,\n",
    "                model.segment_ids: batch_segments,\n",
    "                model.input_masks: batch_masks,\n",
    "            },\n",
    "        )\n",
    "        train_loss.append(cost)\n",
    "        train_acc.append(acc)\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "        \n",
    "    pbar = tqdm(range(0, len(test_X), batch_size), desc = 'test minibatch loop')\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i : index]\n",
    "        batch_y = test_Y[i : index]\n",
    "        batch_masks = test_masks[i : index]\n",
    "        batch_segments = test_segments[i : index]\n",
    "        acc, cost = sess.run(\n",
    "            [model.accuracy, model.cost],\n",
    "            feed_dict = {\n",
    "                model.X: batch_x,\n",
    "                model.Y: batch_y,\n",
    "                model.segment_ids: batch_segments,\n",
    "                model.input_masks: batch_masks,\n",
    "            },\n",
    "        )\n",
    "        test_loss.append(cost)\n",
    "        test_acc.append(acc)\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "        \n",
    "    train_loss = np.mean(train_loss)\n",
    "    train_acc = np.mean(train_acc)\n",
    "    test_loss = np.mean(test_loss)\n",
    "    test_acc = np.mean(test_acc)\n",
    "        \n",
    "    if test_acc > CURRENT_ACC:\n",
    "        print(\n",
    "            'epoch: %d, pass acc: %f, current acc: %f'\n",
    "            % (EPOCH, CURRENT_ACC, test_acc)\n",
    "        )\n",
    "        CURRENT_ACC = test_acc\n",
    "        CURRENT_CHECKPOINT = 0\n",
    "    else:\n",
    "        CURRENT_CHECKPOINT += 1\n",
    "        \n",
    "    print('time taken:', time.time() - lasttime)\n",
    "    print(\n",
    "        'epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'\n",
    "        % (EPOCH, train_loss, train_acc, test_loss, test_acc)\n",
    "    )\n",
    "    EPOCH += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'alxlnet-base-toxicity/model.ckpt'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'alxlnet-base-toxicity/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "kwargs = dict(\n",
    "      is_training=False,\n",
    "      use_tpu=False,\n",
    "      use_bfloat16=False,\n",
    "      dropout=0.0,\n",
    "      dropatt=0.0,\n",
    "      init='normal',\n",
    "      init_range=0.1,\n",
    "      init_std=0.05,\n",
    "      clamp_len=-1)\n",
    "\n",
    "xlnet_parameters = xlnet.RunConfig(**kwargs)\n",
    "xlnet_config = xlnet.XLNetConfig(json_path='alxlnet-base/config.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:memory input None\n",
      "INFO:tensorflow:Use float type <dtype: 'float32'>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py:1750: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    }
   ],
   "source": [
    "dimension_output = len(labels)\n",
    "learning_rate = 2e-5\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(\n",
    "    dimension_output,\n",
    "    learning_rate\n",
    ")\n",
    "\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from alxlnet-base-toxicity/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.restore(sess, 'alxlnet-base-toxicity/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "test minibatch loop: 100%|██████████| 6010/6010 [1:58:40<00:00,  1.18s/it]  \n"
     ]
    }
   ],
   "source": [
    "stack = []\n",
    "\n",
    "pbar = tqdm(range(0, len(test_X), batch_size), desc = 'test minibatch loop')\n",
    "for i in pbar:\n",
    "    index = min(i + batch_size, len(test_X))\n",
    "    batch_x = test_X[i : index]\n",
    "    batch_y = test_Y[i : index]\n",
    "    batch_masks = test_masks[i : index]\n",
    "    batch_segments = test_segments[i : index]\n",
    "    stack.append(sess.run(tf.nn.sigmoid(model.logits),\n",
    "            feed_dict = {\n",
    "                model.X: batch_x,\n",
    "                model.segment_ids: batch_segments,\n",
    "                model.input_masks: batch_masks\n",
    "            },\n",
    "    ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                     precision    recall  f1-score   support\n",
      "\n",
      "                       severe toxic    0.81360   0.70005   0.75257      9795\n",
      "                            obscene    0.55653   0.49517   0.52406      2694\n",
      "                    identity attack    0.50581   0.34938   0.41329      1371\n",
      "                             insult    0.68040   0.60330   0.63953     12672\n",
      "                             threat    0.40360   0.35123   0.37560       447\n",
      "                              asian    0.66192   0.49077   0.56364       379\n",
      "                            atheist    0.87151   0.92308   0.89655       169\n",
      "                           bisexual    0.87500   0.60870   0.71795        23\n",
      "                           buddhist    0.60417   0.63043   0.61702        46\n",
      "                          christian    0.88051   0.84715   0.86351      4619\n",
      "                             female    0.86958   0.92578   0.89680      6979\n",
      "                       heterosexual    0.75000   0.82051   0.78367       117\n",
      "                             indian    0.96407   0.87242   0.91596      4029\n",
      "         homosexual, gay or lesbian    0.88719   0.93265   0.90935      1366\n",
      "intellectual or learning disability    0.21429   0.50000   0.30000         6\n",
      "                               male    0.72205   0.65407   0.68638      4897\n",
      "                             muslim    0.82114   0.94992   0.88085      2576\n",
      "                   other disability    0.00000   0.00000   0.00000         0\n",
      "                       other gender    0.00000   0.00000   0.00000         1\n",
      "            other race or ethnicity    0.00000   0.00000   0.00000         9\n",
      "                     other religion    0.00000   0.00000   0.00000         6\n",
      "           other sexual orientation    0.00000   0.00000   0.00000         2\n",
      "                physical disability    0.00000   0.00000   0.00000         1\n",
      "      psychiatric or mental illness    0.57044   0.87589   0.69091       564\n",
      "                        transgender    0.71756   0.88679   0.79325       212\n",
      "                              malay    0.95141   0.97619   0.96364     17051\n",
      "                            chinese    0.92615   0.92417   0.92516      8888\n",
      "\n",
      "                          micro avg    0.83376   0.80221   0.81768     78919\n",
      "                          macro avg    0.56470   0.56732   0.55962     78919\n",
      "                       weighted avg    0.82757   0.80221   0.81282     78919\n",
      "                        samples avg    0.16116   0.15980   0.15799     78919\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/husein/.local/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/husein/.local/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/husein/.local/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "print(metrics.classification_report(np.around(np.array(test_Y)),\n",
    "                                    np.around(np.concatenate(stack,axis=0)),\n",
    "                                    target_names=labels,\n",
    "                                    digits=5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Placeholder_2',\n",
       " 'Placeholder_3',\n",
       " 'model/transformer/r_w_bias',\n",
       " 'model/transformer/r_r_bias',\n",
       " 'model/transformer/word_embedding/lookup_table',\n",
       " 'model/transformer/word_embedding/lookup_table_2',\n",
       " 'model/transformer/r_s_bias',\n",
       " 'model/transformer/seg_embed',\n",
       " 'model/transformer/layer_shared/rel_attn/q/kernel',\n",
       " 'model/transformer/layer_shared/rel_attn/k/kernel',\n",
       " 'model/transformer/layer_shared/rel_attn/v/kernel',\n",
       " 'model/transformer/layer_shared/rel_attn/r/kernel',\n",
       " 'model/transformer/layer_shared/rel_attn/o/kernel',\n",
       " 'model/transformer/layer_shared/rel_attn/LayerNorm/gamma',\n",
       " 'model/transformer/layer_shared/ff/layer_1/kernel',\n",
       " 'model/transformer/layer_shared/ff/layer_1/bias',\n",
       " 'model/transformer/layer_shared/ff/layer_2/kernel',\n",
       " 'model/transformer/layer_shared/ff/layer_2/bias',\n",
       " 'model/transformer/layer_shared/ff/LayerNorm/gamma',\n",
       " 'dense/kernel',\n",
       " 'dense/bias',\n",
       " 'logits_seq',\n",
       " 'logits']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'Adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from alxlnet-base-toxicity/model.ckpt\n",
      "WARNING:tensorflow:From <ipython-input-39-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 21 variables.\n",
      "INFO:tensorflow:Converted 21 variables to const ops.\n",
      "7390 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('alxlnet-base-toxicity', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "\n",
    "bucketName = 'huseinhouse-storage'\n",
    "Key = 'alxlnet-base-toxicity/frozen_model.pb'\n",
    "outPutname = \"v34/toxicity/alxlnet-base-toxicity.pb\"\n",
    "\n",
    "s3 = boto3.client('s3')\n",
    "\n",
    "s3.upload_file(Key,bucketName,outPutname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
