{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malaya-Dataset/master/dictionary/synonym/synonym0.json\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malaya-Dataset/master/dictionary/synonym/synonym1.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10984"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "from collections import defaultdict\n",
    "\n",
    "synonyms = defaultdict(list)\n",
    "\n",
    "files = ['synonym0.json', 'synonym1.json']\n",
    "\n",
    "for file in files:\n",
    "    with open(file) as fopen:\n",
    "        data = json.load(fopen)\n",
    "\n",
    "    for i in data:\n",
    "        if not len(i[1]):\n",
    "            continue\n",
    "        synonyms[i[0]].extend(i[1])\n",
    "        \n",
    "        for r in i[1]:\n",
    "            synonyms[r].append(i[0])\n",
    "\n",
    "for k, v in synonyms.items():\n",
    "    synonyms[k] = list(set(v))\n",
    "\n",
    "len(synonyms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../bert/dumping-wiki.txt') as fopen:\n",
    "    data = list(filter(None, fopen.read().split('\\n')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from tqdm import tqdm\n",
    "\n",
    "def preprocessing(string):\n",
    "    string = re.sub(\n",
    "        'http\\S+|www.\\S+',\n",
    "        '',\n",
    "        ' '.join(\n",
    "            [i for i in string.split() if i.find('#') < 0 and i.find('@') < 0]\n",
    "        ),\n",
    "    )\n",
    "    \n",
    "    chars = ',.()!:\\'\"/;=-'\n",
    "    for c in chars:\n",
    "        string = string.replace(c, f' {c} ')\n",
    "        \n",
    "    string = re.sub(\n",
    "        u'[0-9!@#$%^&*()_\\-+{}|\\~`\\'\";:?/.>,<]',\n",
    "        ' ',\n",
    "        string,\n",
    "        flags = re.UNICODE,\n",
    "    )\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip()\n",
    "    \n",
    "    return string.lower().split()\n",
    "\n",
    "def loop(strings):\n",
    "    results = []\n",
    "    for i in tqdm(range(len(strings))):\n",
    "        p = preprocessing(strings[i])\n",
    "        if len(p) < 400:\n",
    "            results.append(p)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import husein_boilerplate as husein"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 107221/107221 [00:05<00:00, 20106.60it/s]\n",
      "100%|██████████| 107221/107221 [00:05<00:00, 20197.68it/s]\n",
      "100%|██████████| 107221/107221 [00:05<00:00, 19555.31it/s]\n",
      "100%|██████████| 107221/107221 [00:05<00:00, 19461.97it/s]\n",
      " 95%|█████████▍| 101485/107221 [00:04<00:00, 19848.87it/s]\n",
      "100%|██████████| 107221/107221 [00:05<00:00, 19426.04it/s]\n",
      "100%|██████████| 107221/107221 [00:05<00:00, 20793.21it/s]\n",
      "100%|██████████| 107221/107221 [00:05<00:00, 20293.89it/s]\n",
      "100%|██████████| 107221/107221 [00:05<00:00, 18844.05it/s]\n",
      "100%|██████████| 107221/107221 [00:05<00:00, 19489.28it/s]\n",
      "100%|██████████| 107221/107221 [00:05<00:00, 19033.22it/s]\n",
      " 98%|█████████▊| 105421/107221 [00:05<00:00, 13967.78it/s]\n",
      "100%|██████████| 107221/107221 [00:05<00:00, 19759.75it/s]\n",
      "100%|██████████| 15/15 [00:00<00:00, 20400.31it/s].26it/s]\n",
      "100%|██████████| 107221/107221 [00:05<00:00, 19585.90it/s]\n",
      "100%|██████████| 107221/107221 [00:05<00:00, 18206.15it/s]\n",
      "100%|██████████| 107221/107221 [00:05<00:00, 19510.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10.3 s, sys: 2.17 s, total: 12.5 s\n",
      "Wall time: 17.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "data = husein.multiprocessing.multiprocessing(data, loop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1715524/1715524 [00:08<00:00, 210562.01it/s]\n"
     ]
    }
   ],
   "source": [
    "counts = []\n",
    "for string in tqdm(data):\n",
    "    count = 0\n",
    "    for no, word in enumerate(string):\n",
    "        if word in synonyms:\n",
    "            count += 1\n",
    "    counts.append(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "argsort = np.argsort(counts)[::-1][:50000].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "384"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data[argsort[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "def replace(string):\n",
    "    for no, word in enumerate(string):\n",
    "        if word in synonyms:\n",
    "            w = random.choice(synonyms[word])\n",
    "            string[no] = w\n",
    "    return ' '.join(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loop(indices):\n",
    "    results = []\n",
    "    for index in tqdm(indices):\n",
    "        string = data[index]\n",
    "        for _ in range(3):\n",
    "            results.append((' '.join(string), replace(string)))\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3125/3125 [00:00<00:00, 5832.65it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 5124.68it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 4768.70it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 4893.97it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 4647.88it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 4465.31it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 4302.77it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 4361.24it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 4179.11it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 4303.60it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 4258.17it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 4116.03it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 3997.60it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 3692.51it/s]\n",
      "100%|██████████| 3125/3125 [00:00<00:00, 3376.76it/s]\n",
      "100%|██████████| 3125/3125 [00:01<00:00, 3099.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 335 ms, sys: 1.03 s, total: 1.36 s\n",
      "Wall time: 2.15 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "results = husein.multiprocessing.multiprocessing(argsort, loop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('piagam ini bersetuju dokumentasi terkimpal dari parti ranah rakyat kelantan darul naim apabila parti pemerintah kawasan sudah membentuk maktab bebola sepak darul naim mendalam menjumpai kebijakan dan mempertingkatkan bahkan harganya perlawanan bebola sepak di ranah ini',\n",
       "  'program ini membawa bukti padu dari keributan gelanggang negara kelantan darul naim apabila pesta komandan wilayah wilayah telah lukis Akademi tarian sepak darul naim lengkap menentukan kepintaran dan mempertingkatkan pun bayaran pertandingan bola sepak di negara ini'),\n",
       " ('program ini membawa bukti padu dari keributan gelanggang negara kelantan darul naim apabila pesta komandan wilayah wilayah telah lukis Akademi tarian sepak darul naim lengkap menentukan kepintaran dan mempertingkatkan pun bayaran pertandingan bola sepak di negara ini',\n",
       "  'manifesto ini berlanjutan lambang terkimpal dari kehadiran ranah keadaan kelantan darul naim apabila kenduri panglima persekitaran sudah menarik maktab dansa sepak darul naim mendalam menjumpai kecerdikan dan mempertingkatkan bahkan penebusan perlawanan perlawanan tarian sepak di pedesaan ini')]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[-2:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('synonyms.json', 'w') as fopen:\n",
    "    json.dump(results, fopen)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
