{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import malaya"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/malaya/wordvector.py:94: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/malaya/wordvector.py:105: The name tf.InteractiveSession is deprecated. Please use tf.compat.v1.InteractiveSession instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "vocab, embedded = malaya.wordvector.load_social_media()\n",
    "wordvector = malaya.wordvector.load(embedded, vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "jokowi\n",
      "prabowo\n",
      "soekarno\n",
      "soeharto\n",
      "wiranto\n",
      "anjeng\n",
      "goblog\n",
      "kontol\n",
      "pakde\n",
      "bisa\n",
      "ngasilin\n",
      "ngapain\n",
      "gemer\n",
      "partai\n",
      "gue\n",
      "uang\n",
      "kalo\n",
      "pake\n",
      "kampret\n"
     ]
    }
   ],
   "source": [
    "words = ['jokowi', 'prabowo', 'soekarno', 'soeharto', 'wiranto', 'anjeng', 'goblog', 'kontol',\n",
    "        'pakde', 'bisa', 'ngasilin', 'ngapain', 'gemer', 'partai', 'gue', 'uang', 'kalo',\n",
    "        'pake', 'kampret']\n",
    "\n",
    "similars = []\n",
    "for word in words:\n",
    "    print(word)\n",
    "    similars.append(wordvector.n_closest(word=word, num_closest=30, metric='cosine'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://raw.githubusercontent.com/ardwort/freq-dist-id/master/data/twitter.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>di</td>\n",
       "      <td>404300</td>\n",
       "      <td>1.162798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>yg</td>\n",
       "      <td>279458</td>\n",
       "      <td>0.803743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ya</td>\n",
       "      <td>270836</td>\n",
       "      <td>0.778945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>aku</td>\n",
       "      <td>250220</td>\n",
       "      <td>0.719652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>yang</td>\n",
       "      <td>240116</td>\n",
       "      <td>0.690592</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0       1         2\n",
       "0    di  404300  1.162798\n",
       "1    yg  279458  0.803743\n",
       "2    ya  270836  0.778945\n",
       "3   aku  250220  0.719652\n",
       "4  yang  240116  0.690592"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('twitter.csv', header = None)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = set(df[0].tolist())\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2027893"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "with open('malays_word.json') as fopen:\n",
    "    malays = set(json.load(fopen))\n",
    "    \n",
    "len(malays)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2484"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df - malays\n",
    "df = df - malaya.texts._english_words._english_words\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "_list_laughing = {\n",
    "    'huhu',\n",
    "    'haha',\n",
    "    'gaga',\n",
    "    'hihi',\n",
    "    'wkawka',\n",
    "    'wkwk',\n",
    "    'kiki',\n",
    "    'keke',\n",
    "    'huehue',\n",
    "    'hih',\n",
    "    'waka',\n",
    "    'ckck',\n",
    "    'hehe',\n",
    "    'lolol',\n",
    "    'lol',\n",
    "    'wkaka'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2437"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import itertools\n",
    "import re\n",
    "\n",
    "merged = list(itertools.chain(*similars))\n",
    "indon_socialmedia_words = set([i[0] for i in merged if len(i[0]) > 2 and not len(re.findall(r'\\d+',i[0]))])\n",
    "indon_socialmedia_words = indon_socialmedia_words | set(words) | df\n",
    "indon_socialmedia_words = {i for i in indon_socialmedia_words if type(i) == str}\n",
    "indon_socialmedia_words = {i for i in indon_socialmedia_words if len(i) > 2}\n",
    "indon_socialmedia_words = {\n",
    "        word\n",
    "        for word in indon_socialmedia_words\n",
    "        if not any([laugh in word for laugh in _list_laughing])\n",
    "        and word[: len(word) // 2] != word[len(word) // 2 :]\n",
    "}\n",
    "\n",
    "minus = list(indon_socialmedia_words - malays)\n",
    "len(minus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "rejects = ['yup','babi', 'tanda','blajar', 'aah','iyaa','mana','smbil','knpa',\n",
    "          'adekku', 'bigbang','masa','dropbox','scumbag',\n",
    "          'wakaka', 'hmmm', 'emoticon','mana', 'bangun','ehm',\n",
    "          'sia','ehh', 'mey', 'marah','isn','duh','eek',\n",
    "          'akuu','twiter','twitter','ferrari','hahha',\n",
    "          'mlayu','woyy','ost','yaa','lgs','otw','okeee',\n",
    "          'waah','waaah','iyaa','eee','berenti','iphone',\n",
    "          'mbayar','mbahas','brenti','tanya','yeee','ayooo','kaaak',\n",
    "          'hooh','haha','hha','yaaa','faisal','pliss','rvp',\n",
    "          'ciee','fifa','kkk','argh','tweetnya','oon','lagi',\n",
    "          'loooh','looh','everytime','10rb','gaza',\n",
    "          'entahlah','amin','gile','crita','besok',\n",
    "          'aaakk','singap','lama','kemana','jangan',\n",
    "          'usb','moh','eeh','gsm','nihh','amin',\n",
    "          'haaa','akhir','nego','bljr','bts',\n",
    "          'jeh','kih','xdd','guw','adele','uhuy',\n",
    "          'apa','bgitu','amiin','amiiin','mantab',\n",
    "          'bored','tmpt','skt','tau','laen','tidor',\n",
    "          'tuhh','cari','sblm','baca','setaun','tmbh',\n",
    "          'mkan','wassap','innali','htc','zzz','ahhh',\n",
    "          'banyak2','mancing','main','jgn','lohh',\n",
    "          'itu','jauh','ha3','hugs','anj','ntah',\n",
    "          'aww','kfc','pkai','kawan','tdi',\n",
    "          'dulu','kali2','oww','acc','waa','dtg',\n",
    "           'bgn','fyi','inii','yey','yah',\n",
    "          'lgi','syg','eaa','bnyk','ngaku2',\n",
    "          'wtf','bukak','hii','pfft','obama','aaak',\n",
    "          'mmm','hbo','yeay','huaa','tlg','malu','avenger',\n",
    "          'knp','php','oppa','broo','hus','sucks','weh','rmh','bln','cmn','wib','ihh',\n",
    "          'ics','hardisk','ooh','yooo','beres','yamaha','seo','smua','mauu','pagi',\n",
    "          'satu2','heu','hhe','skrng','err', 'ronaldo','cita','omg','socmed',\n",
    "          'ktm','uitm','nyet','ahh','dkt','ikut','klau','nikmat','guling','gpp',\n",
    "          'ubuntu','krja','fav','bkn','makan','uhuk','pesbuk','slalu','retweets',\n",
    "          'xperia','smoga','messi','friendzone','soundcloud','bagi', 'bro']\n",
    "\n",
    "minus = [m for m in minus if all([r not in m for r in rejects])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "minus.extend(['takbisa', 'ngapen', 'paskabayar', 'kok', 'sampe', 'asin', 'gurih', 'sih',\n",
    "             'buk', 'neng', 'uang', 'pacar', 'jasmu', 'ujung', 'telen', 'banget',\n",
    "             'bisa', 'pengen', 'gue', 'wong', 'ganteng', 'gwe', 'apaan', 'dengerinnya',\n",
    "             'dengerinya', 'dengerin', 'fonsel'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2005"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(minus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'u' in minus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('indon-words-socialmedia.json', 'w') as fopen:\n",
    "    json.dump(minus, fopen)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
