{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "additional = \"\"\"\n",
    "Esbuk/aisbuk–peti ais\n",
    "Akun–setuju\n",
    "Ampai–letak. Contoh:Ampai seja tu baju sana(letak je baju tu kat sana)\n",
    "Ampai-ampai–pengsan, hilang kesedaran\n",
    "Ampus–lelah\n",
    "Babat–gemuk\n",
    "Bangas–basi\n",
    "Bahai–plastik(bahasa Sandakan). Kalau di Tawau plastik ialah “kompek”.\n",
    "Bubut–kejar. Contoh: Main bubut-bubut(main kejar-kejar)\n",
    "Betagar–berkarat\n",
    "Bida–hodoh, tak cantik\n",
    "Bikin–buat\n",
    "Bilang–kata/cakap\n",
    "Biut–tidak lurus, senget\n",
    "Budu–bodoh\n",
    "Buyuk–tipu, bohong\n",
    "Champin–hebat, terer, atau champion\n",
    "Celana/selana–seluar\n",
    "Durang–dorang\n",
    "Garit–calar\n",
    "Gerigitan–geram\n",
    "Ging–geng atau kawan=\n",
    "Tu hari–haritu, semalam, kelmarin\n",
    "Jerak/belak–bosan\n",
    "Jepit–kepit\n",
    "Karan–elektrik. Contoh: Mati karan(takda elektrik)\n",
    "Kasi–beri, bagi\n",
    "Kutang–baju dalam\n",
    "Kunuk–konon\n",
    "Kuyok–anjing\n",
    "Limpang–baring\n",
    "Limpas–lalu depan\n",
    "Lugai-lugai–tak tentu arah, lepak tanpa tujuan\n",
    "Mengkali–mungkin atau barangkali\n",
    "Miring–sewel\n",
    "ping–digunakan untuk minuman sejuk. Contoh: Milo ping(milo ais)\n",
    "Piring–pinggan\n",
    "Pistak–seluar dalam\n",
    "Sandi-teruk\n",
    "Santut/Seluar katak/spender–seluar dalam\n",
    "Seja/sijak–saja atau sahaja\n",
    "Sigup–rokok\n",
    "Sikul/skula–sekolah\n",
    "Siring–tepi\n",
    "Sungkai–buka puasa/iftar\n",
    "Tingu/tinguk–tengok\n",
    "Taruk–bubuh atau letak\n",
    "Tapuk–sorok\n",
    "Temahak–kuat makan\n",
    "Aku/sia/sa–saya\n",
    "Tia payah/nda payah/inda payah–tak payah\n",
    "Ndapapa/nda papa–tidak ada apa-apa\n",
    "Nda/inda/indak/tida/teda lah–tidak\n",
    "Sumandak–gadis\n",
    "Tambirang/betrip/minta puji/tapuji–berlagak\n",
    "Kin panas–buat orang marah\n",
    "Dari kau seja tu–up to you\n",
    "Palui–bodoh\n",
    "Talingung–bodoh\n",
    "Kitai\n",
    "Adui dogo/doi dogo’–macam terkejut\n",
    "Aku tampiling–aku tampar\n",
    "Pindik tagap–lelaki kuat\n",
    "Aramaitii!–ucapan sebelum minum/moginum tapi kami guna meluas bukan untuk minum/moginum seja.\n",
    "Uina!/wuina jo!-kiranya macam “woi!” atau wow\n",
    "Buli bha/bah kalau kau!\n",
    "Santiki!–macam high 5 tapi guna ibu jari\n",
    "Teda-teda bah kau ini–ape2 je lah kau ni\n",
    "Andu/ndu bah–ala kesiannya…\n",
    "Iya bah\n",
    "Bah\n",
    "anu tu\n",
    "1. tuuuna/ saaaana - points with the lips - (there-- at a distance)(the longer the 'tuuuuuuuuuuuuuuna / saaaaaaaaaaaaaana' the further the place is.)\n",
    "2. limpas \n",
    "3. palui \n",
    "4. tapuk \n",
    "5. bida \n",
    "6. bubut\n",
    "8. andang-andang \n",
    "9. sapak \n",
    "10. inda\n",
    "11. wicin\n",
    "12. talampau \n",
    "13. takajut \n",
    "14. siorang \n",
    "15. ngam-ngam \n",
    "16. mangkali \n",
    "17. kamarin \n",
    "18. kanapa \n",
    "19. gia \n",
    "20. bah \n",
    "23. basar\n",
    "24. siok \n",
    "25. urang \n",
    "26. karing \n",
    "29. padih \n",
    "30. numbur \n",
    "33. panat \n",
    "34. katawa \n",
    "35. lanjang \n",
    "36. putung \n",
    "37. Buduh \n",
    "38. tongo/bongo\n",
    "39. kabaru-baruan\n",
    "40. giuk\n",
    "41. Sikui\n",
    "42. Santut\n",
    "43. celana/salana \n",
    "44. gipit\n",
    "45. kanapatan/kadapatan \n",
    "46. kebangkalan\n",
    "47. ketulahan\n",
    "48. bahai\n",
    "49. uinnaaa!\n",
    "50. ging\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import malaya\n",
    "\n",
    "malays = malaya.texts._malay_words._malay_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "\n",
    "def cleaning(string):\n",
    "    string = unidecode(string).replace('.', '. ').replace(',', ' , ')\n",
    "    string = re.sub('[^\\'\"A-Za-z\\-/ ]+', ' ', string)\n",
    "    string = re.sub(r'[ ]+', ' ', string.lower()).strip()\n",
    "    return string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "146"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = []\n",
    "for line in additional.split('\\n'):\n",
    "    cleaned = cleaning(line)\n",
    "    if len(cleaned) < 3:\n",
    "        continue\n",
    "    c = cleaning(unidecode(line).replace('=', '-').replace(':', '-').split('-')[0]).replace('atau', '/')\\\n",
    "    .replace('\\\\', '/').replace('@', '/').split('/')\n",
    "    \n",
    "    words.extend([cleaning(i) for i in c])\n",
    "    \n",
    "len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "97"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = [i for i in words if len(i) > 3]\n",
    "    \n",
    "words = set(words) - malays\n",
    "len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('sabah-words.json', 'w') as fopen:\n",
    "    json.dump(list(words), fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'adui dogo',\n",
       " 'aisbuk',\n",
       " 'aku tampiling',\n",
       " 'akun',\n",
       " 'ampus',\n",
       " 'andu',\n",
       " 'antam',\n",
       " 'anu tu',\n",
       " 'aramaitii',\n",
       " 'bah kalau kau',\n",
       " 'bahai',\n",
       " 'basar',\n",
       " 'betagar',\n",
       " 'betrip',\n",
       " 'bongo',\n",
       " 'buduh',\n",
       " 'buli bha',\n",
       " 'buyuk',\n",
       " 'champin',\n",
       " 'dari kau seja tu',\n",
       " \"doi dogo'\",\n",
       " 'durang',\n",
       " 'esbuk',\n",
       " 'gerigitan',\n",
       " 'ging',\n",
       " 'gipit',\n",
       " 'inda',\n",
       " 'inda payah',\n",
       " 'indak',\n",
       " 'iya bah',\n",
       " 'kabaru',\n",
       " 'kadapatan',\n",
       " 'kamarin',\n",
       " 'kanapa',\n",
       " 'kanapatan',\n",
       " 'katawa',\n",
       " 'kebangkalan',\n",
       " 'kin panas',\n",
       " 'kitai',\n",
       " 'kunuk',\n",
       " 'kuyok',\n",
       " 'limpang',\n",
       " 'limpas',\n",
       " 'lugai',\n",
       " 'mangkali',\n",
       " 'mengkali',\n",
       " 'minta puji',\n",
       " 'nda papa',\n",
       " 'nda payah',\n",
       " 'ndapapa',\n",
       " 'ndu bah',\n",
       " 'ngam',\n",
       " 'numbur',\n",
       " 'padih',\n",
       " 'palui',\n",
       " 'panat',\n",
       " 'pindik tagap',\n",
       " 'ping',\n",
       " 'pistak',\n",
       " 'putung',\n",
       " 'saaaana',\n",
       " 'salana',\n",
       " 'santiki',\n",
       " 'santut',\n",
       " 'sapak',\n",
       " 'seja',\n",
       " 'selana',\n",
       " 'seluar katak',\n",
       " 'sigup',\n",
       " 'sijak',\n",
       " 'sikui',\n",
       " 'sikul',\n",
       " 'siok',\n",
       " 'siorang',\n",
       " 'skula',\n",
       " 'spender',\n",
       " 'sumandak',\n",
       " 'takajut',\n",
       " 'talampau',\n",
       " 'talingung',\n",
       " 'tambirang',\n",
       " 'tapuji',\n",
       " 'teda',\n",
       " 'teda lah',\n",
       " 'temahak',\n",
       " 'tia payah',\n",
       " 'tida',\n",
       " 'tingu',\n",
       " 'tinguk',\n",
       " 'tongo',\n",
       " 'tu hari',\n",
       " 'tuuuna',\n",
       " 'uina',\n",
       " 'uinnaaa',\n",
       " 'urang',\n",
       " 'wicin',\n",
       " 'wuina jo'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
