{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from unidecode import unidecode\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('subjectivity-negative-bm.txt','r') as fopen:\n",
    "    texts = fopen.read().split('\\n')\n",
    "labels = [0] * len(texts)\n",
    "\n",
    "with open('subjectivity-positive-bm.txt','r') as fopen:\n",
    "    positive_texts = fopen.read().split('\\n')\n",
    "labels += [1] * len(positive_texts)\n",
    "texts += positive_texts\n",
    "\n",
    "assert len(labels) == len(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "rules_normalizer = {\n",
    "    'sapa': 'siapa',\n",
    "    'tq': 'terima kasih',\n",
    "    'ty': 'terima kasih',\n",
    "    'january': 'januari',\n",
    "    'february': 'februari',\n",
    "    'march': 'mac',\n",
    "    'may': 'mei',\n",
    "    'june': 'jun',\n",
    "    'july': 'julai',\n",
    "    'august': 'ogos',\n",
    "    'october': 'oktober',\n",
    "    'december': 'disember',\n",
    "    'dec': 'dis',\n",
    "    'oct': 'okt',\n",
    "    'monday': 'isnin',\n",
    "    'mon': 'isn',\n",
    "    'tuesday': 'selasa',\n",
    "    'tues': 'sel',\n",
    "    'wednesday': 'rabu',\n",
    "    'wed': 'rab',\n",
    "    'thursday': 'khamis',\n",
    "    'thurs': 'kha',\n",
    "    'friday': 'jumaat',\n",
    "    'fri': 'jum',\n",
    "    'saturday': 'sabtu',\n",
    "    'sat': 'sab',\n",
    "    'sunday': 'ahad',\n",
    "    'sun': 'ahd',\n",
    "    'gemen': 'kerajaan',\n",
    "    'camtu': 'macam itu',\n",
    "    'experience': 'pengalaman',\n",
    "    'kpd': 'kepada',\n",
    "    'bengng': 'bengang',\n",
    "    'mntak': 'minta',\n",
    "    'bagasi': 'bagasi',\n",
    "    'kg': 'kampung',\n",
    "    'kilo': 'kilogram',\n",
    "    'g': 'pergi',\n",
    "    'grm': 'gram',\n",
    "    'k': 'okay',\n",
    "    'abgkat': 'abang dekat',\n",
    "    'abis': 'habis',\n",
    "    'ade': 'ada',\n",
    "    'adoi': 'aduh',\n",
    "    'adoii': 'aduh',\n",
    "    'aerodarat': 'kapal darat',\n",
    "    'agkt': 'angkat',\n",
    "    'ahh': 'ah',\n",
    "    'ailior': 'air liur',\n",
    "    'airasia': 'air asia x',\n",
    "    'airasiax': 'penerbangan',\n",
    "    'airline': 'penerbangan',\n",
    "    'airlines': 'penerbangan',\n",
    "    'airport': 'lapangan terbang',\n",
    "    'airpot': 'lapangan terbang',\n",
    "    'aje': 'sahaja',\n",
    "    'ajelah': 'sahajalah',\n",
    "    'ajer': 'sahaja',\n",
    "    'ak': 'aku',\n",
    "    'aq': 'aku',\n",
    "    'all': 'semua',\n",
    "    'ambik': 'ambil',\n",
    "    'amek': 'ambil',\n",
    "    'amer': 'amir',\n",
    "    'amik': 'ambil',\n",
    "    'ana': 'saya',\n",
    "    'angkt': 'angkat',\n",
    "    'anual': 'tahunan',\n",
    "    'apapun': 'apa pun',\n",
    "    'ape': 'apa',\n",
    "    'arab': 'arab',\n",
    "    'area': 'kawasan',\n",
    "    'aritu': 'hari itu',\n",
    "    'ask': 'tanya',\n",
    "    'astro': 'astro',\n",
    "    'at': 'pada',\n",
    "    'attitude': 'sikap',\n",
    "    'babi': 'khinzir',\n",
    "    'back': 'belakang',\n",
    "    'bag': 'beg',\n",
    "    'bang': 'abang',\n",
    "    'bangla': 'bangladesh',\n",
    "    'banyk': 'banyak',\n",
    "    'bard': 'pujangga',\n",
    "    'bargasi': 'bagasi',\n",
    "    'bawak': 'bawa',\n",
    "    'bawanges': 'bawang',\n",
    "    'be': 'jadi',\n",
    "    'behave': 'berkelakuan baik',\n",
    "    'belagak': 'berlagak',\n",
    "    'berdisiplin': 'berdisplin',\n",
    "    'berenti': 'berhenti',\n",
    "    'beskal': 'basikal',\n",
    "    'bff': 'rakan karib',\n",
    "    'bg': 'bagi',\n",
    "    'bgi': 'bagi',\n",
    "    'biase': 'biasa',\n",
    "    'big': 'besar',\n",
    "    'bike': 'basikal',\n",
    "    'bile': 'bila',\n",
    "    'binawe': 'binatang',\n",
    "    'bini': 'isteri',\n",
    "    'bkn': 'bukan',\n",
    "    'bla': 'bila',\n",
    "    'blom': 'belum',\n",
    "    'bnyak': 'banyak',\n",
    "    'body': 'tubuh',\n",
    "    'bole': 'boleh',\n",
    "    'boss': 'bos',\n",
    "    'bowling': 'boling',\n",
    "    'bpe': 'berapa',\n",
    "    'brand': 'jenama',\n",
    "    'brg': 'barang',\n",
    "    'briefing': 'taklimat',\n",
    "    'brng': 'barang',\n",
    "    'bro': 'abang',\n",
    "    'bru': 'baru',\n",
    "    'bruntung': 'beruntung',\n",
    "    'bsikal': 'basikal',\n",
    "    'btnggjwb': 'bertanggungjawab',\n",
    "    'btul': 'betul',\n",
    "    'buatlh': 'buatlah',\n",
    "    'buh': 'letak',\n",
    "    'buka': 'buka',\n",
    "    'but': 'tetapi',\n",
    "    'bwk': 'bawa',\n",
    "    'by': 'dengan',\n",
    "    'byr': 'bayar',\n",
    "    'bz': 'sibuk',\n",
    "    'camera': 'kamera',\n",
    "    'camni': 'macam ini',\n",
    "    'cane': 'macam mana',\n",
    "    'cant': 'tak boleh',\n",
    "    'carakerja': 'cara kerja',\n",
    "    'care': 'jaga',\n",
    "    'cargo': 'kargo',\n",
    "    'cctv': 'kamera litar tertutup',\n",
    "    'celako': 'celaka',\n",
    "    'cer': 'cerita',\n",
    "    'cheap': 'murah',\n",
    "    'check': 'semak',\n",
    "    'ciput': 'sedikit',\n",
    "    'cite': 'cerita',\n",
    "    'citer': 'cerita',\n",
    "    'ckit': 'sikit',\n",
    "    'cikit': 'sikit',\n",
    "    'ckp': 'cakap',\n",
    "    'class': 'kelas',\n",
    "    'cm': 'macam',\n",
    "    'cmni': 'macam ini',\n",
    "    'cmpak': 'campak',\n",
    "    'committed': 'komited',\n",
    "    'company': 'syarikat',\n",
    "    'complain': 'aduan',\n",
    "    'corn': 'jagung',\n",
    "    'couldnt': 'tak boleh',\n",
    "    'cr': 'cari',\n",
    "    'crew': 'krew',\n",
    "    'cube': 'cuba',\n",
    "    'cuma': 'cuma',\n",
    "    'curinyaa': 'curinya',\n",
    "    'cust': 'pelanggan',\n",
    "    'customer': 'pelanggan',\n",
    "    'd': 'di',\n",
    "    'da': 'dah',\n",
    "    'dn': 'dan',\n",
    "    'dahh': 'dah',\n",
    "    'damaged': 'rosak',\n",
    "    'dapek': 'dapat',\n",
    "    'day': 'hari',\n",
    "    'dazrin': 'dazrin',\n",
    "    'dbalingnya': 'dibalingnya',\n",
    "    'de': 'ada',\n",
    "    'deep': 'dalam',\n",
    "    'deliberately': 'sengaja',\n",
    "    'depa': 'mereka',\n",
    "    'dessa': 'desa',\n",
    "    'dgn': 'dengan',\n",
    "    'dh': 'dah',\n",
    "    'didunia': 'di dunia',\n",
    "    'diorang': 'mereka',\n",
    "    'diorng': 'mereka',\n",
    "    'direct': 'secara terus',\n",
    "    'diving': 'junam',\n",
    "    'dkt': 'dekat',\n",
    "    'dlempar': 'dilempar',\n",
    "    'dlm': 'dalam',\n",
    "    'dlt': 'padam',\n",
    "    'dlu': 'dulu',\n",
    "    'done': 'siap',\n",
    "    'dont': 'jangan',\n",
    "    'dorg': 'mereka',\n",
    "    'dpermudhkn': 'dipermudahkan',\n",
    "    'dpt': 'dapat',\n",
    "    'dri': 'dari',\n",
    "    'dsb': 'dan sebagainya',\n",
    "    'dy': 'dia',\n",
    "    'educate': 'mendidik',\n",
    "    'ensure': 'memastikan',\n",
    "    'everything': 'semua',\n",
    "    'ewahh': 'wah',\n",
    "    'expect': 'sangka',\n",
    "    'fb': 'facebook',\n",
    "    'fired': 'pecat',\n",
    "    'first': 'pertama',\n",
    "    'fkr': 'fikir',\n",
    "    'flight': 'kapal terbang',\n",
    "    'for': 'untuk',\n",
    "    'free': 'percuma',\n",
    "    'friend': 'kawan',\n",
    "    'fyi': 'untuk pengetahuan anda',\n",
    "    'gantila': 'gantilah',\n",
    "    'gantirugi': 'ganti rugi',\n",
    "    'gentlemen': 'lelaki budiman',\n",
    "    'gerenti': 'jaminan',\n",
    "    'gile': 'gila',\n",
    "    'gk': 'juga',\n",
    "    'gnti': 'ganti',\n",
    "    'go': 'pergi',\n",
    "    'gomen': 'kerajaan',\n",
    "    'goment': 'kerajaan',\n",
    "    'good': 'baik',\n",
    "    'ground': 'tanah',\n",
    "    'guarno': 'macam mana',\n",
    "    'hampa': 'mereka',\n",
    "    'hampeh': 'teruk',\n",
    "    'hanat': 'jahanam',\n",
    "    'handle': 'kawal',\n",
    "    'handling': 'kawalan',\n",
    "    'hanta': 'hantar',\n",
    "    'haritu': 'hari itu',\n",
    "    'harini': 'hari ini',\n",
    "    'hate': 'benci',\n",
    "    'have': 'ada',\n",
    "    'hawau': 'celaka',\n",
    "    'henpon': 'telefon',\n",
    "    'heran': 'hairan',\n",
    "    'him': 'dia',\n",
    "    'his': 'dia',\n",
    "    'hmpa': 'mereka',\n",
    "    'hntr': 'hantar',\n",
    "    'hotak': 'otak',\n",
    "    'hr': 'hari',\n",
    "    'i': 'saya',\n",
    "    'hrga': 'harga',\n",
    "    'hrp': 'harap',\n",
    "    'hu': 'sedih',\n",
    "    'humble': 'merendah diri',\n",
    "    'ibon': 'ikon',\n",
    "    'ichi': 'inci',\n",
    "    'idung': 'hidung',\n",
    "    'if': 'jika',\n",
    "    'ig': 'instagram',\n",
    "    'iklas': 'ikhlas',\n",
    "    'improve': 'menambah baik',\n",
    "    'in': 'masuk',\n",
    "    'isn t': 'tidak',\n",
    "    'isyaallah': 'insyallah',\n",
    "    'ja': 'sahaja',\n",
    "    'japan': 'jepun',\n",
    "    'jd': 'jadi',\n",
    "    'saja': 'sahaja',\n",
    "    'saje': 'sahaja',\n",
    "    'je': 'sahaja',\n",
    "    'jee': 'sahaja',\n",
    "    'jek': 'sahaja',\n",
    "    'jepun': 'jepun',\n",
    "    'jer': 'sahaja',\n",
    "    'jerr': 'sahaja',\n",
    "    'jez': 'sahaja',\n",
    "    'jg': 'juga',\n",
    "    'jgk': 'juga',\n",
    "    'jgn': 'jangan',\n",
    "    'jgnla': 'janganlah',\n",
    "    'jibake': 'celaka',\n",
    "    'jjur': 'jujur',\n",
    "    'job': 'kerja',\n",
    "    'jobscope': 'skop kerja',\n",
    "    'jogja': 'jogjakarta',\n",
    "    'jpam': 'jpam',\n",
    "    'jth': 'jatuh',\n",
    "    'jugak': 'juga',\n",
    "    'ka': 'ke',\n",
    "    'kalo': 'kalau',\n",
    "    'kalu': 'kalau',\n",
    "    'kang': 'nanti',\n",
    "    'kantoi': 'temberang',\n",
    "    'kasi': 'beri',\n",
    "    'kat': 'dekat',\n",
    "    'kbye': 'ok bye',\n",
    "    'kearah': 'ke arah',\n",
    "    'kecik': 'kecil',\n",
    "    'keja': 'kerja',\n",
    "    'keje': 'kerja',\n",
    "    'kejo': 'kerja',\n",
    "    'keksongan': 'kekosongan',\n",
    "    'kemana': 'ke mana',\n",
    "    'kene': 'kena',\n",
    "    'kenekan': 'kenakan',\n",
    "    'kesah': 'kisah',\n",
    "    'ketempat': 'ke tempat',\n",
    "    'kije': 'kerja',\n",
    "    'kijo': 'kerja',\n",
    "    'kiss': 'cium',\n",
    "    'kite': 'kita',\n",
    "    'kito': 'kita',\n",
    "    'kje': 'kerja',\n",
    "    'kjr': 'kerja',\n",
    "    'kk': 'okay',\n",
    "    'kmi': 'kami',\n",
    "    'kt': 'kat',\n",
    "    'tlg': 'tolong',\n",
    "    'kl': 'kuala lumpur',\n",
    "    'klai': 'kalau',\n",
    "    'klau': 'kalau',\n",
    "    'klia': 'klia',\n",
    "    'klo': 'kalau',\n",
    "    'klu': 'kalau',\n",
    "    'kn': 'kan',\n",
    "    'knapa': 'kenapa',\n",
    "    'kne': 'kena',\n",
    "    'ko': 'kau',\n",
    "    'kompom': 'sah',\n",
    "    'korang': 'kamu semua',\n",
    "    'korea': 'korea',\n",
    "    'korg': 'kamu semua',\n",
    "    'kot': 'mungkin',\n",
    "    'krja': 'kerja',\n",
    "    'ksalahan': 'kesalahan',\n",
    "    'kta': 'kita',\n",
    "    'kuar': 'keluar',\n",
    "    'kut': 'mungkin',\n",
    "    'la': 'lah',\n",
    "    'laa': 'lah',\n",
    "    'lahabau': 'celaka',\n",
    "    'lahanat': 'celaka',\n",
    "    'lainda': 'lain dah',\n",
    "    'lak': 'pula',\n",
    "    'last': 'akhir',\n",
    "    'le': 'lah',\n",
    "    'leader': 'ketua',\n",
    "    'leave': 'pergi',\n",
    "    'ler': 'lah',\n",
    "    'less': 'kurang',\n",
    "    'letter': 'surat',\n",
    "    'lg': 'lagi',\n",
    "    'lgi': 'lagi',\n",
    "    'lngsong': 'langsung',\n",
    "    'lol': 'hehe',\n",
    "    'lorr': 'lah',\n",
    "    'low': 'rendah',\n",
    "    'lps': 'lepas',\n",
    "    'luggage': 'bagasi',\n",
    "    'lumbe': 'lumba',\n",
    "    'lyak': 'layak',\n",
    "    'maap': 'maaf',\n",
    "    'maapkan': 'maafkan',\n",
    "    'mahai': 'mahal',\n",
    "    'mampos': 'mampus',\n",
    "    'mart': 'kedai',\n",
    "    'mau': 'mahu',\n",
    "    'mcm': 'macam',\n",
    "    'mcmtu': 'macam itu',\n",
    "    'memerlukn': 'memerlukan',\n",
    "    'mengembirakan': 'menggembirakan',\n",
    "    'mengmbilnyer': 'mengambilnya',\n",
    "    'mengtasi': 'mengatasi',\n",
    "    'mg': 'memang',\n",
    "    'mihak': 'memihak',\n",
    "    'min': 'admin',\n",
    "    'mingu': 'minggu',\n",
    "    'mintak': 'minta',\n",
    "    'mjtuhkn': 'menjatuhkan',\n",
    "    'mkyong': 'mak yong',\n",
    "    'mlibatkn': 'melibatkan',\n",
    "    'mmg': 'memang',\n",
    "    'mmnjang': 'memanjang',\n",
    "    'mmpos': 'mampus',\n",
    "    'mn': 'mana',\n",
    "    'mna': 'mana',\n",
    "    'mntak': 'minta',\n",
    "    'mntk': 'minta',\n",
    "    'mnyusun': 'menyusun',\n",
    "    'mood': 'suasana',\n",
    "    'most': 'paling',\n",
    "    'mr': 'tuan',\n",
    "    'msa': 'masa',\n",
    "    'msia': 'malaysia',\n",
    "    'mst': 'mesti',\n",
    "    'mu': 'awak',\n",
    "    'much': 'banyak',\n",
    "    'muko': 'muka',\n",
    "    'mum': 'emak',\n",
    "    'n': 'dan',\n",
    "    'nah': 'nah',\n",
    "    'nanny': 'nenek',\n",
    "    'napo': 'kenapa',\n",
    "    'nati': 'nanti',\n",
    "    'ngan': 'dengan',\n",
    "    'ngn': 'dengan',\n",
    "    'ni': 'ini',\n",
    "    'nie': 'ini',\n",
    "    'nii': 'ini',\n",
    "    'nk': 'nak',\n",
    "    'nmpk': 'nampak',\n",
    "    'nye': 'nya',\n",
    "    'ofis': 'pejabat',\n",
    "    'ohh': 'oh',\n",
    "    'oii': 'hoi',\n",
    "    'one': 'satu',\n",
    "    'online': 'dalam talian',\n",
    "    'or': 'atau',\n",
    "    'org': 'orang',\n",
    "    'orng': 'orang',\n",
    "    'otek': 'otak',\n",
    "    'p': 'pergi',\n",
    "    'paid': 'dah bayar',\n",
    "    'palabana': 'kepala otak',\n",
    "    'pasni': 'lepas ini',\n",
    "    'passengers': 'penumpang',\n",
    "    'passengger': 'penumpang',\n",
    "    'pastu': 'lepas itu',\n",
    "    'pd': 'pada',\n",
    "    'pegi': 'pergi',\n",
    "    'pekerje': 'pekerja',\n",
    "    'pekrja': 'pekerja',\n",
    "    'perabih': 'perabis',\n",
    "    'perkerja': 'pekerja',\n",
    "    'pg': 'pergi',\n",
    "    'phuii': 'puih',\n",
    "    'pikir': 'fikir',\n",
    "    'pilot': 'juruterbang',\n",
    "    'pk': 'fikir',\n",
    "    'pkerja': 'pekerja',\n",
    "    'pkerjaan': 'pekerjaan',\n",
    "    'pki': 'pakai',\n",
    "    'please': 'tolong',\n",
    "    'pls': 'tolong',\n",
    "    'pn': 'pun',\n",
    "    'pnh': 'pernah',\n",
    "    'pnt': 'penat',\n",
    "    'pnya': 'punya',\n",
    "    'pon': 'pun',\n",
    "    'priority': 'keutamaan',\n",
    "    'properties': 'harta benda',\n",
    "    'ptugas': 'petugas',\n",
    "    'pub': 'kelab malam',\n",
    "    'pulak': 'pula',\n",
    "    'puye': 'punya',\n",
    "    'pwrcuma': 'percuma',\n",
    "    'pyahnya': 'payahnya',\n",
    "    'quality': 'kualiti',\n",
    "    'quit': 'keluar',\n",
    "    'ramly': 'ramly',\n",
    "    'rege': 'harga',\n",
    "    'reger': 'harga',\n",
    "    'report': 'laporan',\n",
    "    'resigned': 'meletakkan jawatan',\n",
    "    'respect': 'hormat',\n",
    "    'rizal': 'rizal',\n",
    "    'rosak': 'rosak',\n",
    "    'rosok': 'rosak',\n",
    "    'rse': 'rasa',\n",
    "    'sacked': 'buang',\n",
    "    'sado': 'tegap',\n",
    "    'salute': 'sanjung',\n",
    "    'sam': 'sama',\n",
    "    'same': 'sama',\n",
    "    'samp': 'sampah',\n",
    "    'sbb': 'sebab',\n",
    "    'sbgai': 'sebagai',\n",
    "    'sblm': 'sebelum',\n",
    "    'sblum': 'sebelum',\n",
    "    'sbnarnya': 'sebenarnya',\n",
    "    'sbum': 'sebelum',\n",
    "    'sdg': 'sedang',\n",
    "    'sebb': 'sebab',\n",
    "    'sebijik': 'sebiji',\n",
    "    'see': 'lihat',\n",
    "    'seen': 'dilihat',\n",
    "    'selangor': 'selangor',\n",
    "    'selfie': 'swafoto',\n",
    "    'sempoi': 'cantik',\n",
    "    'senaraihitam': 'senarai hitam',\n",
    "    'seorg': 'seorang',\n",
    "    'service': 'perkhidmatan',\n",
    "    'sgt': 'sangat',\n",
    "    'shared': 'kongsi',\n",
    "    'shirt': 'kemeja',\n",
    "    'shut': 'tutup',\n",
    "    'sib': 'nasib',\n",
    "    'skali': 'sekali',\n",
    "    'sket': 'sikit',\n",
    "    'sma': 'sama',\n",
    "    'smoga': 'semoga',\n",
    "    'smpoi': 'cantik',\n",
    "    'sndiri': 'sendiri',\n",
    "    'sndr': 'sendiri',\n",
    "    'sndri': 'sendiri',\n",
    "    'sne': 'sana',\n",
    "    'so': 'jadi',\n",
    "    'sop': 'tatacara pengendalian piawai',\n",
    "    'sorang': 'seorang',\n",
    "    'spoting': 'pembintikan',\n",
    "    'sronok': 'seronok',\n",
    "    'ssh': 'susah',\n",
    "    'staff': 'staf',\n",
    "    'standing': 'berdiri',\n",
    "    'start': 'mula',\n",
    "    'steady': 'mantap',\n",
    "    'stiap': 'setiap',\n",
    "    'stress': 'stres',\n",
    "    'student': 'pelajar',\n",
    "    'study': 'belajar',\n",
    "    'studycase': 'kajian kes',\n",
    "    'sure': 'pasti',\n",
    "    'sykt': 'syarikat',\n",
    "    'tah': 'entah',\n",
    "    'taik': 'tahi',\n",
    "    'takan': 'tak akan',\n",
    "    'takat': 'setakat',\n",
    "    'takde': 'tak ada',\n",
    "    'takkan': 'tak akan',\n",
    "    'taknak': 'tak nak',\n",
    "    'tang': 'tentang',\n",
    "    'tanggungjawab': 'bertanggungjawab',\n",
    "    'taraa': 'sementara',\n",
    "    'tau': 'tahu',\n",
    "    'tbabit': 'terbabit',\n",
    "    'team': 'pasukan',\n",
    "    'terbaekk': 'terbaik',\n",
    "    'teruknye': 'teruknya',\n",
    "    'tgk': 'tengok',\n",
    "    'that': 'itu',\n",
    "    'thinking': 'fikir',\n",
    "    'those': 'itu',\n",
    "    'time': 'masa',\n",
    "    'tk': 'tak',\n",
    "    'tnggongjwb': 'tanggungjawab',\n",
    "    'tngok': 'tengok',\n",
    "    'tngu': 'tunggu',\n",
    "    'to': 'kepada',\n",
    "    'tosak': 'rosak',\n",
    "    'tp': 'tapi',\n",
    "    'tpi': 'tapi',\n",
    "    'tpon': 'telefon',\n",
    "    'transfer': 'pindah',\n",
    "    'trgelak': 'tergelak',\n",
    "    'ts': 'tan sri',\n",
    "    'tstony': 'tan sri tony',\n",
    "    'tu': 'itu',\n",
    "    'tuh': 'itu',\n",
    "    'tula': 'itulah',\n",
    "    'umeno': 'umno',\n",
    "    'unfortunately': 'malangnya',\n",
    "    'unhappy': 'tidak gembira',\n",
    "    'up': 'naik',\n",
    "    'upkan': 'naikkan',\n",
    "    'ur': 'awak',\n",
    "    'utk': 'untuk',\n",
    "    'very': 'sangat',\n",
    "    'viral': 'tular',\n",
    "    'vote': 'undi',\n",
    "    'warning': 'amaran',\n",
    "    'warranty': 'waranti',\n",
    "    'wassap': 'whatsapp',\n",
    "    'wat': 'apa',\n",
    "    'weii': 'wei',\n",
    "    'well': 'maklumlah',\n",
    "    'win': 'menang',\n",
    "    'with': 'dengan',\n",
    "    'wt': 'buat',\n",
    "    'x': 'tak',\n",
    "    'tw': 'tahu',\n",
    "    'ye': 'ya',\n",
    "    'yee': 'ya',\n",
    "    'yg': 'yang',\n",
    "    'yng': 'yang',\n",
    "    'you': 'awak',\n",
    "    'your': 'awak',\n",
    "    'sakai': 'selekeh',\n",
    "    'rmb': 'billion ringgit',\n",
    "    'rmj': 'juta ringgit',\n",
    "    'rmk': 'ribu ringgit',\n",
    "    'rm': 'ringgit',\n",
    "    ':*': '<kiss>',\n",
    "    ':-*': '<kiss>',\n",
    "    ':x': '<kiss>',\n",
    "    ':-)': '<happy>',\n",
    "    ':-))': '<happy>',\n",
    "    ':-)))': '<happy>',\n",
    "    ':-))))': '<happy>',\n",
    "    ':-)))))': '<happy>',\n",
    "    ':-))))))': '<happy>',\n",
    "    ':)': '<happy>',\n",
    "    ':))': '<happy>',\n",
    "    ':)))': '<happy>',\n",
    "    ':))))': '<happy>',\n",
    "    ':)))))': '<happy>',\n",
    "    ':))))))': '<happy>',\n",
    "    ':)))))))': '<happy>',\n",
    "    ':o)': '<happy>',\n",
    "    ':]': '<happy>',\n",
    "    ':3': '<happy>',\n",
    "    ':c)': '<happy>',\n",
    "    ':>': '<happy>',\n",
    "    '=]': '<happy>',\n",
    "    '8)': '<happy>',\n",
    "    '=)': '<happy>',\n",
    "    ':}': '<happy>',\n",
    "    ':^)': '<happy>',\n",
    "    '|;-)': '<happy>',\n",
    "    \":'-)\": '<happy>',\n",
    "    \":')\": '<happy>',\n",
    "    '\\o/': '<happy>',\n",
    "    '*\\\\0/*': '<happy>',\n",
    "    ':-D': '<laugh>',\n",
    "    ':D': '<laugh>',\n",
    "    '8-D': '<laugh>',\n",
    "    '8D': '<laugh>',\n",
    "    'x-D': '<laugh>',\n",
    "    'xD': '<laugh>',\n",
    "    'X-D': '<laugh>',\n",
    "    'XD': '<laugh>',\n",
    "    '=-D': '<laugh>',\n",
    "    '=D': '<laugh>',\n",
    "    '=-3': '<laugh>',\n",
    "    '=3': '<laugh>',\n",
    "    'B^D': '<laugh>',\n",
    "    '>:[': '<sad>',\n",
    "    ':-(': '<sad>',\n",
    "    ':-((': '<sad>',\n",
    "    ':-(((': '<sad>',\n",
    "    ':-((((': '<sad>',\n",
    "    ':-(((((': '<sad>',\n",
    "    ':-((((((': '<sad>',\n",
    "    ':-(((((((': '<sad>',\n",
    "    ':(': '<sad>',\n",
    "    ':((': '<sad>',\n",
    "    ':(((': '<sad>',\n",
    "    ':((((': '<sad>',\n",
    "    ':(((((': '<sad>',\n",
    "    ':((((((': '<sad>',\n",
    "    ':(((((((': '<sad>',\n",
    "    ':((((((((': '<sad>',\n",
    "    ':-c': '<sad>',\n",
    "    ':c': '<sad>',\n",
    "    ':-<': '<sad>',\n",
    "    ':<': '<sad>',\n",
    "    ':-[': '<sad>',\n",
    "    ':[': '<sad>',\n",
    "    ':{': '<sad>',\n",
    "    ':-||': '<sad>',\n",
    "    ':@': '<sad>',\n",
    "    \":'-(\": '<sad>',\n",
    "    \":'(\": '<sad>',\n",
    "    'D:<': '<sad>',\n",
    "    'D:': '<sad>',\n",
    "    'D8': '<sad>',\n",
    "    'D;': '<sad>',\n",
    "    'D=': '<sad>',\n",
    "    'DX': '<sad>',\n",
    "    'v.v': '<sad>',\n",
    "    \"D-':\": '<sad>',\n",
    "    '(>_<)': '<sad>',\n",
    "    ':|': '<sad>',\n",
    "    '>:O': '<surprise>',\n",
    "    ':-O': '<surprise>',\n",
    "    ':-o': '<surprise>',\n",
    "    ':O': '<surprise>',\n",
    "    '°o°': '<surprise>',\n",
    "    'o_O': '<surprise>',\n",
    "    'o_0': '<surprise>',\n",
    "    'o.O': '<surprise>',\n",
    "    'o-o': '<surprise>',\n",
    "    '8-0': '<surprise>',\n",
    "    '|-O': '<surprise>',\n",
    "    ';-)': '<wink>',\n",
    "    ';)': '<wink>',\n",
    "    '*-)': '<wink>',\n",
    "    '*)': '<wink>',\n",
    "    ';-]': '<wink>',\n",
    "    ';]': '<wink>',\n",
    "    ';D': '<wink>',\n",
    "    ';^)': '<wink>',\n",
    "    ':-,': '<wink>',\n",
    "    '>:P': '<tong>',\n",
    "    ':-P': '<tong>',\n",
    "    ':P': '<tong>',\n",
    "    'X-P': '<tong>',\n",
    "    'x-p': '<tong>',\n",
    "    'xp': '<tong>',\n",
    "    'XP': '<tong>',\n",
    "    ':-p': '<tong>',\n",
    "    ':p': '<tong>',\n",
    "    '=p': '<tong>',\n",
    "    ':-Þ': '<tong>',\n",
    "    ':Þ': '<tong>',\n",
    "    ':-b': '<tong>',\n",
    "    ':b': '<tong>',\n",
    "    ':-&': '<tong>',\n",
    "    '>:\\\\': '<annoyed>',\n",
    "    '>:/': '<annoyed>',\n",
    "    ':-/': '<annoyed>',\n",
    "    ':-.': '<annoyed>',\n",
    "    ':/': '<annoyed>',\n",
    "    ':\\\\': '<annoyed>',\n",
    "    '=/': '<annoyed>',\n",
    "    '=\\\\': '<annoyed>',\n",
    "    ':L': '<annoyed>',\n",
    "    '=L': '<annoyed>',\n",
    "    ':S': '<annoyed>',\n",
    "    '>.<': '<annoyed>',\n",
    "    ':-|': '<annoyed>',\n",
    "    '<:-|': '<annoyed>',\n",
    "    ':-X': '<seallips>',\n",
    "    ':X': '<seallips>',\n",
    "    ':-#': '<seallips>',\n",
    "    ':#': '<seallips>',\n",
    "    'O:-)': '<angel>',\n",
    "    '0:-3': '<angel>',\n",
    "    '0:3': '<angel>',\n",
    "    '0:-)': '<angel>',\n",
    "    '0:)': '<angel>',\n",
    "    '0;^)': '<angel>',\n",
    "    '>:)': '<devil>',\n",
    "    '>:D': '<devil>',\n",
    "    '>:-D': '<devil>',\n",
    "    '>;)': '<devil>',\n",
    "    '>:-)': '<devil>',\n",
    "    '}:-)': '<devil>',\n",
    "    '}:)': '<devil>',\n",
    "    '3:-)': '<devil>',\n",
    "    '3:)': '<devil>',\n",
    "    'o/\\o': '<highfive>',\n",
    "    '^5': '<highfive>',\n",
    "    '>_>^': '<highfive>',\n",
    "    '^<_<': '<highfive>',\n",
    "    '<3': '<heart>',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from unidecode import unidecode\n",
    "from rules import normalized_chars\n",
    "import random\n",
    "\n",
    "permulaan = [\n",
    "    'bel',\n",
    "    'se',\n",
    "    'ter',\n",
    "    'men',\n",
    "    'meng',\n",
    "    'mem',\n",
    "    'memper',\n",
    "    'di',\n",
    "    'pe',\n",
    "    'me',\n",
    "    'ke',\n",
    "    'ber',\n",
    "    'pen',\n",
    "    'per',\n",
    "]\n",
    "\n",
    "hujung = ['kan', 'kah', 'lah', 'tah', 'nya', 'an', 'wan', 'wati', 'ita']\n",
    "\n",
    "laughing = {\n",
    "    'huhu',\n",
    "    'haha',\n",
    "    'gagaga',\n",
    "    'hihi',\n",
    "    'wkawka',\n",
    "    'wkwk',\n",
    "    'kiki',\n",
    "    'keke',\n",
    "    'huehue',\n",
    "    'hshs',\n",
    "    'hoho',\n",
    "    'hewhew',\n",
    "    'uwu',\n",
    "    'sksk',\n",
    "    'ksks',\n",
    "    'gituu',\n",
    "    'gitu',\n",
    "    'mmeeooww',\n",
    "    'meow',\n",
    "    'alhamdulillah',\n",
    "    'muah',\n",
    "    'mmuahh',\n",
    "    'hehe',\n",
    "    'salamramadhan',\n",
    "    'happywomensday',\n",
    "    'jahagaha',\n",
    "    'ahakss',\n",
    "    'ahksk'\n",
    "}\n",
    "\n",
    "def naive_stemmer(word):\n",
    "    assert isinstance(word, str), 'input must be a string'\n",
    "    hujung_result = [e for e in hujung if word.endswith(e)]\n",
    "    if len(hujung_result):\n",
    "        hujung_result = max(hujung_result, key = len)\n",
    "        if len(hujung_result):\n",
    "            word = word[: -len(hujung_result)]\n",
    "    permulaan_result = [e for e in permulaan if word.startswith(e)]\n",
    "    if len(permulaan_result):\n",
    "        permulaan_result = max(permulaan_result, key = len)\n",
    "        if len(permulaan_result):\n",
    "            word = word[len(permulaan_result) :]\n",
    "    return word\n",
    "\n",
    "def make_cleaning(s, c_dict):\n",
    "    s = s.translate(c_dict)\n",
    "    return s\n",
    "\n",
    "def cleaning(string):\n",
    "    string = unidecode(string)\n",
    "    \n",
    "    string = ' '.join(\n",
    "        [make_cleaning(w, normalized_chars) for w in string.split()]\n",
    "    )\n",
    "    string = re.sub('\\(dot\\)', '.', string)\n",
    "    string = (\n",
    "        re.sub(re.findall(r'\\<a(.*?)\\>', string)[0], '', string)\n",
    "        if (len(re.findall(r'\\<a (.*?)\\>', string)) > 0)\n",
    "        and ('href' in re.findall(r'\\<a (.*?)\\>', string)[0])\n",
    "        else string\n",
    "    )\n",
    "    string = re.sub(\n",
    "        r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', ' ', string\n",
    "    )\n",
    "    \n",
    "    chars = '.,/'\n",
    "    for c in chars:\n",
    "        string = string.replace(c, f' {c} ')\n",
    "        \n",
    "    string = re.sub(r'[ ]+', ' ', string).strip()\n",
    "    string = [rules_normalizer.get(w, w) for w in string.split()]\n",
    "    string = [naive_stemmer(word) for word in string]\n",
    "    string = [w for w in string if len(w)]\n",
    "    string = [w for w in string if w[0] != '@']\n",
    "    x = []\n",
    "    for word in string:\n",
    "        word = word.lower()\n",
    "        if any([laugh in word for laugh in laughing]):\n",
    "            if random.random() >= 0.5:\n",
    "                x.append(word)\n",
    "        else:\n",
    "            x.append(word)\n",
    "    return ' '.join(x).lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9962/9962 [00:01<00:00, 5942.68it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "for i in tqdm(range(len(texts))):\n",
    "    texts[i] = cleaning(texts[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9962/9962 [00:00<00:00, 1339133.92it/s]\n"
     ]
    }
   ],
   "source": [
    "actual_t, actual_l = [], []\n",
    "\n",
    "for i in tqdm(range(len(texts))):\n",
    "    if len(texts[i]) > 2:\n",
    "        actual_t.append(texts[i])\n",
    "        actual_l.append(labels[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('combined.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(actual_t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import youtokentome as yttm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.44 s, sys: 1.17 s, total: 2.6 s\n",
      "Wall time: 784 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "bpe = yttm.BPE.train(data='combined.txt', \n",
    "               vocab_size=30000, model='subjectivity.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22933"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab = {v: i for i, v in enumerate(bpe.vocab())}\n",
    "rev_vocab = {i: v for i, v in enumerate(bpe.vocab())}\n",
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "r = re.compile(r'[\\S]+').findall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'▁filem ▁mula ▁pada ▁masa ▁lalu ▁mana ▁orang ▁budak ▁lelaki ▁nama ▁sama ▁cuba ▁yelamat ▁celebi ▁dari ▁mburu'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subs = [' '.join(s) for s in bpe.encode(actual_t, output_type=yttm.OutputType.SUBWORD)]\n",
    "subs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf = TfidfVectorizer(vocabulary = vocab, token_pattern = r'[\\S]+').fit(subs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('tfidf-subjectivity.pkl','wb') as fopen:\n",
    "    pickle.dump(tfidf,fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "vector = tfidf.transform(subs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9962, 22933)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vector.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((7969, 22933), (1993, 22933))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_X, test_X, train_Y, test_Y = train_test_split(vector, actual_l, test_size = 0.2)\n",
    "train_X.shape, test_X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import ComplementNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "multinomial = ComplementNB().fit(train_X, train_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative       0.96      0.93      0.94      3987\n",
      "    positive       0.93      0.96      0.95      3982\n",
      "\n",
      "    accuracy                           0.94      7969\n",
      "   macro avg       0.95      0.94      0.94      7969\n",
      "weighted avg       0.95      0.94      0.94      7969\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "print(\n",
    "    metrics.classification_report(\n",
    "        train_Y,\n",
    "        multinomial.predict(train_X),\n",
    "        target_names = ['negative', 'positive'],\n",
    "        \n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative    0.91527   0.87238   0.89331      1003\n",
      "    positive    0.87657   0.91818   0.89689       990\n",
      "\n",
      "    accuracy                        0.89513      1993\n",
      "   macro avg    0.89592   0.89528   0.89510      1993\n",
      "weighted avg    0.89605   0.89513   0.89509      1993\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\n",
    "    metrics.classification_report(\n",
    "        test_Y,\n",
    "        multinomial.predict(test_X),\n",
    "        target_names = ['negative', 'positive'],\n",
    "        digits = 5\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('multinomial-subjectivity.pkl','wb') as fopen:\n",
    "    pickle.dump(multinomial,fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "s3 = boto3.client('s3')\n",
    "bucketName = 'huseinhouse-storage'\n",
    "\n",
    "Key = 'multinomial-subjectivity.pkl'\n",
    "outPutname = \"v34/subjective/multinomial.pkl\"\n",
    "s3.upload_file(Key,bucketName,outPutname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "Key = 'tfidf-subjectivity.pkl'\n",
    "outPutname = \"v34/subjective/tfidf.pkl\"\n",
    "s3.upload_file(Key,bucketName,outPutname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "Key = 'subjectivity.model'\n",
    "outPutname = \"v34/subjective/bpe.model\"\n",
    "s3.upload_file(Key,bucketName,outPutname)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
