{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['other', 'malay', 'ind', 'eng'])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('formal-language.json') as fopen:\n",
    "    formal = json.load(fopen)\n",
    "    \n",
    "formal.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['rojak', 'malay', 'ind'])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('social-language.json') as fopen:\n",
    "    social = json.load(fopen)\n",
    "    \n",
    "social.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('en-socialmedia.json') as fopen:\n",
    "    en_socialmedia = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "908120"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('manglish.json') as fopen:\n",
    "    manglish = json.load(fopen)\n",
    "    \n",
    "len(manglish)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1289172"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('eng.json') as fopen:\n",
    "    eng = json.load(fopen)\n",
    "    \n",
    "len(eng)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6816517"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('others.json') as fopen:\n",
    "    others = json.load(fopen)\n",
    "    \n",
    "len(others)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "98"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('ms.json') as fopen:\n",
    "    ms = json.load(fopen)\n",
    "    \n",
    "len(ms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12515"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('ind.json') as fopen:\n",
    "    ind = json.load(fopen)\n",
    "    \n",
    "len(ind)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, Y = [], []\n",
    "\n",
    "for k in formal.keys():\n",
    "    X.extend(formal[k])\n",
    "    Y.extend([k] * len(formal[k]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in social.keys():\n",
    "    X.extend(social[k])\n",
    "    Y.extend([k] * len(social[k]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.extend(manglish)\n",
    "Y.extend(['manglish'] * len(manglish))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.extend(eng)\n",
    "Y.extend(['eng'] * len(eng))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.extend(others)\n",
    "Y.extend(['other'] * len(others))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.extend(ms)\n",
    "Y.extend(['malay'] * len(ms))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.extend(ind)\n",
    "Y.extend(['ind'] * len(ind))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.extend(en_socialmedia)\n",
    "Y.extend(['eng'] * len(en_socialmedia))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(X) == len(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 23678033/23678033 [00:13<00:00, 1712080.28it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "actual_X, actual_Y = [], []\n",
    "\n",
    "for i in tqdm(range(len(X))):\n",
    "    if len(X[i]):\n",
    "        actual_X.append(X[i])\n",
    "        actual_Y.append(Y[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(23648246, 23678033)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(actual_X), len(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "\n",
    "train_X, test_X, train_Y, test_Y = train_test_split(actual_X, actual_Y, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array(['eng', 'ind', 'malay', 'manglish', 'other', 'rojak'], dtype='<U8'),\n",
       " array([2215975, 2295708, 7202654,  726678, 5720022,  757559]))"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(train_Y, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array(['eng', 'ind', 'malay', 'manglish', 'other', 'rojak'], dtype='<U8'),\n",
       " array([ 553739,  576059, 1800649,  181442, 1428083,  189678]))"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(test_Y, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train-test.json', 'w') as fopen:\n",
    "    json.dump({'train_X': train_X, 'test_X': test_X, 'train_Y': train_Y, 'test_Y': test_Y}, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "\n",
    "bucketName = 'malaya-dataset'\n",
    "Key = 'train-test.json'\n",
    "outPutname = \"language-detection/train-test.json\"\n",
    "\n",
    "s3 = boto3.client('s3')\n",
    "s3.upload_file(Key,bucketName,outPutname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__label__rus Ты можешь позвонить Тому и сказать, чтобы он немедленно приезжал?\r\n",
      "__label__rus А рыльце-то у него в пушку.\r\n",
      "__label__ind Sebagaimana kita memerlukan udara yang nyaman, juga ikan memerlukan air yang bersih.\r\n",
      "__label__epo Italio havas du montoĉenojn, Alpojn kaj Apeninojn.\r\n",
      "__label__rus Этот ботинок дерьмовый, потому что он из Китая.\r\n",
      "__label__tur Sami bilgisayarların nasıl hackleneceğini bilmiyor.\r\n",
      "__label__rus Президент остался в постели.\r\n",
      "__label__eng I can't remember anything else about Tom.\r\n",
      "__label__eng It's what I wanted.\r\n",
      "__label__ber Sami yessen yelli-s n yiwen n yimsujji.\r\n"
     ]
    }
   ],
   "source": [
    "!head valid.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 18918596/18918596 [00:19<00:00, 960268.18it/s] \n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "texts = []\n",
    "\n",
    "for i in tqdm(range(len(train_X))):\n",
    "    texts.append('__label__%s %s'%(train_Y[i], train_X[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train-bahasa.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(texts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__label__other voi avete conosciuto delle canadesi qua a boston\r\n",
      "__label__malay = = ahli ahli = = = = = ahli ahli semasa = = = sekarang vokal sekarang gitar vokal sokongan sekarang sekarang = = = bekas ahli = = = nik vokal o gitar gitar f fedeski gitar vokal gitar gitar p gitar gitar vokal gitar meninggal dunia vokal meninggal dunia = = = kegiatan berkait = = = ahli ahli telah bergerak untuk membentuk seperti\r\n",
      "__label__malay bahkan enggan mentaati suruhan pintu hatinya kelamaan tertutup daripada mendapat hidayah menjadikan mereka semakin sesat terus melakukan\r\n",
      "__label__ind kenapa sih free transfer bisa kejadian di sepak bola hmm\r\n",
      "__label__malay kat kl pun sama cuma buayanya di darat\r\n",
      "__label__ind mbak mbaknya tukang parfum\r\n",
      "__label__malay next comingsoon maknanya bekas kerajaan paling hancing nie low class semua yg dilantik\r\n",
      "__label__eng i use a frying pan at least once a day\r\n",
      "__label__other tom un öyle bir şey söylediğini hiç duymadım\r\n",
      "__label__malay aku kecewa sungguh dgn subject tersebut\r\n"
     ]
    }
   ],
   "source": [
    "!head train-bahasa.txt"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
