{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['train_X', 'test_X', 'train_Y', 'test_Y'])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('train-test.json') as fopen:\n",
    "    train_test = json.load(fopen)\n",
    "    \n",
    "train_test.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('combined.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(train_test['train_X'] + train_test['test_X']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import youtokentome as yttm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3min 25s, sys: 36.7 s, total: 4min 2s\n",
      "Wall time: 57 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "bpe = yttm.BPE.train(data='combined.txt', \n",
    "               vocab_size=400000, model='language-detection.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "400000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab = {v: i for i, v in enumerate(bpe.vocab())}\n",
    "rev_vocab = {i: v for i, v in enumerate(bpe.vocab())}\n",
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "r = re.compile(r'[\\S]+').findall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined = train_test['train_X'] + train_test['test_X']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'▁voi ▁avete ▁conosciuto ▁delle ▁canadesi ▁qua ▁a ▁boston'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subs = [' '.join(s) for s in bpe.encode(combined[:3], output_type=yttm.OutputType.SUBWORD)]\n",
    "subs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "subs = [' '.join(s) for s in bpe.encode(combined, output_type=yttm.OutputType.SUBWORD)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23648246"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(subs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "bow = CountVectorizer(vocabulary = vocab, token_pattern = r'[\\S]+').fit(subs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['▁我该']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tsubs = [' '.join(s) for s in bpe.encode(['我该'], output_type=yttm.OutputType.SUBWORD)]\n",
    "tsubs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x400000 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 1 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bow.transform(tsubs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'▁= ▁= ▁ahli ▁ahli ▁= ▁= ▁= ▁= ▁= ▁ahli ▁ahli ▁semasa ▁= ▁= ▁= ▁sekarang ▁vokal ▁sekarang ▁gitar ▁vokal ▁sokongan ▁sekarang ▁sekarang ▁= ▁= ▁= ▁bekas ▁ahli ▁= ▁= ▁= ▁nik ▁vokal ▁o ▁gitar ▁gitar ▁f ▁fed eski ▁gitar ▁vokal ▁gitar ▁gitar ▁p ▁gitar ▁gitar ▁vokal ▁gitar ▁meninggal ▁dunia ▁vokal ▁meninggal ▁dunia ▁= ▁= ▁= ▁kegiatan ▁berkait ▁= ▁= ▁= ▁ahli ▁ahli ▁telah ▁bergerak ▁untuk ▁membentuk ▁seperti'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subs[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((array([342793]),), array([1]))"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "v = np.array(bow.transform(tsubs).todense())[0]\n",
    "np.where(v > 0), v[v > 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('bow-language-detection.pkl','wb') as fopen:\n",
    "    pickle.dump(bow,fopen)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
