{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### preqs\n",
    "`start bert-en`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T06:42:53.101414Z",
     "start_time": "2019-11-17T06:42:53.053621Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chapter</th>\n",
       "      <th>index</th>\n",
       "      <th>text</th>\n",
       "      <th>translate</th>\n",
       "      <th>translit</th>\n",
       "      <th>audio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The time</td>\n",
       "      <td>1</td>\n",
       "      <td>Excuse me!</td>\n",
       "      <td>对不起 ！</td>\n",
       "      <td>duìbùqǐ!</td>\n",
       "      <td>https://www.book2.nl/book2/ZH/SOUND/0143.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The time</td>\n",
       "      <td>2</td>\n",
       "      <td>What time is it, please?</td>\n",
       "      <td>请问, 现在 几点 了 ?</td>\n",
       "      <td>Qǐngwèn, xiànzài jǐ diǎnle?</td>\n",
       "      <td>https://www.book2.nl/book2/ZH/SOUND/0144.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The time</td>\n",
       "      <td>3</td>\n",
       "      <td>Thank you very much.</td>\n",
       "      <td>非常 感谢 ！</td>\n",
       "      <td>Fēicháng gǎnxiè!</td>\n",
       "      <td>https://www.book2.nl/book2/ZH/SOUND/0145.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The time</td>\n",
       "      <td>4</td>\n",
       "      <td>It is one o’clock.</td>\n",
       "      <td>现在 一点 。</td>\n",
       "      <td>Xiànzài yīdiǎn.</td>\n",
       "      <td>https://www.book2.nl/book2/ZH/SOUND/0146.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The time</td>\n",
       "      <td>5</td>\n",
       "      <td>It is two o’clock.</td>\n",
       "      <td>现在 二点 。</td>\n",
       "      <td>Xiànzài èr diǎn.</td>\n",
       "      <td>https://www.book2.nl/book2/ZH/SOUND/0147.mp3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    chapter  index                      text      translate  \\\n",
       "0  The time      1                Excuse me!          对不起 ！   \n",
       "1  The time      2  What time is it, please?  请问, 现在 几点 了 ?   \n",
       "2  The time      3      Thank you very much.        非常 感谢 ！   \n",
       "3  The time      4        It is one o’clock.        现在 一点 。   \n",
       "4  The time      5        It is two o’clock.        现在 二点 。   \n",
       "\n",
       "                      translit                                         audio  \n",
       "0                     duìbùqǐ!  https://www.book2.nl/book2/ZH/SOUND/0143.mp3  \n",
       "1  Qǐngwèn, xiànzài jǐ diǎnle?  https://www.book2.nl/book2/ZH/SOUND/0144.mp3  \n",
       "2             Fēicháng gǎnxiè!  https://www.book2.nl/book2/ZH/SOUND/0145.mp3  \n",
       "3              Xiànzài yīdiǎn.  https://www.book2.nl/book2/ZH/SOUND/0146.mp3  \n",
       "4             Xiànzài èr diǎn.  https://www.book2.nl/book2/ZH/SOUND/0147.mp3  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "dfjson = pd.read_json(f'/pi/stack/crawlers/langcrs/all_zh.json')\n",
    "dfjson.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T11:44:55.484317Z",
     "start_time": "2019-11-17T11:44:55.478745Z"
    }
   },
   "outputs": [],
   "source": [
    "# quotes['text'].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T11:46:04.017114Z",
     "start_time": "2019-11-17T11:46:03.960746Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1760 1760 set()\n"
     ]
    }
   ],
   "source": [
    "dfjson_ja = pd.read_json(f'/pi/stack/crawlers/langcrs/all_ja.json')\n",
    "dfjson_id = pd.read_json(f'/pi/stack/crawlers/langcrs/all_id.json')\n",
    "set_ja=set(dfjson_ja['text'].to_list())\n",
    "set_id=set(dfjson_id['text'].to_list())\n",
    "print(len(set_ja), len(set_id), set_ja-set_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T11:48:53.193197Z",
     "start_time": "2019-11-17T11:48:53.156954Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chapter</th>\n",
       "      <th>index_x</th>\n",
       "      <th>text</th>\n",
       "      <th>translate_x</th>\n",
       "      <th>translit_x</th>\n",
       "      <th>audio_x</th>\n",
       "      <th>index_y</th>\n",
       "      <th>translate_y</th>\n",
       "      <th>translit_y</th>\n",
       "      <th>audio_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>People</td>\n",
       "      <td>1</td>\n",
       "      <td>I</td>\n",
       "      <td>私</td>\n",
       "      <td>watashi</td>\n",
       "      <td>https://www.book2.nl/book2/JA/SOUND/0003.mp3</td>\n",
       "      <td>1</td>\n",
       "      <td>saya</td>\n",
       "      <td></td>\n",
       "      <td>https://www.book2.nl/book2/ID/SOUND/0003.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>People</td>\n",
       "      <td>2</td>\n",
       "      <td>I and you</td>\n",
       "      <td>私と あなた</td>\n",
       "      <td>watashi to anata</td>\n",
       "      <td>https://www.book2.nl/book2/JA/SOUND/0004.mp3</td>\n",
       "      <td>2</td>\n",
       "      <td>saya dan kamu</td>\n",
       "      <td></td>\n",
       "      <td>https://www.book2.nl/book2/ID/SOUND/0004.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>People</td>\n",
       "      <td>3</td>\n",
       "      <td>both of us</td>\n",
       "      <td>私達</td>\n",
       "      <td>watashitachi</td>\n",
       "      <td>https://www.book2.nl/book2/JA/SOUND/0005.mp3</td>\n",
       "      <td>3</td>\n",
       "      <td>kami berdua</td>\n",
       "      <td></td>\n",
       "      <td>https://www.book2.nl/book2/ID/SOUND/0005.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>People</td>\n",
       "      <td>4</td>\n",
       "      <td>he</td>\n",
       "      <td>彼</td>\n",
       "      <td>kare</td>\n",
       "      <td>https://www.book2.nl/book2/JA/SOUND/0006.mp3</td>\n",
       "      <td>4</td>\n",
       "      <td>dia (laki-laki)</td>\n",
       "      <td></td>\n",
       "      <td>https://www.book2.nl/book2/ID/SOUND/0006.mp3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>People</td>\n",
       "      <td>5</td>\n",
       "      <td>he and she</td>\n",
       "      <td>彼と 彼女</td>\n",
       "      <td>kare to kanojo</td>\n",
       "      <td>https://www.book2.nl/book2/JA/SOUND/0007.mp3</td>\n",
       "      <td>5</td>\n",
       "      <td>dia (laki-laki) dan dia (perempuan)</td>\n",
       "      <td></td>\n",
       "      <td>https://www.book2.nl/book2/ID/SOUND/0007.mp3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  chapter  index_x        text translate_x        translit_x  \\\n",
       "0  People        1           I           私           watashi   \n",
       "1  People        2   I and you      私と あなた  watashi to anata   \n",
       "2  People        3  both of us          私達      watashitachi   \n",
       "3  People        4          he           彼              kare   \n",
       "4  People        5  he and she       彼と 彼女    kare to kanojo   \n",
       "\n",
       "                                        audio_x  index_y  \\\n",
       "0  https://www.book2.nl/book2/JA/SOUND/0003.mp3        1   \n",
       "1  https://www.book2.nl/book2/JA/SOUND/0004.mp3        2   \n",
       "2  https://www.book2.nl/book2/JA/SOUND/0005.mp3        3   \n",
       "3  https://www.book2.nl/book2/JA/SOUND/0006.mp3        4   \n",
       "4  https://www.book2.nl/book2/JA/SOUND/0007.mp3        5   \n",
       "\n",
       "                           translate_y translit_y  \\\n",
       "0                                 saya              \n",
       "1                        saya dan kamu              \n",
       "2                          kami berdua              \n",
       "3                      dia (laki-laki)              \n",
       "4  dia (laki-laki) dan dia (perempuan)              \n",
       "\n",
       "                                        audio_y  \n",
       "0  https://www.book2.nl/book2/ID/SOUND/0003.mp3  \n",
       "1  https://www.book2.nl/book2/ID/SOUND/0004.mp3  \n",
       "2  https://www.book2.nl/book2/ID/SOUND/0005.mp3  \n",
       "3  https://www.book2.nl/book2/ID/SOUND/0006.mp3  \n",
       "4  https://www.book2.nl/book2/ID/SOUND/0007.mp3  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df = pd.merge(dfjson_ja, dfjson_id,  how='left', left_on=['text','chapter'], right_on = ['text','chapter'])\n",
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T14:35:29.227979Z",
     "start_time": "2019-11-17T14:35:29.185915Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'chapter': 'People',\n",
       "  'index': 3,\n",
       "  'text': 'both of us',\n",
       "  'translate': '私達',\n",
       "  'translit': 'watashitachi',\n",
       "  'audio': 'https://www.book2.nl/book2/JA/SOUND/0005.mp3'}]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "def search_in(text, lang):\n",
    "    with open(f'/pi/stack/crawlers/langcrs/all_{lang}.json') as json_file:\n",
    "        sents=json.load(json_file)\n",
    "        return [sent for sent in sents if sent['text']==text]\n",
    "def search_in_list(text, langs):\n",
    "    rs={}\n",
    "    for lang in langs:\n",
    "        rs[lang]=search_in(text, lang)\n",
    "    return rs\n",
    "\n",
    "search_in('both of us', 'ja')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T14:36:27.810108Z",
     "start_time": "2019-11-17T14:36:27.708321Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ja': [{'chapter': 'Reading and writing',\n",
       "   'index': 14,\n",
       "   'text': 'I write a letter.',\n",
       "   'translate': '私は 手紙を 書きます 。',\n",
       "   'translit': 'watashi wa tegami o kakimasu.',\n",
       "   'audio': 'https://www.book2.nl/book2/JA/SOUND/0116.mp3'}],\n",
       " 'fa': [{'chapter': 'Reading and writing',\n",
       "   'index': 14,\n",
       "   'text': 'I write a letter.',\n",
       "   'translate': '\\u202bمن یک نامه می\\u200cنویسم.\\u202c',\n",
       "   'translit': 'man yek nâme râ minevisam.',\n",
       "   'audio': 'https://www.book2.nl/book2/FA/SOUND/0116.mp3'}],\n",
       " 'id': [{'chapter': 'Reading and writing',\n",
       "   'index': 14,\n",
       "   'text': 'I write a letter.',\n",
       "   'translate': 'Saya menulis selembar surat.',\n",
       "   'translit': '',\n",
       "   'audio': 'https://www.book2.nl/book2/ID/SOUND/0116.mp3'}]}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search_in_list('I write a letter.', ['ja', 'fa', 'id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T06:44:31.258282Z",
     "start_time": "2019-11-17T06:44:31.136637Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from bert_serving.client import BertClient\n",
    "\n",
    "# Bert Client must be running locally\n",
    "bc = BertClient()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T06:47:23.571928Z",
     "start_time": "2019-11-17T06:44:37.561052Z"
    }
   },
   "outputs": [],
   "source": [
    "def train(quotes):\n",
    "    embeddings = bc.encode(quotes.text.to_list())\n",
    "    quotes['EMBEDDINGS'] = embeddings.tolist()\n",
    "\n",
    "    # Persist to pickle\n",
    "    quotes.to_pickle('data/embedded_corpus.pkl')\n",
    "train(dfjson)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T06:53:53.169031Z",
     "start_time": "2019-11-17T06:53:52.978839Z"
    }
   },
   "outputs": [],
   "source": [
    "def load_quotes_and_embeddings(file):\n",
    "    quotes = pd.read_pickle(file)\n",
    "\n",
    "    # change dtype in place for memory efficiency\n",
    "    quotes['EMBEDDINGS'] = quotes['EMBEDDINGS'].apply(\n",
    "        lambda arr: np.array(arr, dtype='float32')\n",
    "    )\n",
    "\n",
    "    quote_embeddings = np.stack(quotes.EMBEDDINGS.values)\n",
    "\n",
    "    # reduce memory footprint by dropping column\n",
    "    quotes.drop('EMBEDDINGS', axis='columns')\n",
    "\n",
    "    # normalize embeddings for cosine distance\n",
    "    embedding_sums = quote_embeddings.sum(axis=1)\n",
    "    normed_embeddings = quote_embeddings / embedding_sums[:, np.newaxis]\n",
    "    return quotes, normed_embeddings\n",
    "\n",
    "quotes, embeddings = load_quotes_and_embeddings('data/embedded_corpus.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T06:54:24.982167Z",
     "start_time": "2019-11-17T06:54:24.934058Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_index(embeddings):\n",
    "    import faiss\n",
    "    \"\"\"\n",
    "    Create an index over the quote embeddings for fast similarity search.\n",
    "    \"\"\"\n",
    "    dim = embeddings.shape[1]\n",
    "    index = faiss.IndexFlatL2(dim)\n",
    "    index.add(embeddings)\n",
    "    return index\n",
    "\n",
    "index = create_index(embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T11:07:13.198222Z",
     "start_time": "2019-11-17T11:07:13.191129Z"
    }
   },
   "outputs": [],
   "source": [
    "def search(text):\n",
    "    text_embedding = bc.encode([text])\n",
    "    normalized_text_embedding = text_embedding / text_embedding.sum()\n",
    "\n",
    "    _, idx = index.search(normalized_text_embedding, 5)\n",
    "\n",
    "    relevant_quotes = quotes.iloc[idx.flatten()]['text'].values\n",
    "    relevant_chapters = quotes.iloc[idx.flatten()]['chapter'].values\n",
    "    return relevant_quotes, relevant_chapters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T11:07:15.148244Z",
     "start_time": "2019-11-17T11:07:14.896571Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">I read a letter.\n",
      "Reading and writing\n",
      ">I read a book.\n",
      "Reading and writing\n",
      ">I want to sleep.\n",
      "to need – to want to\n",
      ">I read a sentence.\n",
      "Reading and writing\n",
      ">He wrote a letter.\n",
      "Past tense 1\n"
     ]
    }
   ],
   "source": [
    "# bc = BertClient()\n",
    "text=\"I dreamed a dream.\"\n",
    "\n",
    "relevant_quotes, relevant_chapters=search(text)\n",
    "for q in range(5):\n",
    "    print('>'+relevant_quotes[q])\n",
    "    print(relevant_chapters[q])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T10:58:55.446150Z",
     "start_time": "2019-11-17T10:58:55.203968Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">I write a letter.\n",
      "Reading and writing\n",
      ">I write a book.\n",
      "Reading and writing\n",
      ">I write.\n",
      "Reading and writing\n",
      ">I write.\n",
      "Reading and writing\n",
      ">I write a sentence.\n",
      "Reading and writing\n"
     ]
    }
   ],
   "source": [
    "text=\"I write a letter.\"\n",
    "\n",
    "relevant_quotes, relevant_chapters=search(text)\n",
    "for q in range(5):\n",
    "    print('>'+relevant_quotes[q])\n",
    "    print(relevant_chapters[q])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-11-17T11:18:53.624142Z",
     "start_time": "2019-11-17T11:18:53.616339Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n"
     ]
    }
   ],
   "source": [
    "x,y=[1,2]\n",
    "print(y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
