{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import multiprocessing\n",
    "import pandas as pd\n",
    "import spacy\n",
    "from spacy.lang.en import English\n",
    "from spacy.tokenizer import Tokenizer\n",
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = [\n",
    "    f\"{i} I went to the park, yesterday, wasn't here after school? Today. --2\"\n",
    "    for i in range(50_000)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0 I went to the park, yesterday, wasn't here a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1 I went to the park, yesterday, wasn't here a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2 I went to the park, yesterday, wasn't here a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3 I went to the park, yesterday, wasn't here a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4 I went to the park, yesterday, wasn't here a...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   a\n",
       "0  0 I went to the park, yesterday, wasn't here a...\n",
       "1  1 I went to the park, yesterday, wasn't here a...\n",
       "2  2 I went to the park, yesterday, wasn't here a...\n",
       "3  3 I went to the park, yesterday, wasn't here a...\n",
       "4  4 I went to the park, yesterday, wasn't here a..."
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(text, columns=['a'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<spacy.lang.en.English at 0x12a251a58>"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp = spacy.load('en_core_web_sm')\n",
    "nlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ~400 seconds\n",
    "x = df.a.apply(nlp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_sm',\n",
    "                 disable=['ner', 'tagger', 'parser'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        (0, I, went, to, the, park, ,, yesterday, ,, w...\n",
       "1        (1, I, went, to, the, park, ,, yesterday, ,, w...\n",
       "2        (2, I, went, to, the, park, ,, yesterday, ,, w...\n",
       "3        (3, I, went, to, the, park, ,, yesterday, ,, w...\n",
       "4        (4, I, went, to, the, park, ,, yesterday, ,, w...\n",
       "                               ...                        \n",
       "49995    (49995, I, went, to, the, park, ,, yesterday, ...\n",
       "49996    (49996, I, went, to, the, park, ,, yesterday, ...\n",
       "49997    (49997, I, went, to, the, park, ,, yesterday, ...\n",
       "49998    (49998, I, went, to, the, park, ,, yesterday, ...\n",
       "49999    (49999, I, went, to, the, park, ,, yesterday, ...\n",
       "Name: a, Length: 50000, dtype: object"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ~9 seconds after disabling extra parsers.\n",
    "df.a.apply(nlp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(x):\n",
    "    return [t.text for t in nlp(x)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        [0, I, went, to, the, park, ,, yesterday, ,, w...\n",
       "1        [1, I, went, to, the, park, ,, yesterday, ,, w...\n",
       "2        [2, I, went, to, the, park, ,, yesterday, ,, w...\n",
       "3        [3, I, went, to, the, park, ,, yesterday, ,, w...\n",
       "4        [4, I, went, to, the, park, ,, yesterday, ,, w...\n",
       "                               ...                        \n",
       "49995    [49995, I, went, to, the, park, ,, yesterday, ...\n",
       "49996    [49996, I, went, to, the, park, ,, yesterday, ...\n",
       "49997    [49997, I, went, to, the, park, ,, yesterday, ...\n",
       "49998    [49998, I, went, to, the, park, ,, yesterday, ...\n",
       "49999    [49999, I, went, to, the, park, ,, yesterday, ...\n",
       "Name: a, Length: 50000, dtype: object"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ~9 seconds with text extraction (unchanged)\n",
    "df.a.apply(tokenize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ~5 seconds with multiprocessing\n",
    "with multiprocessing.Pool(5) as p:\n",
    "    x = p.map(tokenize, df.a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ~5 seconds as long as chunksize is large\n",
    "with multiprocessing.Pool() as p:\n",
    "    y = list(p.imap(tokenize, df.a, chunksize=1_000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from concurrent import futures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ~10 seconds\n",
    "with futures.ThreadPoolExecutor(max_workers=4) as ex:\n",
    "    res = list(ex.map(tokenize, df.a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7da3b98607454e01adfb87099e52692d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# ~12 seconds - Fluent Python book code\n",
    "res = []\n",
    "with futures.ThreadPoolExecutor(max_workers=4) as executor:\n",
    "    to_do_map = {}\n",
    "    for text in df.a.values:\n",
    "        future = executor.submit(tokenize, text)\n",
    "        to_do_map[future] = text\n",
    "        done_iter = futures.as_completed(to_do_map)\n",
    "    done_iter = tqdm(done_iter, total=df.shape[0])\n",
    "    for future in done_iter:\n",
    "        res.append(future.result())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trying to figure out diff between nlp() and separate tokenizer\n",
    "\n",
    "Followup: A stackoverflow commenter claims this is just splitting on white space because we haven't defined any rules. But isn't that what the English() object is doing?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eng = English()\n",
    "tok = Tokenizer(eng.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x1 = df.a.head(100).apply(tokenize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_tok(text):\n",
    "    return [t.text for t in tok(text)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x2 = df.a.head(100).apply(tokenize_tok)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('park', 'park,'),\n",
       " (',', 'yesterday,'),\n",
       " ('yesterday', \"wasn't\"),\n",
       " (',', 'here'),\n",
       " ('was', 'after'),\n",
       " (\"n't\", 'school?'),\n",
       " ('here', 'Today.'),\n",
       " ('after', '--2')]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(k1, k2) for k1, k2 in zip(x1[0], x2[0]) if k1 != k2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('0', '0'),\n",
       " ('I', 'I'),\n",
       " ('went', 'went'),\n",
       " ('to', 'to'),\n",
       " ('the', 'the'),\n",
       " ('park', 'park,'),\n",
       " (',', 'yesterday,'),\n",
       " ('yesterday', \"wasn't\"),\n",
       " (',', 'here'),\n",
       " ('was', 'after'),\n",
       " (\"n't\", 'school?'),\n",
       " ('here', 'Today.'),\n",
       " ('after', '--2')]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(x1[0], x2[0]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
