{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp nlp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLP\n",
    "\n",
    "> API details."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "from collections import Counter\n",
    "from textblob import TextBlob\n",
    "\n",
    "from htools import save, load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "class Vocabulary:\n",
    "\n",
    "    def __init__(self, w2idx, w2vec=None, idx_misc=None, corpus_counts=None,\n",
    "                 all_lower=True):\n",
    "        \"\"\"Defines a vocabulary object for NLP problems, allowing users to\n",
    "        encode text with indices or embeddings.\n",
    "\n",
    "        Parameters\n",
    "        -----------\n",
    "        w2idx: dict[str, int]\n",
    "            Dictionary mapping words to their integer index in a vocabulary.\n",
    "            The indices must allow for idx_misc to be added to the dictionary,\n",
    "            so in the default case this should have a minimum index of 2. If\n",
    "            a longer idx_misc is passed in, the minimum index would be larger.\n",
    "        w2vec: dict[str, np.array]\n",
    "            Dictionary mapping words to their embedding vectors stored as\n",
    "            numpy arrays (optional).\n",
    "        idx_misc: dict\n",
    "            A dictionary mapping non-word tokens to indices. If none is passed\n",
    "            in, a default version will be used with keys for unknown tokens\n",
    "            and padding. A customized version might pass in additional tokens\n",
    "            for repeated characters or all caps, for example.\n",
    "        corpus_counts: collections.Counter\n",
    "            Counter dict mapping words to their number of occurrences in a\n",
    "            corpus (optional).\n",
    "        all_lower: bool\n",
    "            Specifies whether the data you've passed in (w2idx, w2vec, i2w) is\n",
    "            all lowercase. Note that this will NOT change any of this data. If\n",
    "            True, it simply lowercases user-input words when looking up their\n",
    "            index or vector.\n",
    "        \"\"\"\n",
    "        if not idx_misc:\n",
    "            idx_misc = {'<PAD>': 0,\n",
    "                        '<UNK>': 1}\n",
    "        self.idx_misc = idx_misc\n",
    "        # Check that space has been left for misc keys.\n",
    "        assert len(idx_misc) == min(w2idx.values())\n",
    "\n",
    "        # Core data structures.\n",
    "        self.w2idx = {**self.idx_misc, **w2idx}\n",
    "        self.i2w = [word for word, idx in sorted(self.w2idx.items(),\n",
    "                                                 key=lambda x: x[1])]\n",
    "        self.w2vec = w2vec or dict()\n",
    "\n",
    "        # Miscellaneous other attributes.\n",
    "        if w2vec:\n",
    "            self.dim = len(w2vec[self[-1]])\n",
    "        else:\n",
    "            self.dim = 1\n",
    "        self.corpus_counts = corpus_counts\n",
    "        self.embedding_matrix = None\n",
    "        self.w2vec['<UNK>'] = np.zeros(self.dim)\n",
    "        self.all_lower = all_lower\n",
    "\n",
    "    @classmethod\n",
    "    def from_glove_file(cls, path, max_lines=float('inf'), idx_misc=None):\n",
    "        \"\"\"Create a new Vocabulary object by loading GloVe vectors from a text\n",
    "        file. The embeddings are all lowercase so the user does not have the\n",
    "        option to set the all_lower parameter.\n",
    "\n",
    "        Parameters\n",
    "        -----------\n",
    "        path: str\n",
    "            Path to file containing glove vectors.\n",
    "        max_lines: int, float (optional)\n",
    "            Loading the GloVe vectors can be slow, so for testing purposes\n",
    "            it can be helpful to read in a subset. If no value is provided,\n",
    "            all 400,000 lines in the file will be read in.\n",
    "        idx_misc: dict\n",
    "            Map non-standard tokens to indices. See constructor docstring.\n",
    "        \"\"\"\n",
    "        w2idx = dict()\n",
    "        w2vec = dict()\n",
    "        misc_len = 2 if not idx_misc else len(idx_misc)\n",
    "\n",
    "        with open(path, 'r') as f:\n",
    "            for i, line in enumerate(f):\n",
    "                if i >= max_lines:\n",
    "                    break\n",
    "                word, *values = line.strip().split(' ')\n",
    "                w2idx[word] = i + misc_len\n",
    "                w2vec[word] = np.array(values, dtype=np.float)\n",
    "\n",
    "        return cls(w2idx, w2vec, idx_misc)\n",
    "\n",
    "    @classmethod\n",
    "    def from_tokens(cls, tokens, idx_misc=None, all_lower=True):\n",
    "        \"\"\"Construct a Vocabulary object from a list or array of tokens.\n",
    "\n",
    "        Parameters\n",
    "        -----------\n",
    "        tokens: list[str]\n",
    "            The word-tokenized corpus.\n",
    "        idx_misc: dict\n",
    "            Map non-standard tokens to indices. See constructor docstring.\n",
    "        all_lower: bool\n",
    "            Specifies whether your tokens are all lowercase.\n",
    "\n",
    "        Returns\n",
    "        --------\n",
    "        Vocabulary\n",
    "        \"\"\"\n",
    "        misc_len = 2 if not idx_misc else len(idx_misc)\n",
    "        counts = Counter(tokens)\n",
    "        w2idx = {word: i for i, (word, freq)\n",
    "                 in enumerate(counts.most_common(), misc_len)}\n",
    "        return cls(w2idx, idx_misc=idx_misc, corpus_counts=counts,\n",
    "                   all_lower=all_lower)\n",
    "\n",
    "    @staticmethod\n",
    "    def from_pickle(path):\n",
    "        \"\"\"Load a previously saved Vocabulary object.\n",
    "\n",
    "        Parameters\n",
    "        -----------\n",
    "        path: str\n",
    "            Location of pickled Vocabulary file.\n",
    "\n",
    "        Returns\n",
    "        --------\n",
    "        Vocabulary\n",
    "        \"\"\"\n",
    "        return load(path)\n",
    "\n",
    "    def save(self, path, verbose=True):\n",
    "        \"\"\"Pickle Vocabulary object for later use. We can then quickly load\n",
    "        the object using torch.load(path), which can be much faster than\n",
    "        re-computing everything when the vocab size becomes large.\n",
    "\n",
    "        Parameters\n",
    "        -----------\n",
    "        path: str\n",
    "            Where to save the output file.\n",
    "        verbose: bool\n",
    "            If True, print message showing where the object was saved to.\n",
    "        \"\"\"\n",
    "        save(self, path, verbose)\n",
    "\n",
    "    def filter_tokens(self, tokens, max_words=None, min_freq=0, inplace=False,\n",
    "                      recompute=False):\n",
    "        \"\"\"Filter your vocabulary by specifying a max number of words or a min\n",
    "        frequency in the corpus. When done in place, this also sorts vocab by\n",
    "        frequency with more common words coming first (after idx_misc).\n",
    "\n",
    "        Parameters\n",
    "        -----------\n",
    "        tokens: list[str]\n",
    "            A tokenized list of words in the corpus (must be all lowercase\n",
    "            when self.all_lower=True, such as when using GloVe vectors). There\n",
    "            is no need to hold out test data here since we are not using\n",
    "            labels.\n",
    "        max_words: int (optional)\n",
    "            Provides an upper threshold for the number of words in the\n",
    "            vocabulary. If no value is passed in, no maximum limit will be\n",
    "            enforced.\n",
    "        min_freq: int (optional)\n",
    "            Provides a lower threshold for the number of times a word must\n",
    "            appear in the corpus to remain in the vocabulary. If no value is\n",
    "            passed in, no minimum limit will be enforced.\n",
    "\n",
    "            Note that we can specify values for both max_words and min_freq\n",
    "            if desired. If no values are passed in for either, no pruning of\n",
    "            the vocabulary will be performed.\n",
    "        inplace: bool\n",
    "            If True, will change the object's attributes\n",
    "            (w2idx, w2vec, and i2w) to reflect the newly filtered vocabulary.\n",
    "            If False, will not change the object, but will simply compute word\n",
    "            counts and return what the new w2idx would be. This can be helpful\n",
    "            for experimentation, as we may want to try out multiple values of\n",
    "            min_freq to decide how many words to keep. After the first call,\n",
    "            the attribute corpus_counts can also be examined to help determine\n",
    "            the desired vocab size.\n",
    "        recompute: bool\n",
    "            If True, will calculate word counts from the given tokens. If\n",
    "            False (the default), this will use existing counts if there are\n",
    "            any.\n",
    "\n",
    "            The idea is that if we call this method, then realize we want\n",
    "            to change the corpus, we should calculate new word counts.\n",
    "            However, if we are simply calling this method multiple times on\n",
    "            the same corpus while deciding on the exact vocab size we want,\n",
    "            we should not recompute the word counts.\n",
    "\n",
    "        Returns\n",
    "        --------\n",
    "        dict or None: When called inplace, nothing is returned. When not\n",
    "        inplace,\n",
    "        \"\"\"\n",
    "        misc_len = len(self.idx_misc)\n",
    "        if recompute or not self.corpus_counts:\n",
    "            self.corpus_counts = Counter(tokens)\n",
    "        filtered = {word: i for i, (word, freq)\n",
    "                    in enumerate(self.corpus_counts.most_common(max_words),\n",
    "                                 misc_len)\n",
    "                    if freq >= min_freq}\n",
    "        filtered = {**self.idx_misc, **filtered}\n",
    "\n",
    "        if inplace:\n",
    "            # Relies on python3.7 dicts retaining insertion order.\n",
    "            self.i2w = list(filtered.keys())\n",
    "            self.w2idx = filtered\n",
    "            self.w2vec = {word: self.vector(word) for word in filtered}\n",
    "        else:\n",
    "            return filtered\n",
    "\n",
    "    def build_embedding_matrix(self, inplace=False):\n",
    "        \"\"\"Create a 2D numpy array of embedding vectors where row[i]\n",
    "        corresponds to word i in the vocabulary. This can be used to\n",
    "        initialize weights in the model's embedding layer.\n",
    "\n",
    "        Parameters\n",
    "        -----------\n",
    "        inplace: bool\n",
    "            If True, will store the output in the object's embedding_matrix\n",
    "            attribute. If False (default behavior), will simply return the\n",
    "            matrix without storing it as part of the object. In the\n",
    "            recommended case where inplace==False, we can store the output\n",
    "            in another variable which we can use to initialize the weights in\n",
    "            Torch, then delete the object and free up memory using\n",
    "            gc.collect().\n",
    "        \"\"\"\n",
    "        emb = np.zeros((len(self), self.dim))\n",
    "        for i, word in enumerate(self):\n",
    "            emb[i] = self.vector(word)\n",
    "\n",
    "        if inplace:\n",
    "            self.embedding_matrix = emb\n",
    "        else:\n",
    "            return emb\n",
    "\n",
    "    def idx(self, word):\n",
    "        \"\"\"This will map a word (str) to its index (int) in the vocabulary.\n",
    "        If a string is passed in and the word is not present, the index\n",
    "        corresponding to the <UNK> token is returned.\n",
    "\n",
    "        Parameters\n",
    "        -----------\n",
    "        word: str\n",
    "            A word that needs to be mapped to an integer index.\n",
    "\n",
    "        Returns\n",
    "        --------\n",
    "        int: The index of the given word in the vocabulary.\n",
    "\n",
    "        Examples\n",
    "        ---------\n",
    "        >>> vocab.idx('the')\n",
    "        2\n",
    "        \"\"\"\n",
    "        if self.all_lower and word not in self.idx_misc:\n",
    "            word = word.lower()\n",
    "        return self.w2idx.get(word, self.w2idx['<UNK>'])\n",
    "\n",
    "    def vector(self, word):\n",
    "        \"\"\"This maps a word to its corresponding embedding vector. If not\n",
    "        contained in the vocab, a vector of zeros will be returned.\n",
    "\n",
    "        Parameters\n",
    "        -----------\n",
    "        word: str\n",
    "            A word that needs to be mapped to a vector.\n",
    "\n",
    "        Returns\n",
    "        --------\n",
    "        np.array\n",
    "        \"\"\"\n",
    "        if self.all_lower and word not in self.idx_misc:\n",
    "            word = word.lower()\n",
    "        return self.w2vec.get(word, self.w2vec['<UNK>'])\n",
    "\n",
    "    def encode(self, text, nlp, max_len, pad_end=True, trim_start=True):\n",
    "        \"\"\"Encode text so that each token is replaced by its integer index in\n",
    "        the vocab.\n",
    "\n",
    "        Parameters\n",
    "        -----------\n",
    "        text: str\n",
    "            Raw text to be encoded.\n",
    "        nlp: spacy.lang.en.English\n",
    "            Spacy tokenizer. Typically want to disable 'parser', 'tagger', and\n",
    "            'ner' as they aren't used here and slow down the encoding process.\n",
    "        max_len: int\n",
    "            Length of output encoding. If text is shorter, it will be padded\n",
    "            to fit the specified length. If text is longer, it will be\n",
    "            trimmed.\n",
    "        pad_end: bool\n",
    "            If True, add padding to the end of short sentences. If False, pad\n",
    "            the start of these sentences.\n",
    "        trim_start: bool\n",
    "            If True, trim off the start of sentences that are too long. If\n",
    "            False, trim off the end.\n",
    "\n",
    "        Returns\n",
    "        --------\n",
    "        np.array[int]: Array of length max_len containing integer indices\n",
    "            corresponding to the words passed in.\n",
    "        \"\"\"\n",
    "        output = np.ones(max_len) * self.idx('<PAD>')\n",
    "        encoded = [self.idx(tok.text) for tok in nlp(text)]\n",
    "\n",
    "        # Trim sentence in case it's longer than max_len.\n",
    "        if len(encoded) > max_len:\n",
    "            if trim_start:\n",
    "                encoded = encoded[len(encoded) - max_len:]\n",
    "            else:\n",
    "                encoded = encoded[:max_len]\n",
    "\n",
    "        # Replace zeros at start or end, depending on choice of pad_end.\n",
    "        if pad_end:\n",
    "            output[:len(encoded)] = encoded\n",
    "        else:\n",
    "            output[max_len-len(encoded):] = encoded\n",
    "        return output.astype(int)\n",
    "\n",
    "    def decode(self, idx):\n",
    "        \"\"\"Convert a list of indices to a string of words/tokens.\n",
    "\n",
    "        Parameters\n",
    "        -----------\n",
    "        idx: list[int]\n",
    "            A list of integers indexing into the vocabulary. This will often\n",
    "            be the output of the encode() method.\n",
    "\n",
    "        Returns\n",
    "        --------\n",
    "        list[str]: A list of words/tokens reconstructed by indexing into the\n",
    "            vocabulary.\n",
    "        \"\"\"\n",
    "        return [self[i] for i in idx]\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        \"\"\"This will map an index (int) to a word (str).\n",
    "\n",
    "        Parameters\n",
    "        -----------\n",
    "        i: int\n",
    "            Integer index for a word.\n",
    "\n",
    "        Returns\n",
    "        --------\n",
    "        str: Word corresponding to the given index.\n",
    "\n",
    "        Examples\n",
    "        ---------\n",
    "        >>> vocab = Vocabulary(w2idx, w2vec)\n",
    "        >>> vocab[1]\n",
    "        '<UNK>'\n",
    "        \"\"\"\n",
    "        return self.i2w[i]\n",
    "\n",
    "    def __len__(self):\n",
    "        \"\"\"Number of words in vocabulary.\"\"\"\n",
    "        return len(self.w2idx)\n",
    "\n",
    "    def __iter__(self):\n",
    "        for word in self.w2idx.keys():\n",
    "            yield word\n",
    "\n",
    "    def __contains__(self, word):\n",
    "        return word in self.w2idx.keys()\n",
    "\n",
    "    def __eq__(self, obj):\n",
    "        if not isinstance(obj, Vocabulary):\n",
    "            return False\n",
    "\n",
    "        ignore = {'w2vec', 'embedding_matrix'}\n",
    "        attrs = [k for k, v in hdir(vocab).items()\n",
    "                 if v == 'attribute' and k not in ignore]\n",
    "        return all([getattr(self, attr) == getattr(obj, attr)\n",
    "                    for attr in attrs])\n",
    "\n",
    "    def __repr__(self):\n",
    "        msg = f'Vocabulary({len(self)} words'\n",
    "        if self.dim > 1:\n",
    "            msg += f', {self.dim}-D embeddings'\n",
    "        return msg + ')'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "def back_translate(text, to, from_lang='en'):\n",
    "    \"\"\"\n",
    "    Parameters\n",
    "    ----------\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    str: Same language and basically the same content as the original text, \n",
    "        but usually with slightly altered grammar, sentence structure, and/or\n",
    "        vocabulary.\n",
    "    \"\"\"\n",
    "    return str(\n",
    "        TextBlob(text)\\\n",
    "        .translate(to=to)\\\n",
    "        .translate(from_lang=to, to=from_lang)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Visit ESPN to get coverage of sports news, scores, highlights and comments from the NFL, MLB, NBA, college football, NCAA basketball and more.'"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"\"\"\n",
    "Visit ESPN to get up-to-the-minute sports news coverage, scores, highlights and commentary for NFL, MLB, NBA, College Football, NCAA Basketball and more.\n",
    "\"\"\"\n",
    "back_translate(text, 'es')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Visit ESPN for up-to-date sports information, scores, highlights and commentary for the NFL, MLB, NBA, college football, NCAA basketball and more.'"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"\"\"\n",
    "Visit ESPN to get up-to-the-minute sports news coverage, scores, highlights and commentary for NFL, MLB, NBA, College Football, NCAA Basketball and more.\n",
    "\"\"\"\n",
    "back_translate(text, 'fr')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
