{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary\n",
    "\n",
    "`# TODO: summary here`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:19:22.783011Z",
     "start_time": "2020-11-19T04:19:22.751141Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:19:27.540828Z",
     "start_time": "2020-11-19T04:19:23.026075Z"
    }
   },
   "outputs": [],
   "source": [
    "from abc import ABC, abstractmethod\n",
    "from collections.abc import Iterable, Mapping\n",
    "import gc\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "from pathlib import Path\n",
    "import torch\n",
    "from transformers import PegasusForConditionalGeneration, PegasusTokenizer, \\\n",
    "    PegasusTokenizerFast, pipeline\n",
    "\n",
    "from htools import *\n",
    "from incendio.utils import DEVICE, gpu_setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:18:25.193311Z",
     "start_time": "2020-11-19T04:18:25.182964Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hmamin/anaconda3/lib/python3.7/site-packages/ipykernel/ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n",
      "/Users/hmamin/anaconda3/lib/python3.7/site-packages/incendio/utils.py:38: UserWarning: Cuda not available.\n",
      "  if not torch.cuda.is_available(): warnings.warn('Cuda not available.')\n",
      "/Users/hmamin/anaconda3/lib/python3.7/site-packages/incendio/utils.py:41: UserWarning: Incendio device is not cuda.\n",
      "  warnings.warn('Incendio device is not cuda.')\n"
     ]
    }
   ],
   "source": [
    "gpu_setup(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T03:57:21.223228Z",
     "start_time": "2020-11-09T03:56:57.662913Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hmamin/anaconda3/lib/python3.7/site-packages/transformers/convert_slow_tokenizer.py:69: ResourceWarning: unclosed file <_io.BufferedReader name='/Users/hmamin/.cache/torch/transformers/fa4532c0035b101d7abcd5c0c9c34a83288902b66c5616034db1a47643e05c75.efce77b8dcd2c57b109b0d10170fcdcd53f23c21286974d4f66706536758ab6e'>\n",
      "  m.ParseFromString(open(filename, \"rb\").read())\n",
      "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n"
     ]
    }
   ],
   "source": [
    "version = 'tuner007/pegasus_paraphrase'\n",
    "net = PegasusForConditionalGeneration.from_pretrained(version).to(DEVICE)\n",
    "tok = PegasusTokenizerFast.from_pretrained(version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T03:58:28.625990Z",
     "start_time": "2020-11-09T03:58:28.610237Z"
    }
   },
   "outputs": [],
   "source": [
    "texts = [\n",
    "    'Educational games and digital learning materials to provide K-12 '\n",
    "    'students with enriching experiences.',\n",
    "    'The world\\'s largest social network. Helping people build and maintain '\n",
    "    'relationships in a disconnected world.',\n",
    "    'I hate school. I wish my teacher would leave me alone. I don\\'t think '\n",
    "    ' he likes me.',\n",
    "    'Today the president announced new plans to revamp the private '\n",
    "    'healthcare system. Pundits questioned how he would manage to pass the '\n",
    "    'bill.'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-03T05:11:06.682140Z",
     "start_time": "2020-11-03T05:11:06.638522Z"
    }
   },
   "outputs": [],
   "source": [
    "res = tok.prepare_seq2seq_batch(texts, truncation=True, padding='longest')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-03T05:11:08.427646Z",
     "start_time": "2020-11-03T05:11:08.403336Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[11263,   727,   111,  1016,   761,   917,   112,   319,  1046,  6054,\n",
       "           392,   122, 26838,  1747,   107,     1,     0,     0,     0,     0,\n",
       "             0,     0,     0],\n",
       "        [  139,   278,   131,   116,  1368,   525,   952,   107, 22844,   200,\n",
       "           736,   111,  1634,  2074,   115,   114, 20402,   278,   107,     1,\n",
       "             0,     0,     0],\n",
       "        [  125,  4180,   399,   107,   125,  1216,   161,  2118,   192,   858,\n",
       "           213,  1600,   107,   125,   272,   131,   144,   311,   178,  5606,\n",
       "           213,   107,     1]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],\n",
       "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],\n",
       "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-03T05:13:07.112402Z",
     "start_time": "2020-11-03T05:12:49.471597Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[   0, 1046, 6054,  392,  137,  207, 2387,  727,  111, 1016,  761,  917,\n",
       "          107,    1,    0,    0,    0],\n",
       "        [   0,  139,  278,  131,  116, 1368,  525,  952,  107,    1,    0,    0,\n",
       "            0,    0,    0,    0,    0],\n",
       "        [   0,  125,  272,  131,  144,  172,  399,  111,  161, 2118,  591,  131,\n",
       "          144,  172,  213,  107,    1]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gen = net.generate(**res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-03T05:15:57.224563Z",
     "start_time": "2020-11-03T05:15:57.215548Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['K-12 students can use educational games and digital learning materials.',\n",
       " \"The world's largest social network.\",\n",
       " \"I don't like school and my teacher doesn't like me.\"]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tok.batch_decode(gen.tolist(), skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-03T05:36:34.347485Z",
     "start_time": "2020-11-03T05:36:34.337003Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hmamin/anaconda3/lib/python3.7/site-packages/ipykernel/ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    }
   ],
   "source": [
    "@add_docstring(net.generate)\n",
    "def paraphrase(text, n=1, temperature=1.5, **gen_kwargs):\n",
    "    batch = tok.prepare_seq2seq_batch([text], truncation=True, \n",
    "                                      padding='longest').to(DEVICE)\n",
    "    paraphrased = net.generate(**batch, num_return_sequences=n,\n",
    "                               temperature=temperature, **gen_kwargs)\n",
    "    return tok.batch_decode(paraphrased.tolist(), skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-03T05:35:36.051823Z",
     "start_time": "2020-11-03T05:35:21.489262Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['The president announced plans to reform the private healthcare system.',\n",
       " 'Pundits wondered how the president would pass the bill.',\n",
       " 'Pundits questioned how the president would pass the bill.',\n",
       " 'Pundits were questioning how the president would pass the bill.',\n",
       " 'Pundits questioned how he would get the bill passed.']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paraphrase(texts[3], n=5, num_beams=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-03T05:35:36.113484Z",
     "start_time": "2020-11-03T05:35:36.066617Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Today the president announced new plans to revamp the private healthcare system. Pundits questioned how he would manage to pass the bill.'"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-03T05:31:10.784475Z",
     "start_time": "2020-11-03T05:31:10.771992Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': <Parameter \"input_ids: Union[torch.LongTensor, NoneType] = None\">,\n",
       " 'decoder_input_ids': <Parameter \"decoder_input_ids: Union[torch.LongTensor, NoneType] = None\">,\n",
       " 'max_length': <Parameter \"max_length: Union[int, NoneType] = None\">,\n",
       " 'min_length': <Parameter \"min_length: Union[int, NoneType] = None\">,\n",
       " 'do_sample': <Parameter \"do_sample: Union[bool, NoneType] = None\">,\n",
       " 'early_stopping': <Parameter \"early_stopping: Union[bool, NoneType] = None\">,\n",
       " 'num_beams': <Parameter \"num_beams: Union[int, NoneType] = None\">,\n",
       " 'temperature': <Parameter \"temperature: Union[float, NoneType] = None\">,\n",
       " 'top_k': <Parameter \"top_k: Union[int, NoneType] = None\">,\n",
       " 'top_p': <Parameter \"top_p: Union[float, NoneType] = None\">,\n",
       " 'repetition_penalty': <Parameter \"repetition_penalty: Union[float, NoneType] = None\">,\n",
       " 'bad_words_ids': <Parameter \"bad_words_ids: Union[Iterable[int], NoneType] = None\">,\n",
       " 'bos_token_id': <Parameter \"bos_token_id: Union[int, NoneType] = None\">,\n",
       " 'pad_token_id': <Parameter \"pad_token_id: Union[int, NoneType] = None\">,\n",
       " 'eos_token_id': <Parameter \"eos_token_id: Union[int, NoneType] = None\">,\n",
       " 'length_penalty': <Parameter \"length_penalty: Union[float, NoneType] = None\">,\n",
       " 'no_repeat_ngram_size': <Parameter \"no_repeat_ngram_size: Union[int, NoneType] = None\">,\n",
       " 'num_return_sequences': <Parameter \"num_return_sequences: Union[int, NoneType] = None\">,\n",
       " 'attention_mask': <Parameter \"attention_mask: Union[torch.LongTensor, NoneType] = None\">,\n",
       " 'decoder_start_token_id': <Parameter \"decoder_start_token_id: Union[int, NoneType] = None\">,\n",
       " 'use_cache': <Parameter \"use_cache: Union[bool, NoneType] = None\">,\n",
       " 'model_kwargs': <Parameter \"**model_kwargs\">}"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.generate()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ParaphrasePipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T04:29:41.438438Z",
     "start_time": "2020-11-09T04:29:41.430001Z"
    }
   },
   "outputs": [],
   "source": [
    "class ParaphrasePipeline:\n",
    "    \n",
    "    def __init__(self, name='tuner007/pegasus_paraphrase', net=None, \n",
    "                 tok=None):\n",
    "        self.name = name\n",
    "        self.net = (net \n",
    "                    or PegasusForConditionalGeneration.from_pretrained(name))\\\n",
    "                    .to(DEVICE)\n",
    "        self.tok = tok or PegasusTokenizerFast.from_pretrained(self.name)\n",
    "        \n",
    "    @add_docstring(PegasusForConditionalGeneration.generate)\n",
    "    def __call__(self, text, n=1, temperature=1.5, **kwargs):\n",
    "        # TODO: not sure how many rows of text can be done in a single batch.\n",
    "        # Maybe look at other pipelines to see what they do. I'm thinking we\n",
    "        # could auto-batch longer sequences (e.g. a list w/ 10_000 strings\n",
    "        # might become 100 batches of 100).\n",
    "        texts = tolist(text)\n",
    "        batch = self.tok.prepare_seq2seq_batch(texts, truncation=True, \n",
    "                                               padding='longest').to(DEVICE)\n",
    "        \n",
    "        # Number of beams must be >= number of sequences to return.\n",
    "        num_beams = max(n, self.net.config.num_beams,\n",
    "                        kwargs.pop('num_beams', -1))\n",
    "        gen_tokens = self.net.generate(**batch, num_return_sequences=n,\n",
    "                                       temperature=temperature,\n",
    "                                       num_beams=num_beams,\n",
    "                                       **kwargs)\n",
    "        gen = self.tok.batch_decode(gen_tokens.tolist(), \n",
    "                                    skip_special_tokens=True)\n",
    "        if not isinstance(text, str) and len(text) > 1: \n",
    "            gen = [gen[i*n:(i+1)*n] for i in range(len(text))]\n",
    "        return gen\n",
    "    \n",
    "# Slightly updated version (if at all) from incendio. Ended up deleting this\n",
    "# # because I found we can use Text2TextGenerationPipeline.\n",
    "# class ParaphrasePipeline:\n",
    "#     \"\"\"Similar to a transformers Pipeline, this provides a high level \n",
    "#     interface for paraphrasing text. It's pretty slow so it's worth using this\n",
    "#     on a GPU when processing many examples.\n",
    "#     \"\"\"\n",
    "    \n",
    "#     def __init__(self, name='tuner007/pegasus_paraphrase', net=None, \n",
    "#                  tok=None):\n",
    "#         \"\"\"\n",
    "#         Parameters\n",
    "#         ----------\n",
    "#         name: str\n",
    "#             Name of pretrained model to load. This will download weights from\n",
    "#             Huggingface's model hub (https://huggingface.co/models. In \n",
    "#             practice the name should rarely change but we want to give users\n",
    "#             the option in case you train a better paraphrasing model.\n",
    "            \n",
    "#             Name                            Parameters    Download Size\n",
    "#             tuner007/pegasus_paraphrase     568,822,784   2.28 GB\n",
    "#             ramsrigouthamg/t5_paraphraser   222,903,936   892 MB\n",
    "            \n",
    "#         net: None or nn.Module\n",
    "#             Pytorch model, usually PegasusForConditionalGeneration. If None,\n",
    "#             a new model will be instantiated.\n",
    "#         tok: None or transformers tokenizer\n",
    "#             A new one will be instantiated by default, but you can also pass\n",
    "#             one in. This must be the correct tokenizer for the `net` being \n",
    "#             used.\n",
    "#         \"\"\"\n",
    "#         self.name = name\n",
    "#         self.net = (net \n",
    "#                     or PegasusForConditionalGeneration.from_pretrained(name))\\\n",
    "#                     .to(DEVICE)\n",
    "#         self.tok = tok or PegasusTokenizerFast.from_pretrained(self.name)\n",
    "        \n",
    "#     @add_docstring(PreTrainedModel.generate)\n",
    "#     def __call__(self, text, n=1, **kwargs):\n",
    "#         \"\"\"Paraphrase one or more pieces of text. We do no auto-batching yet\n",
    "#         so you may need to split your data up into mini batches when working\n",
    "#         with many rows.\n",
    "        \n",
    "#         Parameters\n",
    "#         ----------\n",
    "#         text: str or Iterable[str]\n",
    "#         n: int\n",
    "#             Number of variations to generate per sample.\n",
    "#         kwargs: any\n",
    "#             Passed on to net's generate function. Its docstring is included\n",
    "#             below for convenience.\n",
    "            \n",
    "#         Returns\n",
    "#         -------\n",
    "#         list: If input is a single string, a list of n strings is returned.\n",
    "#         If input is a lsit of strings, a list of nested lists, each of length\n",
    "#         n, is returned.\n",
    "#         \"\"\"\n",
    "#         texts = tolist(text)\n",
    "#         batch = self.tok.prepare_seq2seq_batch(texts, truncation=True, \n",
    "#                                                padding='longest').to(DEVICE)\n",
    "        \n",
    "#         # Number of beams must be >= number of sequences to return.\n",
    "#         num_beams = max(n, self.net.config.num_beams,\n",
    "#                         kwargs.pop('num_beams', -1))\n",
    "#         gen_tokens = self.net.generate(**batch, num_return_sequences=n,\n",
    "#                                        num_beams=num_beams,\n",
    "#                                        **{'temperature': 1.5, **kwargs})\n",
    "#         gen = self.tok.batch_decode(gen_tokens.tolist(), \n",
    "#                                     skip_special_tokens=True)\n",
    "#         if not isinstance(text, str) and len(text) > 1: \n",
    "#             gen = [gen[i*n:(i+1)*n] for i in range(len(text))]\n",
    "#         return gen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T04:29:42.015189Z",
     "start_time": "2020-11-09T04:29:42.001402Z"
    }
   },
   "outputs": [],
   "source": [
    "p_pipe = ParaphrasePipeline(net=net, tok=tok)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T04:29:42.531248Z",
     "start_time": "2020-11-09T04:29:42.519633Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_pipe.net.config.num_beams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T04:29:46.210910Z",
     "start_time": "2020-11-09T04:29:43.264768Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['There was a beautiful day.']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_pipe('It was a beautiful rainy day.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-06T06:22:58.925380Z",
     "start_time": "2020-11-06T06:22:53.288989Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['There was a beautiful day.',\n",
       " 'It was a nice day.',\n",
       " 'It was raining but it was nice.']"
      ]
     },
     "execution_count": 303,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_pipe('It was a beautiful rainy day.', n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-04T02:12:06.328322Z",
     "start_time": "2020-11-04T02:12:02.868563Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['There was a beautiful day.'], ['The duck was fluffy and yellow.']]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_pipe(['It was a beautiful rainy day.', 'The duck was yellow and fluffy.'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-04T02:12:09.898263Z",
     "start_time": "2020-11-04T02:12:06.330746Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['There was a beautiful day.',\n",
       "  'It was a nice day.',\n",
       "  'It was raining but it was nice.'],\n",
       " ['The duck was fluffy and yellow.',\n",
       "  'The duck was fluffy.',\n",
       "  'The duck was big and fluffy.']]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_pipe(['It was a beautiful rainy day.', 'The duck was yellow and fluffy.'], \n",
    "     n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-04T02:12:12.933407Z",
     "start_time": "2020-11-04T02:12:09.901067Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['There was a beautiful day.', 'It was a nice day.']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_pipe(['It was a beautiful rainy day.'], n=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T04:18:34.210761Z",
     "start_time": "2020-11-09T04:18:34.204191Z"
    }
   },
   "outputs": [],
   "source": [
    "# Decided to abandon this. Too many ways children differ: paraphrase pipeline\n",
    "# can't create pipe with pipeline(name) because it's not part of huggingface,\n",
    "# paraphrase transform doesn't need to check if listlike because preprocess\n",
    "# does nothing and __call__ literally just calls pipe. Leaving this here as\n",
    "# an examle of init_subclass.\n",
    "class TransformerTransformBase(ABC):\n",
    "    \n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def __call__(self, text, **kwargs):\n",
    "        if listlike(text):\n",
    "            return [self.transform(t, **kwargs) for t in text]\n",
    "        return self.transform(text, **kwargs)\n",
    "    \n",
    "    @abstractmethod\n",
    "    def transform(self, text, **kwargs):\n",
    "        pass\n",
    "    \n",
    "    def __init_subclass__(cls, **kwargs):\n",
    "        if not hasattr(cls, 'pipe_name'):\n",
    "            raise RuntimeError(f'{cls} must have class attr \"pipe_name\".')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T04:18:35.013005Z",
     "start_time": "2020-11-09T04:18:35.008017Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "As expected, got RuntimeError(<class '__main__.Tmp'> must have class attr \"pipe_name\".).\n"
     ]
    }
   ],
   "source": [
    "with assert_raises(RuntimeError):\n",
    "    class Tmp(TransformerTransformBase):\n",
    "        \"\"\"a\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FillMaskTransform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:11:43.956171Z",
     "start_time": "2020-11-09T05:11:43.937039Z"
    }
   },
   "outputs": [],
   "source": [
    "@auto_repr\n",
    "class FillMaskTransform:    \n",
    "\n",
    "    MASK = '<mask>'\n",
    "    name = 'fill-mask'\n",
    "    \n",
    "    def __init__(self, pipe=None, n=1, max_n=3):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        n: int\n",
    "            n is intentionally bigger than the default n in __call__. This is\n",
    "            the number of candidates generated, so if we use strategy='random'\n",
    "            it makes sense for this to be larger.\n",
    "        \"\"\"\n",
    "        self.pipe = pipe or pipeline(self.name)\n",
    "        self.n = n\n",
    "        self.max_n = max_n\n",
    "        \n",
    "        assert type(self.pipe).__name__ == 'FillMaskPipeline'\n",
    "    \n",
    "    def _preprocess(self, text, min_keep=3, errors='raise'):\n",
    "        \"\"\"\n",
    "        errors: str\n",
    "            If 'warn', we show a warning when min_keep is violated but allow\n",
    "            masking to take place.\n",
    "        \"\"\"\n",
    "        if listlike(text):\n",
    "            return [self._preprocess(row, min_keep, errors) for row in text]\n",
    "        \n",
    "        tokens = text.split()\n",
    "        if len(tokens) < min_keep + 1:\n",
    "            msg = (f'Text \"{text[:25]}...\" is too short to mask while '\n",
    "                   f'enforcing min_keep={min_keep}.')\n",
    "            if errors == 'warn':\n",
    "                warnings.warn(msg)\n",
    "            else:\n",
    "                raise ValueError(msg)\n",
    "        \n",
    "        idx = np.random.choice(range(len(tokens)))\n",
    "        return ' '.join(self.MASK if i == idx else t \n",
    "                        for i, t in enumerate(tokens))\n",
    "    \n",
    "    def __call__(self, text, n=None, n_mask=1, min_keep=3, return_all=False, \n",
    "                 errors:('warn', 'raise')='raise', \n",
    "                 strategy:('random', 'best')='best'):\n",
    "        \"\"\"\n",
    "        n: int or None\n",
    "            If -1, return all generated examples for the given mask count.\n",
    "            This can become very large when n_mask is large. Recall pipeline\n",
    "            can only fill a single mask at a time. e.g. if self.max_n is\n",
    "            3, n=-1, and n_mask is 4, we first mask once and generate 3\n",
    "            samples. Then we mask each of those 3 and generate a total of 9 \n",
    "            samples, then 27, then finally 81 which is what will be returned.\n",
    "            The intermediate samples can be returned with `return_all=True`.\n",
    "        \"\"\"\n",
    "        # Make sure we generate adequate number of sequences. Model topk must\n",
    "        # be >= our desired n.\n",
    "        n = n or self.n\n",
    "        if n > self.max_n:\n",
    "            self.max_n = n\n",
    "            \n",
    "        # Each item will be a list of strings. Each string in res[i]\n",
    "        # will have i words changed. If text is a sequence of strings, we must\n",
    "        # handle each one separately because each is passed through pipeline\n",
    "        # repeatedly.\n",
    "        if listlike(text):\n",
    "            return [self(row, n, n_mask, min_keep, return_all, errors) \n",
    "                    for row in text]\n",
    "\n",
    "        res = [[text]]\n",
    "        for i in range(n_mask):\n",
    "            seqs = self.pipe(self._preprocess(res[-1], min_keep=min_keep,\n",
    "                                              errors=errors))\n",
    "            # Transformers returns either list of dicts or list of list of \n",
    "            # dicts depending on whether input list has 1 item or multiple.\n",
    "            if isinstance(seqs[0], list): \n",
    "                seqs = [seq for group in seqs for seq in group]\n",
    "            text = [seq['sequence'].replace('<s>', '').replace('</s>', '') \n",
    "                    for seq in seqs]\n",
    "            \n",
    "            # Keep all generated samples when n is -1.\n",
    "            if n != -1:\n",
    "                if strategy == 'random':\n",
    "                    text = np.random.choice(text, n, replace=False)\n",
    "                elif strategy == 'best':\n",
    "                    text = text[:n]\n",
    "            res.append(text)\n",
    "        if not return_all: res = res[n_mask]\n",
    "        return res\n",
    "    \n",
    "    @property\n",
    "    def max_n(self):\n",
    "        return self.pipe.topk\n",
    "    \n",
    "    @max_n.setter\n",
    "    def max_n(self, max_n):\n",
    "        if not isinstance(max_n, int):\n",
    "            raise TypeError('max_n must be an integer.')\n",
    "        if max_n < self.n:\n",
    "            raise ValueError(f'max_n must be >= self.n (currently {self.n}.')\n",
    "        self.pipe.topk = max_n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:10:13.061236Z",
     "start_time": "2020-11-09T05:10:13.055858Z"
    }
   },
   "outputs": [],
   "source": [
    "t = 'I went to the store today to buy eggs.'\n",
    "ts = [t, 'The bird swooped down onto the picnic table and squawked loudly.']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:10:13.552253Z",
     "start_time": "2020-11-09T05:10:13.547654Z"
    }
   },
   "outputs": [],
   "source": [
    "# m_pipe = pipeline('fill-mask')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:10:14.113194Z",
     "start_time": "2020-11-09T05:10:14.103013Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FillMaskTransform(pipe=<transformers.pipelines.FillMaskPipeline object at 0x1b45980438>, n=1)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_tfm = FillMaskTransform(m_pipe)\n",
    "m_tfm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:10:14.821987Z",
     "start_time": "2020-11-09T05:10:14.807204Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'I went to the store today <mask> buy eggs.'"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_tfm._preprocess(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:10:15.416610Z",
     "start_time": "2020-11-09T05:10:15.409129Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I went to <mask> store today to buy eggs.',\n",
       " 'The bird swooped <mask> onto the picnic table and squawked loudly.']"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_tfm._preprocess(ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:10:17.010948Z",
     "start_time": "2020-11-09T05:10:16.926858Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I went to the store today to buy eggs.',\n",
       " 'I went to the store today and buy eggs.']"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_tfm(t, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:10:20.375227Z",
     "start_time": "2020-11-09T05:10:20.281806Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['I went to the store today to buy eggs.'],\n",
       " ['I went to the store today to buy pizza',\n",
       "  'I went to the store today to buy...',\n",
       "  'I went to the store today to buy groceries']]"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_tfm(t, 3, return_all=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:10:20.780891Z",
     "start_time": "2020-11-09T05:10:20.523177Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I went to grocery store today to buy eggs.',\n",
       " 'I went to the store today to buy eggs.',\n",
       " 'I went to my store today to buy eggs.',\n",
       " 'I went to a store today to buy eggs.']"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_tfm(t, 4, n_mask=2, strategy='best')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:10:21.314970Z",
     "start_time": "2020-11-09T05:10:21.097975Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['I went to the store wanting to buy eggs.',\n",
       "       'I went to the store today to buy eggs.',\n",
       "       'I went through the store today to buy eggs.',\n",
       "       'I went around the store today to buy eggs.'], dtype='<U46')"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_tfm(t, 4, n_mask=2, strategy='random')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:10:21.874216Z",
     "start_time": "2020-11-09T05:10:21.696451Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['I went to the store today to buy eggs.'],\n",
       " ['I went to the supermarket today to buy eggs.',\n",
       "  'I went to the store today to buy eggs.',\n",
       "  'I went to the grocery today to buy eggs.'],\n",
       " ['I went to the supermarket today to buy eggs.',\n",
       "  'I went to the supermarket today to purchase eggs.',\n",
       "  'I went to the supermarket today to get eggs.']]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_tfm(t, 3, n_mask=2, return_all=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:10:22.449357Z",
     "start_time": "2020-11-09T05:10:22.321368Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['I went to the store today to buy eggs.'],\n",
       " ['I went to grocery store today to buy eggs.']]"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_tfm(t, 1, return_all=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:10:23.185513Z",
     "start_time": "2020-11-09T05:10:23.024025Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['I went to the store today to buy pizza'],\n",
       " ['The bird swooped down onto the picnic table and squawked.']]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_tfm(ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:10:23.916862Z",
     "start_time": "2020-11-09T05:10:23.732789Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['I went to the store today to buy pizza',\n",
       "  'I went to the store today to buy...'],\n",
       " ['The bird swooped down onto a picnic table and squawked loudly.',\n",
       "  'The bird swooped down onto the picnic table and squawked loudly.']]"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_tfm(ts, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:10:24.859055Z",
     "start_time": "2020-11-09T05:10:24.575367Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['I went to the store today to buy pizza'],\n",
       " ['The bird swooped down onto the picnic table and sang loudly.']]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_tfm(ts, n=None, n_mask=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:10:25.618092Z",
     "start_time": "2020-11-09T05:10:25.279568Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[['I went to the store today to buy eggs.'],\n",
       "  ['I went to the store today to buy pizza'],\n",
       "  ['I went to the store today to buy pizza']],\n",
       " [['The bird swooped down onto the picnic table and squawked loudly.'],\n",
       "  ['The bird swooped down onto the picnic table and squawked loudly.'],\n",
       "  ['The birds swooped down onto the picnic table and squawked loudly.']]]"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_tfm(ts, n=None, n_mask=2, return_all=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:10:55.512934Z",
     "start_time": "2020-11-09T05:10:55.410422Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hmamin/anaconda3/lib/python3.7/site-packages/ipykernel/ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['I went to the store today to buy eggs.',\n",
       " 'I headed to the store today to buy eggs.']"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_tfm = FillMaskTransform(m_pipe, 2, 5)\n",
    "m_tfm(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:11:06.505766Z",
     "start_time": "2020-11-09T05:11:06.296514Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['I went to the store today to buy eggs.',\n",
       "  'I went to the store today to purchase eggs.'],\n",
       " ['The bird swooped down onto the picnic table and squawked loudly.',\n",
       "  'The bird swooped down from the picnic table and squawked loudly.']]"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_tfm(ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T05:11:17.474807Z",
     "start_time": "2020-11-09T05:11:17.265154Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['I went to grocery store today to buy eggs.',\n",
       "  'I went to the store today to buy eggs.',\n",
       "  'I went to my store today to buy eggs.',\n",
       "  'I went to a store today to buy eggs.',\n",
       "  'I went to another store today to buy eggs.',\n",
       "  'I went to our store today to buy eggs.'],\n",
       " ['The bird swooped down onto the picnic table and sang loudly.',\n",
       "  'The bird swooped down onto the picnic table and cheered loudly.',\n",
       "  'The bird swooped down onto the picnic table and cried loudly.',\n",
       "  'The bird swooped down onto the picnic table and complained loudly.',\n",
       "  'The bird swooped down onto the picnic table and roared loudly.',\n",
       "  'The bird swooped down onto the picnic table and spoke loudly.']]"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_tfm(ts, n=6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ParaphraseTransform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T04:31:10.418340Z",
     "start_time": "2020-11-09T04:31:10.408232Z"
    }
   },
   "outputs": [],
   "source": [
    "@auto_repr\n",
    "class ParaphraseTransform:\n",
    "    \"\"\"Not sure how useful this will really be but this basically just \n",
    "    wraps ParaphrasePipeline to share a more similar interface with the other\n",
    "    NLP transforms.\n",
    "    \"\"\"\n",
    "    \n",
    "    name = 'tuner007/pegasus_paraphrase'\n",
    "\n",
    "    def __init__(self, pipe=None, n=1):\n",
    "        # We let user pass in pipe at least for now since re-instantiating the\n",
    "        # class can be very slow during development. Need to consider whether\n",
    "        # I want this behavior to remain.\n",
    "        self.pipe = pipe or ParaphrasePipeline(self.name)\n",
    "        self.n = n\n",
    "            \n",
    "        assert type(self.pipe).__name__ == 'ParaphrasePipeline'\n",
    "        \n",
    "    def _preprocess(self, text):\n",
    "        \"\"\"Does nothing (just want shared interface with other transforms).\"\"\"\n",
    "        return text\n",
    "    \n",
    "    def __call__(self, text, n=None, **kwargs):\n",
    "        return self.pipe(text, n=n or self.n, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-06T06:23:14.099343Z",
     "start_time": "2020-11-06T06:23:14.091373Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ParaphraseTransform(pipe=<__main__.ParaphrasePipeline object at 0x1a47cda198>)"
      ]
     },
     "execution_count": 304,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_tfm = ParaphraseTransform(p_pipe)\n",
    "p_tfm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-06T06:23:24.441803Z",
     "start_time": "2020-11-06T06:23:17.058658Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['The beach is crowded.',\n",
       " 'There is a lot of people on the beach.',\n",
       " 'There is a lot of people at the beach.']"
      ]
     },
     "execution_count": 305,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_tfm('The beach is loud and crowded today.', n=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-05T06:22:36.982196Z",
     "start_time": "2020-11-05T06:22:22.156295Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['I went to buy eggs.'], ['The bird swooped down onto the picnic table.']]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_tfm(ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-05T06:22:44.555755Z",
     "start_time": "2020-11-05T06:22:38.723246Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['I went to buy eggs.', 'I went to the store to buy eggs.'],\n",
       " ['The bird swooped down onto the picnic table.',\n",
       "  'The bird swooped down on the picnic table.']]"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_tfm(ts, n=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-05T06:48:03.758403Z",
     "start_time": "2020-11-05T06:48:03.754099Z"
    }
   },
   "outputs": [],
   "source": [
    "def listlike(x):\n",
    "    \"\"\"Checks if an object is a list/tuple/set/array etc. Strings and\n",
    "    mappings (e.g. dicts) are not considered list-like.\n",
    "    \"\"\"\n",
    "    return isinstance(x, Iterable) and not isinstance(x, (str, Mapping))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-05T06:47:50.367487Z",
     "start_time": "2020-11-05T06:47:50.357976Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'str'> False\n",
      "<class 'int'> False\n",
      "<class 'list'> True\n",
      "<class 'tuple'> True\n",
      "<class 'dict'> False\n",
      "<class 'set'> True\n",
      "<class 'list'> True\n",
      "<class 'list'> True\n",
      "<class 'tuple'> True\n",
      "<class 'dict'> False\n",
      "<class 'set'> True\n",
      "<class 'numpy.ndarray'> True\n"
     ]
    }
   ],
   "source": [
    "for obj in ('a', 6, [], (), {}, set(), [3, 4], ['a', 'b'], ('a',), {'a': 'b'},\n",
    "            {'a', 'b'}, np.arange(5)):\n",
    "    print(type(obj), listlike(obj))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GenerativeTransform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-09T04:56:46.487061Z",
     "start_time": "2020-11-09T04:56:46.474371Z"
    }
   },
   "outputs": [],
   "source": [
    "@auto_repr\n",
    "class GenerativeTransform:\n",
    "    \n",
    "    name = 'text-generation'\n",
    "    \n",
    "    def __init__(self, pipe=None, n=1):\n",
    "        # Allow user to pass in n here to reduce likelihood of needing to \n",
    "        # create a partial from __call__. Maybe should add other __call__\n",
    "        # kwargs here?\n",
    "        self.pipe = pipe or pipeline(self.name)\n",
    "        self.n = n\n",
    "        \n",
    "        assert type(self.pipe).__name__ == 'TextGenerationPipeline'\n",
    "    \n",
    "    def _preprocess(self, text, drop=None, drop_pct=None, rand_low=None, \n",
    "                    rand_high=None, min_keep=3, return_tuple=False):\n",
    "        \"\"\"Truncate text.\"\"\"\n",
    "        if listlike(text):\n",
    "            return [self._preprocess(row, drop, drop_pct, rand_low, rand_high,\n",
    "                                     min_keep, return_tuple) for row in text]\n",
    "        \n",
    "        tokens = text.split()\n",
    "        if len(tokens) <= min_keep:\n",
    "            n_drop = 0\n",
    "        else:\n",
    "            # Default is to truncate the last 20% of the sequence.\n",
    "            if drop:\n",
    "                n_drop = drop\n",
    "            elif drop_pct:\n",
    "                n_drop = int(drop_pct * len(tokens))\n",
    "            elif rand_low is not None and rand_high is not None:\n",
    "                n_drop = np.random.randint(rand_low, rand_high)\n",
    "            else:\n",
    "                n_drop = int(np.ceil(.2 * len(tokens)))\n",
    "            n_drop = np.clip(n_drop, 0, len(tokens) - min_keep)\n",
    "            tokens = tokens[:-n_drop]\n",
    "        truncated = ' '.join(tokens)\n",
    "        return (truncated, n_drop) if return_tuple else truncated\n",
    "    \n",
    "    def __call__(self, text, n=None, min_length=2, max_length=7, \n",
    "                 **generate_kwargs):\n",
    "        n = n or self.n\n",
    "        if listlike(text):\n",
    "            return [self(row, n, min_length, max_length, **generate_kwargs) \n",
    "                    for row in text]\n",
    "    \n",
    "        # `generate` counts current length as part of min_length. \n",
    "        text = self._preprocess(text)\n",
    "        n_curr = len(self.pipe.tokenizer.tokenize(text))\n",
    "        res = self.pipe(text, min_length=n_curr + min_length,\n",
    "                        max_length=n_curr + max_length,\n",
    "                        num_return_sequences=n, **generate_kwargs)\n",
    "        return [row['generated_text'] for row in res]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 535,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T06:27:46.589860Z",
     "start_time": "2020-11-07T06:27:46.583082Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GenerativeTransform(pipe=<transformers.pipelines.TextGenerationPipeline object at 0x1d081425c0>, n=1)"
      ]
     },
     "execution_count": 535,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# g_pipe = pipeline('text-generation')\n",
    "g_tfm = GenerativeTransform(g_pipe)\n",
    "g_tfm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 536,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T06:27:47.212987Z",
     "start_time": "2020-11-07T06:27:47.205670Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'I went to the store today to'"
      ]
     },
     "execution_count": 536,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g_tfm._preprocess(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 537,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T06:27:47.953718Z",
     "start_time": "2020-11-07T06:27:47.947394Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('I went to the store today to', 2),\n",
       " ('The bird swooped down onto the picnic table', 3)]"
      ]
     },
     "execution_count": 537,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g_tfm._preprocess(ts, return_tuple=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 538,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T06:27:49.224676Z",
     "start_time": "2020-11-07T06:27:48.825361Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['I went to the store today to buy a shirt, but I felt']"
      ]
     },
     "execution_count": 538,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g_tfm(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 539,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T06:27:50.906948Z",
     "start_time": "2020-11-07T06:27:50.378713Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['I went to the store today to buy some clothes. The first thing',\n",
       " 'I went to the store today to see where it went and I asked']"
      ]
     },
     "execution_count": 539,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g_tfm(t, n=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 540,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T06:27:52.192960Z",
     "start_time": "2020-11-07T06:27:51.445339Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[['I went to the store today to check out everything. I wanted to'],\n",
       " ['The bird swooped down onto the picnic table so much that two of them noticed']]"
      ]
     },
     "execution_count": 540,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g_tfm(ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 541,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T06:27:55.987724Z",
     "start_time": "2020-11-07T06:27:54.662471Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[['I went to the store today to pick up my kids, got some',\n",
       "  \"I went to the store today to check out his new book, '\"],\n",
       " ['The bird swooped down onto the picnic table, then proceeded to lay itself down',\n",
       "  'The bird swooped down onto the picnic table, where he and the other dogs']]"
      ]
     },
     "execution_count": 541,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g_tfm(ts, n=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-06T06:18:58.435710Z",
     "start_time": "2020-11-06T06:18:58.422790Z"
    }
   },
   "outputs": [],
   "source": [
    "# I'm now thinking this isn't that useful. It also loses our kwarg names \n",
    "# unless I pull some tricks altering signatures. Don't think this offers \n",
    "# enough benefit to justify that.\n",
    "class TransformerTransform:\n",
    "    \n",
    "    def __init__(self, mode, pipe=None):\n",
    "        self.mode = mode\n",
    "        self._transformer = self._get_transformer(pipe)\n",
    "        \n",
    "    def _preprocess(self, text, **kwargs):\n",
    "        return self._transformer._preprocess(text, **kwargs)\n",
    "    \n",
    "    def __call__(self, text, **kwargs):\n",
    "        return self._transformer(text, **kwargs)\n",
    "    \n",
    "    def _get_transformer(self, pipe):\n",
    "        if self.mode == 'mask':\n",
    "            return FillMaskTransform(pipe)\n",
    "        elif self.mode == 'generate':\n",
    "            return GenerativeTransform(pipe)\n",
    "        elif self.mode == 'paraphrase':\n",
    "            return ParaphraseTransform(pipe)\n",
    "        else:\n",
    "            raise ValueError('mode must be in (mask, generate, paraphrase).')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-06T06:19:09.524677Z",
     "start_time": "2020-11-06T06:19:09.520369Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hmamin/anaconda3/lib/python3.7/site-packages/ipykernel/ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    }
   ],
   "source": [
    "mt = TransformerTransform('mask', m_pipe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-06T06:19:10.893643Z",
     "start_time": "2020-11-06T06:19:10.875581Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'I went to <mask> store today to buy eggs.'"
      ]
     },
     "execution_count": 288,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mt._preprocess(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-06T06:19:11.427649Z",
     "start_time": "2020-11-06T06:19:11.421747Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I <mask> to the store today to <mask> eggs.',\n",
       " 'The <mask> swooped down <mask> the picnic table and squawked loudly.']"
      ]
     },
     "execution_count": 289,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mt._preprocess(ts, n=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-06T06:19:12.969444Z",
     "start_time": "2020-11-06T06:19:11.982314Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I went to the store today to buy eggs.',\n",
       " 'I went into the store today to buy eggs.',\n",
       " 'I went through the store today to buy eggs.',\n",
       " 'I went around the store today to buy eggs.',\n",
       " 'I went in the store today to buy eggs.']"
      ]
     },
     "execution_count": 290,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mt(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-06T06:19:14.262802Z",
     "start_time": "2020-11-06T06:19:13.982937Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['I went to the store today to buy eggs.',\n",
       "  'I went into the store today to buy eggs.',\n",
       "  'I went through the store today to buy eggs.',\n",
       "  'I went around the store today to buy eggs.',\n",
       "  'I went in the store today to buy eggs.'],\n",
       " ['The bird swooped down onto the picnic table and squawked loudly.',\n",
       "  'The bird swooped down onto the picnic table, squawked loudly.',\n",
       "  'The bird swooped down onto the picnic table then squawked loudly.',\n",
       "  'The bird swooped down onto the picnic table who squawked loudly.',\n",
       "  'The bird swooped down onto the picnic table but squawked loudly.']]"
      ]
     },
     "execution_count": 291,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mt(ts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BackTranslationTransform\n",
    "\n",
    "Realized Huggingface provides no models for translating back to English."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-12T06:31:23.847974Z",
     "start_time": "2020-11-12T06:31:23.839805Z"
    }
   },
   "outputs": [],
   "source": [
    "class BackTranslationTransform:\n",
    "    \n",
    "    def __init__(self, pipe=None, n=1, from_lang='en', to_lang='fr',\n",
    "                 pipe_rev=None):\n",
    "        self.name = f'translation_{from_lang}_to_{to_lang}'\n",
    "        self.name_rev = f'translation_{to_lang}_to_{from_lang}'\n",
    "        self.pipe = pipe or pipeline(self.name)\n",
    "        self.pipe_rev = pipe_rev or pipeline(self.name_rev)\n",
    "        \n",
    "    def _preprocess(self, text):\n",
    "        return text\n",
    "    \n",
    "    def __call__(self, text):\n",
    "        trans = self.pipe(text)\n",
    "        print(trans)\n",
    "        return self.pipe_rev(trans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-13T05:32:20.109448Z",
     "start_time": "2020-11-13T05:32:20.045078Z"
    }
   },
   "outputs": [],
   "source": [
    "# b_tfm = BackTranslationTransform()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ParaphraseTransform v2\n",
    "\n",
    "Found there actually is a built-in version of the pipeline that I think will work. Try it out."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-13T05:52:03.080191Z",
     "start_time": "2020-11-13T05:52:03.071507Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hmamin/anaconda3/lib/python3.7/site-packages/ipykernel/ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    }
   ],
   "source": [
    "from transformers import Text2TextGenerationPipeline, PreTrainedModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-13T05:35:49.768855Z",
     "start_time": "2020-11-13T05:35:49.743716Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Text2TextGenerationPipeline'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Text2TextGenerationPipeline.__name__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-14T06:30:13.386386Z",
     "start_time": "2020-11-14T06:30:13.366908Z"
    }
   },
   "outputs": [],
   "source": [
    "# export\n",
    "@auto_repr\n",
    "class ParaphraseTransform:\n",
    "    \"\"\"Text transform that paraphrases input text as a method of data\n",
    "    augmentation. This is rather slow so it's recommended to precompute \n",
    "    samples and save them, but you could generate samples on the fly if \n",
    "    desired. One further downside of that approach is you'll have a huge\n",
    "    paraphrasing model on the GPU while (presumably) training another model.\n",
    "    \n",
    "    Note: This just wraps ParaphrasePipeline to share a more similar interface \n",
    "    with the other NLP transforms. Since no preprocessing is required, it's\n",
    "    basically identical to ParaphrasePipeline.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, pipe=None, n=1, name='tuner007/pegasus_paraphrase'):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        pipe: ParaphrasePipeline or None\n",
    "        n: int\n",
    "            Default number of samples to generate. You can override this in\n",
    "            __call__.\n",
    "        \"\"\"\n",
    "        if pipe:\n",
    "            self.pipe = pipe\n",
    "            self.name = pipe.model.config._name_or_path\n",
    "        else:\n",
    "            self.pipe = Text2TextGenerationPipeline(\n",
    "                PegasusForConditionalGeneration.from_pretrained(name),\n",
    "                PegasusTokenizer.from_pretrained(name),\n",
    "                device=0 if torch.cuda.is_available() else -1\n",
    "            )\n",
    "            self.name = name\n",
    "        self.n = n\n",
    "            \n",
    "        assert type(self.pipe).__name__ == 'Text2TextGenerationPipeline'\n",
    "        if 'cuda' not in str(self.pipe.device) and torch.cuda.is_available():\n",
    "            warnings.warn('The pipeline passed in is not using cuda. '\n",
    "                          'Did you mean to use the available GPU?')\n",
    "                \n",
    "    def _preprocess(self, text):\n",
    "        \"\"\"Does nothing (just want shared interface with other transforms).\"\"\"\n",
    "        return text\n",
    "    \n",
    "    @add_docstring(PreTrainedModel.generate)\n",
    "    def __call__(self, text, n=None, **kwargs):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        text: str or Iterable[str]\n",
    "            Raw text to transform.\n",
    "        n: int or None\n",
    "            If None, use the default self.n.\n",
    "        kwargs: any\n",
    "            Additional kwargs are passed to the model's text generation \n",
    "            method. Its docstring is included below for convenience.\n",
    "            \n",
    "        Returns\n",
    "        -------\n",
    "        list: either a list of n strings (if input text is a single string) \n",
    "        or a list of lists, each of length n.\n",
    "        \"\"\"\n",
    "        n = n or self.n\n",
    "        rows = [row['generated_text'] for row in \n",
    "                self.pipe(text, num_return_sequences=n, **kwargs)]\n",
    "        if listlike(text): \n",
    "            rows = [rows[i*n:(i+1)*n] for i in range(len(text))]\n",
    "        return rows"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test on data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T05:19:19.026922Z",
     "start_time": "2020-11-07T05:19:18.952229Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9887"
      ]
     },
     "execution_count": 351,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines = [line.strip() for line \n",
    "         in '\\n'.join(load(f'/Users/hmamin/data/bbc/tech/{n:03}.txt') \n",
    "         for n in range(1, 402)).split('.') if line]\n",
    "len(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 544,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T06:29:33.595508Z",
     "start_time": "2020-11-07T06:29:27.085884Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hmamin/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:28: UserWarning: Text \"And vice versa...\" is too short to mask while enforcing min_keep=3.\n"
     ]
    }
   ],
   "source": [
    "m_res = m_tfm(lines[:100], errors='warn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 549,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T06:30:37.216585Z",
     "start_time": "2020-11-07T06:30:37.207024Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500"
      ]
     },
     "execution_count": 549,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(flatten(m_res))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 545,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T06:29:33.606452Z",
     "start_time": "2020-11-07T06:29:33.597326Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[\"Ink helps drive democracy in Asia The Kyrgyz Republic, a small, mountainous state of the former Soviet republic, is using invisible ink and fingerprint readers in the country's elections as part of a drive to prevent multiple voting\",\n",
       "  \"Ink helps drive democracy in Asia The Kyrgyz Republic, a small, mountainous state of the former Soviet republic, is using invisible ink and electronic readers in the country's elections as part of a drive to prevent multiple voting\",\n",
       "  \"Ink helps drive democracy in Asia The Kyrgyz Republic, a small, mountainous state of the former Soviet republic, is using invisible ink and pen readers in the country's elections as part of a drive to prevent multiple voting\",\n",
       "  \"Ink helps drive democracy in Asia The Kyrgyz Republic, a small, mountainous state of the former Soviet republic, is using invisible ink and paper readers in the country's elections as part of a drive to prevent multiple voting\",\n",
       "  \"Ink helps drive democracy in Asia The Kyrgyz Republic, a small, mountainous state of the former Soviet republic, is using invisible ink and ink readers in the country's elections as part of a drive to prevent multiple voting\"],\n",
       " ['This new technology is causing both worries and guarded optimism among different segments of the population',\n",
       "  'This new technology is causing both worries and guarded optimism among different sectors of the population',\n",
       "  'This new technology is causing both worries and guarded optimism among different sections of the population',\n",
       "  'This new technology is causing both worries and guarded optimism among different parts of the population',\n",
       "  'This new technology is causing both worries and guarded optimism among different quarters of the population']]"
      ]
     },
     "execution_count": 545,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_res[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 543,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T06:29:27.034496Z",
     "start_time": "2020-11-07T06:28:50.493856Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n",
      "Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n"
     ]
    }
   ],
   "source": [
    "g_res = g_tfm(lines[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 548,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T06:30:29.950546Z",
     "start_time": "2020-11-07T06:30:29.942428Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 548,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(flatten(g_res))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 550,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T06:30:48.864764Z",
     "start_time": "2020-11-07T06:30:48.857287Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"Ink helps drive democracy in Asia\\n\\nThe Kyrgyz Republic, a small, mountainous state of the former Soviet republic, is using invisible ink and ultraviolet readers in the country's elections as part of a drive to prevent multiple voting\",\n",
       " 'This new technology is causing both worries and guarded optimism among different sectors of the population']"
      ]
     },
     "execution_count": 550,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 546,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T06:29:33.674019Z",
     "start_time": "2020-11-07T06:29:33.655130Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[\"Ink helps drive democracy in Asia The Kyrgyz Republic, a small, mountainous state of the former Soviet republic, is using invisible ink and ultraviolet readers in the country's elections as an early warning to the rest of\"],\n",
       " ['This new technology is causing both worries and guarded optimism among different groups in Europe about cybersecurity,\" said']]"
      ]
     },
     "execution_count": 546,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g_res[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 558,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T07:02:05.133625Z",
     "start_time": "2020-11-07T06:59:39.088199Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "p_res = p_tfm(lines[:25])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 559,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T07:02:08.645487Z",
     "start_time": "2020-11-07T07:02:08.605932Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25"
      ]
     },
     "execution_count": 559,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(flatten(p_res))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 560,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T07:02:09.447401Z",
     "start_time": "2020-11-07T07:02:09.439494Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"Ink helps drive democracy in Asia\\n\\nThe Kyrgyz Republic, a small, mountainous state of the former Soviet republic, is using invisible ink and ultraviolet readers in the country's elections as part of a drive to prevent multiple voting\",\n",
       " 'This new technology is causing both worries and guarded optimism among different sectors of the population',\n",
       " 'In an effort to live up to its reputation in the 1990s as \"an island of democracy\", the Kyrgyz President, Askar Akaev, pushed through the law requiring the use of ink during the upcoming Parliamentary and Presidential elections',\n",
       " 'The US government agreed to fund all expenses associated with this decision',\n",
       " 'The Kyrgyz Republic is seen by many experts as backsliding from the high point it reached in the mid-1990s with a hastily pushed through referendum in 2003, reducing the legislative branch to one chamber with 75 deputies']"
      ]
     },
     "execution_count": 560,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 561,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-07T07:02:11.119963Z",
     "start_time": "2020-11-07T07:02:11.113942Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[\"The Kyrgyz Republic, a small, mountainous state of the former Soviet republic, is using invisible ink and ultraviolet readers in the country's elections as part of a drive to prevent multiple voting.\"],\n",
       " ['There are both worries and guarded optimism among different sectors of the population.'],\n",
       " ['The law requiring the use of ink during the upcoming Parliamentary and Presidential elections was pushed through by the President in an effort to live up to its reputation as an island of democracy.'],\n",
       " ['All expenses associated with this decision will be funded by the US government.'],\n",
       " ['The high point of the Kyrgyz Republic was in the mid 1990s when it had a legislative branch with 75 deputies and was seen by many experts as backsliding.']]"
      ]
     },
     "execution_count": 561,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_res[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: with GPU, paraphrase transform takes ~9 seconds to generate variations of 100 input sentences. (Different inputs than used here but I don't think the length differs dramatically.)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RandomPipeline\n",
    "\n",
    "Considering the idea of making a pipeline that accepts multiple callables and applies each one in order with a different (or same) probability P. We could construct this with RandomTransform manually but it seems like if everything or most things are random transforms, we might as well obscure this from the user."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T06:20:59.973237Z",
     "start_time": "2020-11-18T06:20:59.906667Z"
    }
   },
   "source": [
    "Trying to brainstorm what desired interface might look like.\n",
    "\n",
    "```\n",
    "pipeline = RandomPipeline(fill_mask, paraphrase, back_translate, p=.5)\n",
    "\n",
    "pipeline = RandomPipeline(fill_mask, paraphrase, back_translate, \n",
    "                          p=[.5, 1., .5])\n",
    "\n",
    "pipeline = RandomPipeline.from_dict(\n",
    "    {fill_mask: .5, paraphrase: 1., back_translate: .25}\n",
    ")\n",
    "```\n",
    "\n",
    "Leaning towards no inverse_transform, or at least making it optional. Some\n",
    "transforms, like ParaphraseTransform, aren't really meant to be reversible.\n",
    "I suppose I could keep a mapping between original and transformed items but\n",
    "that might become infeasible as we process more data.\n",
    "\n",
    "Could also use pipeline.transform(text)\n",
    "\n",
    "```augmented = pipeline(text)\n",
    "text = pipeline.inverse_transform(text)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:19:36.356633Z",
     "start_time": "2020-11-19T04:19:36.350002Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hmamin/anaconda3/lib/python3.7/site-packages/ipykernel/ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    }
   ],
   "source": [
    "from numbers import Real\n",
    "from incendio.data import RandomTransform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:57:52.184102Z",
     "start_time": "2020-11-19T04:57:52.165145Z"
    }
   },
   "outputs": [],
   "source": [
    "def tolist(x, length_like=None, length=None, \n",
    "           error_message='x length does not match desired length.'):\n",
    "    \"\"\"Helper to let a function accept a single value or a list of values for\n",
    "    a certain parameter. \n",
    "    \n",
    "    WARNING: if x is a primitive and you specify a length (either via \n",
    "    `length_like` or `length`, the resulting list will contain multiple \n",
    "    references to the same item). This is mostly intended for use on lists of\n",
    "    floats or ints so I don't think it's a problem, but keep this in mind when\n",
    "    considering using this on mutable objects.\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    x: Iterable\n",
    "        Usually an object that could either be a list/tuple or a primitive,\n",
    "        depending on what user passed in.\n",
    "    strict: bool\n",
    "        If True, returned value will always be a list. If False, we allow\n",
    "        tuples/sets/etc. to retain their initial type.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    Iterable: list if strict is True or if x is a primitive. If strict is\n",
    "    False and x is already a tuple/set/something similar, its type will be\n",
    "    retained.\n",
    "\n",
    "    Examples\n",
    "    --------\n",
    "    def train(lrs):\n",
    "        lrs = tolist(lrs)\n",
    "        ...\n",
    "\n",
    "    >>> train(3e-3)\n",
    "    >>> train([3e-4, 3e-3])\n",
    "    \"\"\"\n",
    "    if length_like is not None: length = len(length_like)\n",
    "        \n",
    "    # Case 1. List-like x\n",
    "    if listlike(x):\n",
    "        if length: \n",
    "            assert len(x) == length, error_message\n",
    "        return list(x)\n",
    "    \n",
    "    # Case 2. Dict-like x\n",
    "    if isinstance(x, Mapping):\n",
    "        raise ValueError('x must not be a mapping. It should probably be a '\n",
    "                         'primitive (str, int, etc.) or a list-like object '\n",
    "                         '(tuple, list, set).')\n",
    "        \n",
    "    # Case 3. Primitive x\n",
    "    return [x] * (length or 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:57:52.338906Z",
     "start_time": "2020-11-19T04:57:52.332592Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = .5\n",
    "t = [1, 2, 3]\n",
    "# p = p if listlike(p) and len(p) == len(t) else [p] * len(t)\n",
    "p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:57:52.528044Z",
     "start_time": "2020-11-19T04:57:52.519984Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.5]"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tolist(.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:57:52.636776Z",
     "start_time": "2020-11-19T04:57:52.632415Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.5, 0.5]"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tolist(.5, length=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:57:53.164274Z",
     "start_time": "2020-11-19T04:57:53.158535Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.5, 0.5, 0.5]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tolist(.5, length_like=t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:57:53.718019Z",
     "start_time": "2020-11-19T04:57:53.712221Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[3, 4, -1]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tolist({3, 4, -1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:57:54.441308Z",
     "start_time": "2020-11-19T04:57:54.436874Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "As expected, got AssertionError(x length does not match desired length.).\n"
     ]
    }
   ],
   "source": [
    "with assert_raises(AssertionError):\n",
    "    tolist({3, 4}, length_like=t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:57:55.720709Z",
     "start_time": "2020-11-19T04:57:55.713755Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[3, 4, -1]"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tolist({3, 4, -1}, length_like=t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:57:55.944299Z",
     "start_time": "2020-11-19T04:57:55.939504Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "As expected, got ValueError(x must not be a mapping. It should probably be a primitive (str, int, etc.) or a list-like object (tuple, list, set).).\n"
     ]
    }
   ],
   "source": [
    "with assert_raises(ValueError):\n",
    "    tolist({3: 1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T05:15:14.970823Z",
     "start_time": "2020-11-19T05:15:14.962135Z"
    }
   },
   "outputs": [],
   "source": [
    "class RandomPipeline(BasicPipeline):\n",
    "    \"\"\"Create a pipeline of callables that are applied in sequence, each with\n",
    "    some random probability p (this can be the same or different for each \n",
    "    step). This is useful for on-the-fly data augmentation (think in the\n",
    "    __getitem__ method of a torch Dataset).\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, *transforms, p=.5):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        transforms: callable\n",
    "            Functions or callable classes that accept a single argument (use\n",
    "            functools.partial if necessary). They will be applied in the order\n",
    "            you pass them in.\n",
    "        p: float or Iterable[float]\n",
    "            Probability that each transform will be applied. If a single \n",
    "            float, each transform will have the same probability. If a list,\n",
    "            its length msut match the number of transforms passed in: p[0] \n",
    "            will be assigned to transforms[0], p[1] to transforms[1], and so \n",
    "            on.\n",
    "        \"\"\"\n",
    "        p = tolist(p, transforms, error_message='p must be a float or a list '\n",
    "                   'with one float for each transform.')\n",
    "        if any(n <= 0 or n > 1 for n in p):\n",
    "            raise ValueError('p must be in range (0, 1]. I.E. you can choose '\n",
    "                             'to always apply a transform, but if you never '\n",
    "                             'want to apply it there\\'s no need to include '\n",
    "                             'it in the pipeline.')\n",
    "        \n",
    "        super().__init__(*[RandomTransform(t, p_) \n",
    "                           for t, p_ in zip(transforms, p)])\n",
    "        \n",
    "    @classmethod\n",
    "    def from_dict(cls, t2p):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        t2p: dict[callable, float]\n",
    "            Maps transform to its corresponding probability.\n",
    "            \n",
    "        Examples\n",
    "        --------\n",
    "        transforms = {times_3: .33,\n",
    "                      to_string: 1.0,\n",
    "                      dashed_join: .67,\n",
    "                      to_upper: .95}\n",
    "        pipeline = RandomPipeline.from_dict(transforms)\n",
    "        \"\"\"\n",
    "        return cls(*t2p.keys(), p=t2p.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T05:12:17.771386Z",
     "start_time": "2020-11-19T05:12:17.767603Z"
    }
   },
   "outputs": [],
   "source": [
    "def to_upper(t):\n",
    "    return t.upper()\n",
    "\n",
    "def times_3(t):\n",
    "    return t * 3\n",
    "\n",
    "def join(t, sep='---'):\n",
    "    return sep.join(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T05:12:18.232844Z",
     "start_time": "2020-11-19T05:12:18.226752Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomPipeline(\n",
       "\tRandomTransform(to_upper, p=0.5),\n",
       "\tRandomTransform(times_3, p=0.5),\n",
       "\tRandomTransform(join, p=0.5)\n",
       ")"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = 'dog'\n",
    "rp = RandomPipeline(to_upper, times_3, join)\n",
    "rp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T05:12:18.900985Z",
     "start_time": "2020-11-19T05:12:18.894277Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "d---o---g---d---o---g---d---o---g\n",
      "D---O---G---D---O---G---D---O---G\n",
      "D---O---G\n",
      "DOGDOGDOG\n",
      "d---o---g\n",
      "DOGDOGDOG\n",
      "dog\n",
      "dogdogdog\n",
      "d---o---g\n",
      "D---O---G---D---O---G---D---O---G\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    print(rp(t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T05:12:22.809128Z",
     "start_time": "2020-11-19T05:12:22.802255Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "D---O---G\n",
      "dogdogdog\n",
      "d---o---g\n",
      "dogdogdog\n",
      "dogdogdog\n",
      "D---O---G\n",
      "dogdogdog\n",
      "D---O---G---D---O---G---D---O---G\n",
      "d---o---g---d---o---g---d---o---g\n",
      "D---O---G---D---O---G---D---O---G\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    print(rp(t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:59:15.309764Z",
     "start_time": "2020-11-19T04:59:15.304369Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomPipeline(\n",
       "\tRandomTransform(to_upper, p=1.0),\n",
       "\tRandomTransform(times_3, p=0.6),\n",
       "\tRandomTransform(join, p=1)\n",
       ")"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = 'dog'\n",
    "rp = RandomPipeline(to_upper, times_3, join, p=[1., .6, 1])\n",
    "rp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:59:16.019490Z",
     "start_time": "2020-11-19T04:59:16.012504Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "D---O---G---D---O---G---D---O---G\n",
      "D---O---G\n",
      "D---O---G---D---O---G---D---O---G\n",
      "D---O---G---D---O---G---D---O---G\n",
      "D---O---G\n",
      "D---O---G---D---O---G---D---O---G\n",
      "D---O---G---D---O---G---D---O---G\n",
      "D---O---G---D---O---G---D---O---G\n",
      "D---O---G\n",
      "D---O---G\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    print(rp(t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:59:44.549340Z",
     "start_time": "2020-11-19T04:59:44.535027Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "As expected, got ValueError(p must be in range (0, 1]. I.E. you can choose to always apply a transform, but if you never want to apply it there's no need to include it in the pipeline.).\n"
     ]
    }
   ],
   "source": [
    "t = 'dog'\n",
    "with assert_raises(ValueError):\n",
    "    rp = RandomPipeline(join, p=0)\n",
    "    rp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T05:00:25.514140Z",
     "start_time": "2020-11-19T05:00:25.507963Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "As expected, got AssertionError(p must be a float or a list with one float for each transform.).\n"
     ]
    }
   ],
   "source": [
    "with assert_raises(AssertionError):\n",
    "    rp = RandomPipeline(to_upper, times_3, join, p=[.2, 1])\n",
    "    rp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:59:18.046357Z",
     "start_time": "2020-11-19T04:59:18.037502Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomPipeline(\n",
       "\tRandomTransform(times_3, p=0.33),\n",
       "\tRandomTransform(join, p=0.67),\n",
       "\tRandomTransform(to_upper, p=0.95)\n",
       ")"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = 'dog'\n",
    "tfms = {times_3: .33,\n",
    "        join: .67,\n",
    "        to_upper: .95}\n",
    "rp = RandomPipeline.from_dict(tfms)\n",
    "rp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:59:19.517335Z",
     "start_time": "2020-11-19T04:59:19.511509Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[RandomTransform(times_3, p=0.33),\n",
       " RandomTransform(join, p=0.67),\n",
       " RandomTransform(to_upper, p=0.95)]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rp.funcs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-19T04:59:20.498597Z",
     "start_time": "2020-11-19T04:59:20.490553Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DOG\n",
      "D---O---G---D---O---G---D---O---G\n",
      "D---O---G---D---O---G---D---O---G\n",
      "d---o---g\n",
      "D---O---G\n",
      "DOG\n",
      "D---O---G\n",
      "D---O---G---D---O---G---D---O---G\n",
      "D---O---G\n",
      "D---O---G\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    print(rp(t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T05:52:17.587008Z",
     "start_time": "2021-02-10T05:52:17.473446Z"
    }
   },
   "outputs": [],
   "source": [
    "class BacktranslateTransform:\n",
    "\n",
    "    names = ['Helsinki-NLP/opus-mt-en-ROMANCE',\n",
    "             'Helsinki-NLP/opus-mt-ROMANCE-en']\n",
    "    \n",
    "    language_codes = {\n",
    "        'es': 'spanish',\n",
    "        'it': 'italian',\n",
    "        'pt': 'portuguese',\n",
    "        'pt_br': 'portuguese (brazil)',\n",
    "        'ro': 'romanian',\n",
    "        'ca': 'catalan',\n",
    "        'gl': 'galician',\n",
    "        'pt_BR': 'portuguese (brazil?)',\n",
    "        'la': 'latin',\n",
    "        'wa': 'walloon',\n",
    "        'fur': 'friulian (?)',\n",
    "        'oc': 'occitan',\n",
    "        'fr_CA': 'french (canada)',\n",
    "        'sc': 'sardianian',\n",
    "        'es_ES': 'spanish',\n",
    "        'es_MX': 'spanish (mexico)',\n",
    "        'es_AR': 'spanish (argentina)',\n",
    "        'es_PR': 'spanish (puerto rico)',\n",
    "        'es_UY': 'spanish (uruguay)',\n",
    "        'es_CL': 'spanish (chile)',\n",
    "        'es_CO': 'spanish (colombia)',\n",
    "        'es_CR': 'spanish (croatia)',\n",
    "        'es_GT': 'spanish (guatemala)',\n",
    "        'es_HN': 'spanish (honduras)',\n",
    "        'es_NI': 'spanish (nicaragua)',\n",
    "        'es_PA': 'spanish (panama)',\n",
    "        'es_PE': 'spanish (peru)',\n",
    "        'es_VE': 'spanish (venezuela)',\n",
    "        'es_DO': 'spanish (dominican republic)',\n",
    "        'es_EC': 'spanish (ecuador)',\n",
    "        'es_SV': 'spanish (el salvador)',\n",
    "        'an': 'aragonese',\n",
    "        'pt_PT': 'portuguese (portugal)',\n",
    "        'frp': 'franco provencal',\n",
    "        'lad': 'ladino',\n",
    "        'vec': 'venetian',\n",
    "        'fr_FR': 'france (france)',\n",
    "        'co': 'corsican',\n",
    "        'it_IT': 'italian (italy)',\n",
    "        'lld': 'ladin',\n",
    "        'lij': 'ligurian',\n",
    "        'lmo': 'lombard',\n",
    "        'nap': 'neapolitan',\n",
    "        'rm': 'rhaetian (?)',\n",
    "        'scn': 'sicilian',\n",
    "        'mwl': 'mirandese'\n",
    "    }\n",
    "\n",
    "    def __init__(self, to_langs, pipes=()):\n",
    "        if not pipes:\n",
    "            pipes = [\n",
    "                TranslationPipeline(\n",
    "                    model=AutoModelForSeq2SeqLM.from_pretrained(name),\n",
    "                    tokenizer=AutoTokenizer.from_pretrained(name),\n",
    "                    device=1 - torch.cuda.is_available()\n",
    "                ) for name in names\n",
    "            ]\n",
    "        self.pipes = pipes\n",
    "        self.to_langs = tolist(to_langs)\n",
    "\n",
    "    # def __call__(self, text, to_langs=()):\n",
    "    #     text = tolist(text)\n",
    "    #     to_langs = tolist(to_langs) or self.to_langs\n",
    "    #     for lang in to_langs:\n",
    "    #         text = [f'>>{lang}<< {t}' for t in text]\n",
    "    #         text = [row['translation_text'] for row in self.pipes[0](text)]\n",
    "    #         text = [row['translation_text'] for row in self.pipes[1](text)]\n",
    "    #     return text\n",
    "\n",
    "    def __call__(self, text, to_langs=()):\n",
    "        text = tolist(text)\n",
    "        to_langs = tolist(to_langs) or self.to_langs\n",
    "        steps = []\n",
    "        for lang in to_langs:\n",
    "            text = [f'>>{lang}<< {t}' for t in text]\n",
    "            text = [row['translation_text'] for row in self.pipes[0](text)]\n",
    "            text = [row['translation_text'] for row in self.pipes[1](text)]\n",
    "            steps.append(text)\n",
    "        return steps\n",
    "\n",
    "    def __repr__(self):\n",
    "        lang_str = \", \".join(repr(lang) for lang in self.to_langs)\n",
    "        return f'{func_name(self)}(to_langs=[{lang_str}])'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
