{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from grimoire.smartimmersion.dataset import SourceDataset\n",
    "\n",
    "df = SourceDataset().load_as_pandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Sample"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3.8/site-packages/ipykernel/ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    },
    {
     "data": {
      "text/plain": "    date_created score                                           sentence  \\\n103   2020-09-29   0.3  The tax records did not reveal any unreported ...   \n104   2020-09-29     1  Elephant rides, tiger cub selfies, dolphin sho...   \n105   2020-09-29   0.9  We’re encouraging the travel industry to seize...   \n106   2020-09-29   0.6  The company also scored highly due to defining...   \n107   2020-09-29     1  we know that people still want to see wild ani...   \n108   2020-09-29     1  More recently, I had a call with the World Ani...   \n109   2020-09-29   0.9  Commitment: Availability and quality of publis...   \n110   2020-09-29   0.1  Availability and scope of published time bound...   \n111   2020-09-29   0.9  Less demand for cruel tourist attractions mean...   \n112   2020-09-29     1  Another way to assess value is to look at how ...   \n113   2020-09-29   0.9  Productivity experts suggest the tactic of sta...   \n114   2020-09-29   0.6  If all string responses from a client should b...   \n115   2020-09-30     1  Doing something that gets you out of your curr...   \n116   2020-09-30     1  In the future, if an external party reaches ou...   \n117   2020-09-30     1  The activity did decrease after the signing of...   \n118   2020-10-01   0.9  Das use von nuklear power ist auf das rise im ...   \n119   2020-10-01   0.9  Except that some people are enjoying their fre...   \n120   2020-10-01   0.9  For Soda, the 12 kilometers of beach are her s...   \n121   2020-10-01   0.8  The Salvini Decree makes it easier for Italian...   \n122   2020-10-01   0.9  It allows coastal towns to apply for state fun...   \n123   2020-10-01   0.9  He has taken to sleeping in a car behind the d...   \n124   2020-10-01     1  It is also possible that the vendors buy their...   \n125   2020-10-01     1  with those who say in the surveys that they ar...   \n126   2020-10-01     1  The two women are sitting in the sand and our ...   \n127   2020-10-01     1  Alexei Navalny, 44, is Russia’s most prominent...   \n128   2020-10-01     1  German politicians and Angela Merkel have take...   \n129   2020-10-01   0.9  When you woke from the coma, you were a global...   \n130   2020-10-01     1  I can’t tell you the details, but we didn’t di...   \n131   2020-10-01   0.9  Perhaps 30 minutes passed from the point where...   \n132   2020-10-01   0.8  I guarantee that with these symptoms, I would ...   \n133   2020-10-01     1  But chemical warfare agents were considered th...   \n134   2020-10-01   0.8  Some interpretations of the law say this organ...   \n135   2020-10-01   0.6  Soon, developers will dumb down the applicatio...   \n136   2020-10-01     1  Don't overconstrain the\\ndetailed design and m...   \n137   2020-10-01   0.2  Because an ill-fitting structure is worse\\ntha...   \n138   2020-10-01     1  Prime Minister Boris Johnson's bill, which thi...   \n139   2020-10-02   0.6  It is one of four corona facilities serving a ...   \n140   2020-10-02   0.9  the number of coronavirus patients was on the ...   \n141   2020-10-02   0.7  We were told that the government will help us ...   \n142   2020-10-02   0.8  the number of infections potentially even as h...   \n143   2020-10-02   0.8  making India the fourth largest economy in the...   \n144   2020-10-02     1  The result was train platforms jammed with hun...   \n145   2020-10-02   0.9  Only eight of the staff of 1,200 were initiall...   \n146   2020-10-02   0.4  In the eyes of the government, at least, the v...   \n147   2020-10-02   0.7  Within just a single decade, 271 million peopl...   \n148   2020-10-02   0.6  . Young Indians usually study engineering or c...   \n149   2020-10-03   0.3  Western politicians who last year advocated bo...   \n150   2020-10-03   0.3  his thoughtful new book aims not so much to ad...   \n151   2020-10-03   0.5  eventually dominant across the globe, and now ...   \n152   2020-10-03   0.3  his thoughtful new book aims not so much to ad...   \n\n                                     uuid  \\\n103  cb35b80d-7a95-48b3-8c9a-e3f8ba22112e   \n104  ccc82c63-3632-44e8-bdfa-b97d2f18a675   \n105  2157ce90-7b2f-40eb-8ba2-6bf91e743134   \n106  66a22771-5dd9-433d-a095-f596d455b4e8   \n107  6fb3c627-8971-40b2-8210-ef12374e61b0   \n108  18e4d90e-4517-4d79-b5a7-7e08c8fa1eb5   \n109  7a4125db-fd34-44a4-92bc-a5afe67e46d4   \n110  a6abdd35-663c-4b39-bee1-6db315a4845b   \n111  8c581d73-f53c-482d-bdc1-99ee503a1864   \n112  cb3835ec-e731-4a74-a438-683604bccae4   \n113  d0645ef4-482c-4fea-9cba-b12c3abaeb7b   \n114  a358c8b7-4630-49cf-afee-dc14eccee6c6   \n115  635af6a5-aac6-4545-a09c-6a9ea73025d9   \n116  75b982ab-9b27-4b9d-9395-17e27fe6b353   \n117  b48af272-1d1d-4d3e-b664-8545243aa063   \n118  cb6a7c73-25ee-426b-9bc1-36a4639e7e60   \n119  b0d83eec-7a37-4164-9478-aa7acfd251b7   \n120  75ecf297-0b27-4b3c-a63a-89221d5dccf6   \n121  415a74cf-5de5-45fc-a537-fd7c18177364   \n122  c8f729c0-a39f-4a8d-8065-9fcda24c38a0   \n123  43229bd3-ad16-4e32-8966-5c1378f99b56   \n124  ab5a7b19-9677-4fe8-a7f9-7eae1a30d562   \n125  18041a07-cd9a-4e5f-8c24-b2fae2293636   \n126  43d2f18c-b2dd-441e-96d9-8177aff0bdcb   \n127  d3e8571a-4d3b-430b-b221-9741c5d28431   \n128  bdce1306-90a3-44bb-ab41-0540fb9fc095   \n129  a60d9754-9320-4eaf-8dc8-c55ed11c4e54   \n130  aa7056f5-0c94-4d26-bd10-a0ab73468cf0   \n131  d71fb405-43b4-40f1-9e03-cef705620444   \n132  ef8d6829-c03e-499c-934b-a6087f17b93d   \n133  9474fd4b-bc4f-40f8-8243-cec064ae0fc4   \n134  32f3484a-3339-49ca-83cb-bad45fd9cfeb   \n135  a14e4587-a94b-4b77-8b1a-c1de1b18fc39   \n136  6436ad5f-d0b6-45e2-9e21-790f7c72bd10   \n137  64a8bd86-4174-42f8-8b65-1a1533b0baa3   \n138  0a7bac50-3fe2-492c-b9a6-bf082f7af50b   \n139  d7cd5cfb-0c24-4962-a650-8b2b20c88dbd   \n140  ed43ef13-08ba-45a2-84d2-8db2881d0880   \n141  f96411e6-60e4-491d-916a-4798b981e1fb   \n142  7e5f5883-a2de-4a21-9a0e-dad4c04cbfb1   \n143  e0a73838-facd-4e6c-8c05-9685afca6bf3   \n144  049b52cb-3232-4687-bc6b-305ec24ba44c   \n145  3eff7a6c-7475-469b-8f87-df5ac415561c   \n146  ae8ab22e-3107-4857-8481-35ff0c9fc19a   \n147  bcf7ad86-46f0-4b18-a4a1-755bf3633e81   \n148  de48c3e1-0ee0-416b-9a29-935f39ef99eb   \n149  f933cae9-8de5-4217-a9c2-3d76060eea64   \n150  8a5855eb-96d0-4488-83be-1aa4cd718ea1   \n151  966d97fb-4ae8-4b3a-84a9-8deeb502b241   \n152  e170d97b-f01a-4fa1-9865-531f1bac11df   \n\n                                             variation  \n103  Das MwSt Aufzeichnungen tat nicht verraten irg...  \n104  Elefant Fahrten, Tiger cub Selfies, dolphin ze...  \n105  We’re encouraging das Reise Industrie zu seize...  \n106  Das Unternehmen ebenfalls erzielte höchst fäll...  \n107  wir kennt Das Menschen immer noch want zu sehe...  \n108  Mehr recently, ich had ein Anruf mit das Welt ...  \n109  Commitment Availability und Qualität von publi...  \n110  Verfügbarkeit und Umfang von published Zeit ge...  \n111  Weniger demand zum cruel Tourist attractions m...  \n112  Ein weiterer Weg zu assess Wert ist zu look be...  \n113  Produktivität Experten vorschlagen das Taktik ...  \n114  If alle Zeichenfolge Antworten von a Klient so...  \n115  Doing etwas Das gets Sie aus von your aktuell ...  \n116  Im das future, if ein extern Party reaches aus...  \n117  Das activity tat decrease nach dem das signing...  \n118  Das use von nuklear power ist auf das rise im ...  \n119  Außer Das etwas people sind genießen ihr frei ...  \n120  Zum Limonade das 12 Kilometer von beach sind i...  \n121  Das Salvini Dekret macht es easier zum Italien...  \n122  Es erlaubt coastal Städte zu apply zum Zustand...  \n123  He hat taken zu Schlafen im ein car hinter das...  \n124  Es ist also possible Das das vendors buy ihr W...  \n125  mit those Wer sagen im das surveys Das Sie are...  \n126  Das zwei Frauen sind Sitzung im das Sand und u...  \n127  Alexei Navalny 44 ist Russlands die meisten pr...  \n128  Deutsche Politiker und Angela Merkel haben tak...  \n129  Wann Sie woke von das Koma Sie wurden ein glob...  \n130  ich can’t sagen Sie das details aber wir nicht...  \n131  Perhaps 30 minutes bestanden von das Punkt wo ...  \n132  ich Garantie Das mit diese Symptome ich würde ...  \n133  Aber chemisch warfare Agenten wurden considere...  \n134  Some Interpretationen von das law sagen Dies o...  \n135  Soon developers will Dumm down das application...  \n136  Don't overconstrain das detailed Design und Mo...  \n137  weil ein schlecht sitzend Struktur ist schlech...  \n138  Prime Minister Boris Johnsons bill welche Dies...  \n139  Es ist einer von vier Corona Einrichtungen ser...  \n140  das Nummer von Coronavirus Patienten war auf d...  \n141  Wir wurden told Das das government will Hilfe ...  \n142  das Nummer von Infektionen potentially sogar w...  \n143  making Indien das vierte largest economy im da...  \n144  Das result war train platforms jammed mit Hund...  \n145  Only eight von das Mitarbeiter von 1,200 wurde...  \n146  Im das eyes von das Regierung mindestens das V...  \n147  Innerhalb gerade ein Single decade 271 Million...  \n148  . Young Indians in normalerweise study Ingenie...  \n149  Western Politiker Wer letzte Jahr befürwortet ...  \n150  seine nachdenklich Neu Buch Ziele nicht damit ...  \n151  schließlich Dominant über das Globus und jetzt...  \n152  seine nachdenklich Neu Buch Ziele nicht so vie...  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>date_created</th>\n      <th>score</th>\n      <th>sentence</th>\n      <th>uuid</th>\n      <th>variation</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>103</th>\n      <td>2020-09-29</td>\n      <td>0.3</td>\n      <td>The tax records did not reveal any unreported ...</td>\n      <td>cb35b80d-7a95-48b3-8c9a-e3f8ba22112e</td>\n      <td>Das MwSt Aufzeichnungen tat nicht verraten irg...</td>\n    </tr>\n    <tr>\n      <th>104</th>\n      <td>2020-09-29</td>\n      <td>1</td>\n      <td>Elephant rides, tiger cub selfies, dolphin sho...</td>\n      <td>ccc82c63-3632-44e8-bdfa-b97d2f18a675</td>\n      <td>Elefant Fahrten, Tiger cub Selfies, dolphin ze...</td>\n    </tr>\n    <tr>\n      <th>105</th>\n      <td>2020-09-29</td>\n      <td>0.9</td>\n      <td>We’re encouraging the travel industry to seize...</td>\n      <td>2157ce90-7b2f-40eb-8ba2-6bf91e743134</td>\n      <td>We’re encouraging das Reise Industrie zu seize...</td>\n    </tr>\n    <tr>\n      <th>106</th>\n      <td>2020-09-29</td>\n      <td>0.6</td>\n      <td>The company also scored highly due to defining...</td>\n      <td>66a22771-5dd9-433d-a095-f596d455b4e8</td>\n      <td>Das Unternehmen ebenfalls erzielte höchst fäll...</td>\n    </tr>\n    <tr>\n      <th>107</th>\n      <td>2020-09-29</td>\n      <td>1</td>\n      <td>we know that people still want to see wild ani...</td>\n      <td>6fb3c627-8971-40b2-8210-ef12374e61b0</td>\n      <td>wir kennt Das Menschen immer noch want zu sehe...</td>\n    </tr>\n    <tr>\n      <th>108</th>\n      <td>2020-09-29</td>\n      <td>1</td>\n      <td>More recently, I had a call with the World Ani...</td>\n      <td>18e4d90e-4517-4d79-b5a7-7e08c8fa1eb5</td>\n      <td>Mehr recently, ich had ein Anruf mit das Welt ...</td>\n    </tr>\n    <tr>\n      <th>109</th>\n      <td>2020-09-29</td>\n      <td>0.9</td>\n      <td>Commitment: Availability and quality of publis...</td>\n      <td>7a4125db-fd34-44a4-92bc-a5afe67e46d4</td>\n      <td>Commitment Availability und Qualität von publi...</td>\n    </tr>\n    <tr>\n      <th>110</th>\n      <td>2020-09-29</td>\n      <td>0.1</td>\n      <td>Availability and scope of published time bound...</td>\n      <td>a6abdd35-663c-4b39-bee1-6db315a4845b</td>\n      <td>Verfügbarkeit und Umfang von published Zeit ge...</td>\n    </tr>\n    <tr>\n      <th>111</th>\n      <td>2020-09-29</td>\n      <td>0.9</td>\n      <td>Less demand for cruel tourist attractions mean...</td>\n      <td>8c581d73-f53c-482d-bdc1-99ee503a1864</td>\n      <td>Weniger demand zum cruel Tourist attractions m...</td>\n    </tr>\n    <tr>\n      <th>112</th>\n      <td>2020-09-29</td>\n      <td>1</td>\n      <td>Another way to assess value is to look at how ...</td>\n      <td>cb3835ec-e731-4a74-a438-683604bccae4</td>\n      <td>Ein weiterer Weg zu assess Wert ist zu look be...</td>\n    </tr>\n    <tr>\n      <th>113</th>\n      <td>2020-09-29</td>\n      <td>0.9</td>\n      <td>Productivity experts suggest the tactic of sta...</td>\n      <td>d0645ef4-482c-4fea-9cba-b12c3abaeb7b</td>\n      <td>Produktivität Experten vorschlagen das Taktik ...</td>\n    </tr>\n    <tr>\n      <th>114</th>\n      <td>2020-09-29</td>\n      <td>0.6</td>\n      <td>If all string responses from a client should b...</td>\n      <td>a358c8b7-4630-49cf-afee-dc14eccee6c6</td>\n      <td>If alle Zeichenfolge Antworten von a Klient so...</td>\n    </tr>\n    <tr>\n      <th>115</th>\n      <td>2020-09-30</td>\n      <td>1</td>\n      <td>Doing something that gets you out of your curr...</td>\n      <td>635af6a5-aac6-4545-a09c-6a9ea73025d9</td>\n      <td>Doing etwas Das gets Sie aus von your aktuell ...</td>\n    </tr>\n    <tr>\n      <th>116</th>\n      <td>2020-09-30</td>\n      <td>1</td>\n      <td>In the future, if an external party reaches ou...</td>\n      <td>75b982ab-9b27-4b9d-9395-17e27fe6b353</td>\n      <td>Im das future, if ein extern Party reaches aus...</td>\n    </tr>\n    <tr>\n      <th>117</th>\n      <td>2020-09-30</td>\n      <td>1</td>\n      <td>The activity did decrease after the signing of...</td>\n      <td>b48af272-1d1d-4d3e-b664-8545243aa063</td>\n      <td>Das activity tat decrease nach dem das signing...</td>\n    </tr>\n    <tr>\n      <th>118</th>\n      <td>2020-10-01</td>\n      <td>0.9</td>\n      <td>Das use von nuklear power ist auf das rise im ...</td>\n      <td>cb6a7c73-25ee-426b-9bc1-36a4639e7e60</td>\n      <td>Das use von nuklear power ist auf das rise im ...</td>\n    </tr>\n    <tr>\n      <th>119</th>\n      <td>2020-10-01</td>\n      <td>0.9</td>\n      <td>Except that some people are enjoying their fre...</td>\n      <td>b0d83eec-7a37-4164-9478-aa7acfd251b7</td>\n      <td>Außer Das etwas people sind genießen ihr frei ...</td>\n    </tr>\n    <tr>\n      <th>120</th>\n      <td>2020-10-01</td>\n      <td>0.9</td>\n      <td>For Soda, the 12 kilometers of beach are her s...</td>\n      <td>75ecf297-0b27-4b3c-a63a-89221d5dccf6</td>\n      <td>Zum Limonade das 12 Kilometer von beach sind i...</td>\n    </tr>\n    <tr>\n      <th>121</th>\n      <td>2020-10-01</td>\n      <td>0.8</td>\n      <td>The Salvini Decree makes it easier for Italian...</td>\n      <td>415a74cf-5de5-45fc-a537-fd7c18177364</td>\n      <td>Das Salvini Dekret macht es easier zum Italien...</td>\n    </tr>\n    <tr>\n      <th>122</th>\n      <td>2020-10-01</td>\n      <td>0.9</td>\n      <td>It allows coastal towns to apply for state fun...</td>\n      <td>c8f729c0-a39f-4a8d-8065-9fcda24c38a0</td>\n      <td>Es erlaubt coastal Städte zu apply zum Zustand...</td>\n    </tr>\n    <tr>\n      <th>123</th>\n      <td>2020-10-01</td>\n      <td>0.9</td>\n      <td>He has taken to sleeping in a car behind the d...</td>\n      <td>43229bd3-ad16-4e32-8966-5c1378f99b56</td>\n      <td>He hat taken zu Schlafen im ein car hinter das...</td>\n    </tr>\n    <tr>\n      <th>124</th>\n      <td>2020-10-01</td>\n      <td>1</td>\n      <td>It is also possible that the vendors buy their...</td>\n      <td>ab5a7b19-9677-4fe8-a7f9-7eae1a30d562</td>\n      <td>Es ist also possible Das das vendors buy ihr W...</td>\n    </tr>\n    <tr>\n      <th>125</th>\n      <td>2020-10-01</td>\n      <td>1</td>\n      <td>with those who say in the surveys that they ar...</td>\n      <td>18041a07-cd9a-4e5f-8c24-b2fae2293636</td>\n      <td>mit those Wer sagen im das surveys Das Sie are...</td>\n    </tr>\n    <tr>\n      <th>126</th>\n      <td>2020-10-01</td>\n      <td>1</td>\n      <td>The two women are sitting in the sand and our ...</td>\n      <td>43d2f18c-b2dd-441e-96d9-8177aff0bdcb</td>\n      <td>Das zwei Frauen sind Sitzung im das Sand und u...</td>\n    </tr>\n    <tr>\n      <th>127</th>\n      <td>2020-10-01</td>\n      <td>1</td>\n      <td>Alexei Navalny, 44, is Russia’s most prominent...</td>\n      <td>d3e8571a-4d3b-430b-b221-9741c5d28431</td>\n      <td>Alexei Navalny 44 ist Russlands die meisten pr...</td>\n    </tr>\n    <tr>\n      <th>128</th>\n      <td>2020-10-01</td>\n      <td>1</td>\n      <td>German politicians and Angela Merkel have take...</td>\n      <td>bdce1306-90a3-44bb-ab41-0540fb9fc095</td>\n      <td>Deutsche Politiker und Angela Merkel haben tak...</td>\n    </tr>\n    <tr>\n      <th>129</th>\n      <td>2020-10-01</td>\n      <td>0.9</td>\n      <td>When you woke from the coma, you were a global...</td>\n      <td>a60d9754-9320-4eaf-8dc8-c55ed11c4e54</td>\n      <td>Wann Sie woke von das Koma Sie wurden ein glob...</td>\n    </tr>\n    <tr>\n      <th>130</th>\n      <td>2020-10-01</td>\n      <td>1</td>\n      <td>I can’t tell you the details, but we didn’t di...</td>\n      <td>aa7056f5-0c94-4d26-bd10-a0ab73468cf0</td>\n      <td>ich can’t sagen Sie das details aber wir nicht...</td>\n    </tr>\n    <tr>\n      <th>131</th>\n      <td>2020-10-01</td>\n      <td>0.9</td>\n      <td>Perhaps 30 minutes passed from the point where...</td>\n      <td>d71fb405-43b4-40f1-9e03-cef705620444</td>\n      <td>Perhaps 30 minutes bestanden von das Punkt wo ...</td>\n    </tr>\n    <tr>\n      <th>132</th>\n      <td>2020-10-01</td>\n      <td>0.8</td>\n      <td>I guarantee that with these symptoms, I would ...</td>\n      <td>ef8d6829-c03e-499c-934b-a6087f17b93d</td>\n      <td>ich Garantie Das mit diese Symptome ich würde ...</td>\n    </tr>\n    <tr>\n      <th>133</th>\n      <td>2020-10-01</td>\n      <td>1</td>\n      <td>But chemical warfare agents were considered th...</td>\n      <td>9474fd4b-bc4f-40f8-8243-cec064ae0fc4</td>\n      <td>Aber chemisch warfare Agenten wurden considere...</td>\n    </tr>\n    <tr>\n      <th>134</th>\n      <td>2020-10-01</td>\n      <td>0.8</td>\n      <td>Some interpretations of the law say this organ...</td>\n      <td>32f3484a-3339-49ca-83cb-bad45fd9cfeb</td>\n      <td>Some Interpretationen von das law sagen Dies o...</td>\n    </tr>\n    <tr>\n      <th>135</th>\n      <td>2020-10-01</td>\n      <td>0.6</td>\n      <td>Soon, developers will dumb down the applicatio...</td>\n      <td>a14e4587-a94b-4b77-8b1a-c1de1b18fc39</td>\n      <td>Soon developers will Dumm down das application...</td>\n    </tr>\n    <tr>\n      <th>136</th>\n      <td>2020-10-01</td>\n      <td>1</td>\n      <td>Don't overconstrain the\\ndetailed design and m...</td>\n      <td>6436ad5f-d0b6-45e2-9e21-790f7c72bd10</td>\n      <td>Don't overconstrain das detailed Design und Mo...</td>\n    </tr>\n    <tr>\n      <th>137</th>\n      <td>2020-10-01</td>\n      <td>0.2</td>\n      <td>Because an ill-fitting structure is worse\\ntha...</td>\n      <td>64a8bd86-4174-42f8-8b65-1a1533b0baa3</td>\n      <td>weil ein schlecht sitzend Struktur ist schlech...</td>\n    </tr>\n    <tr>\n      <th>138</th>\n      <td>2020-10-01</td>\n      <td>1</td>\n      <td>Prime Minister Boris Johnson's bill, which thi...</td>\n      <td>0a7bac50-3fe2-492c-b9a6-bf082f7af50b</td>\n      <td>Prime Minister Boris Johnsons bill welche Dies...</td>\n    </tr>\n    <tr>\n      <th>139</th>\n      <td>2020-10-02</td>\n      <td>0.6</td>\n      <td>It is one of four corona facilities serving a ...</td>\n      <td>d7cd5cfb-0c24-4962-a650-8b2b20c88dbd</td>\n      <td>Es ist einer von vier Corona Einrichtungen ser...</td>\n    </tr>\n    <tr>\n      <th>140</th>\n      <td>2020-10-02</td>\n      <td>0.9</td>\n      <td>the number of coronavirus patients was on the ...</td>\n      <td>ed43ef13-08ba-45a2-84d2-8db2881d0880</td>\n      <td>das Nummer von Coronavirus Patienten war auf d...</td>\n    </tr>\n    <tr>\n      <th>141</th>\n      <td>2020-10-02</td>\n      <td>0.7</td>\n      <td>We were told that the government will help us ...</td>\n      <td>f96411e6-60e4-491d-916a-4798b981e1fb</td>\n      <td>Wir wurden told Das das government will Hilfe ...</td>\n    </tr>\n    <tr>\n      <th>142</th>\n      <td>2020-10-02</td>\n      <td>0.8</td>\n      <td>the number of infections potentially even as h...</td>\n      <td>7e5f5883-a2de-4a21-9a0e-dad4c04cbfb1</td>\n      <td>das Nummer von Infektionen potentially sogar w...</td>\n    </tr>\n    <tr>\n      <th>143</th>\n      <td>2020-10-02</td>\n      <td>0.8</td>\n      <td>making India the fourth largest economy in the...</td>\n      <td>e0a73838-facd-4e6c-8c05-9685afca6bf3</td>\n      <td>making Indien das vierte largest economy im da...</td>\n    </tr>\n    <tr>\n      <th>144</th>\n      <td>2020-10-02</td>\n      <td>1</td>\n      <td>The result was train platforms jammed with hun...</td>\n      <td>049b52cb-3232-4687-bc6b-305ec24ba44c</td>\n      <td>Das result war train platforms jammed mit Hund...</td>\n    </tr>\n    <tr>\n      <th>145</th>\n      <td>2020-10-02</td>\n      <td>0.9</td>\n      <td>Only eight of the staff of 1,200 were initiall...</td>\n      <td>3eff7a6c-7475-469b-8f87-df5ac415561c</td>\n      <td>Only eight von das Mitarbeiter von 1,200 wurde...</td>\n    </tr>\n    <tr>\n      <th>146</th>\n      <td>2020-10-02</td>\n      <td>0.4</td>\n      <td>In the eyes of the government, at least, the v...</td>\n      <td>ae8ab22e-3107-4857-8481-35ff0c9fc19a</td>\n      <td>Im das eyes von das Regierung mindestens das V...</td>\n    </tr>\n    <tr>\n      <th>147</th>\n      <td>2020-10-02</td>\n      <td>0.7</td>\n      <td>Within just a single decade, 271 million peopl...</td>\n      <td>bcf7ad86-46f0-4b18-a4a1-755bf3633e81</td>\n      <td>Innerhalb gerade ein Single decade 271 Million...</td>\n    </tr>\n    <tr>\n      <th>148</th>\n      <td>2020-10-02</td>\n      <td>0.6</td>\n      <td>. Young Indians usually study engineering or c...</td>\n      <td>de48c3e1-0ee0-416b-9a29-935f39ef99eb</td>\n      <td>. Young Indians in normalerweise study Ingenie...</td>\n    </tr>\n    <tr>\n      <th>149</th>\n      <td>2020-10-03</td>\n      <td>0.3</td>\n      <td>Western politicians who last year advocated bo...</td>\n      <td>f933cae9-8de5-4217-a9c2-3d76060eea64</td>\n      <td>Western Politiker Wer letzte Jahr befürwortet ...</td>\n    </tr>\n    <tr>\n      <th>150</th>\n      <td>2020-10-03</td>\n      <td>0.3</td>\n      <td>his thoughtful new book aims not so much to ad...</td>\n      <td>8a5855eb-96d0-4488-83be-1aa4cd718ea1</td>\n      <td>seine nachdenklich Neu Buch Ziele nicht damit ...</td>\n    </tr>\n    <tr>\n      <th>151</th>\n      <td>2020-10-03</td>\n      <td>0.5</td>\n      <td>eventually dominant across the globe, and now ...</td>\n      <td>966d97fb-4ae8-4b3a-84a9-8deeb502b241</td>\n      <td>schließlich Dominant über das Globus und jetzt...</td>\n    </tr>\n    <tr>\n      <th>152</th>\n      <td>2020-10-03</td>\n      <td>0.3</td>\n      <td>his thoughtful new book aims not so much to ad...</td>\n      <td>e170d97b-f01a-4fa1-9865-531f1bac11df</td>\n      <td>seine nachdenklich Neu Buch Ziele nicht so vie...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail(50)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Does the complexity increase with the word size?"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "<matplotlib.axes._subplots.AxesSubplot at 0x7f99669177c0>"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "[encoded data, 11092 characters]\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sentence_size = df.apply(lambda x: len(x['sentence']), axis=1).reset_index(name='sentence_size')\n",
    "merged = pd.concat([df, sentence_size], axis=1)\n",
    "merged.sort_values(by=['sentence_size'])\n",
    "\n",
    "merged.sentence_size = pd.to_numeric(merged.sentence_size)\n",
    "merged.score = pd.to_numeric(merged.score)\n",
    "\n",
    "merged.plot(x='sentence_size', y='score', style='o')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Does the number of words always match the number of translated words?"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            index  original_words_count       index  variation_words_count\n",
      "count   31.000000             31.000000   31.000000              31.000000\n",
      "mean    85.096774             19.096774   85.096774              20.516129\n",
      "std     41.393522              7.359143   41.393522               7.347884\n",
      "min     12.000000              7.000000   12.000000               8.000000\n",
      "25%     55.500000             14.000000   55.500000              15.500000\n",
      "50%     97.000000             16.000000   97.000000              18.000000\n",
      "75%    114.500000             25.000000  114.500000              26.000000\n",
      "max    148.000000             33.000000  148.000000              35.000000\n"
     ]
    },
    {
     "data": {
      "text/plain": "     index  original_words_count\n0        0                     3\n1        1                     4\n2        2                     5\n3        3                     5\n4        4                     9\n..     ...                   ...\n148    148                    25\n149    149                    22\n150    150                    21\n151    151                    10\n152    152                    21\n\n[153 rows x 2 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>index</th>\n      <th>original_words_count</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>9</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>148</th>\n      <td>148</td>\n      <td>25</td>\n    </tr>\n    <tr>\n      <th>149</th>\n      <td>149</td>\n      <td>22</td>\n    </tr>\n    <tr>\n      <th>150</th>\n      <td>150</td>\n      <td>21</td>\n    </tr>\n    <tr>\n      <th>151</th>\n      <td>151</td>\n      <td>10</td>\n    </tr>\n    <tr>\n      <th>152</th>\n      <td>152</td>\n      <td>21</td>\n    </tr>\n  </tbody>\n</table>\n<p>153 rows × 2 columns</p>\n</div>"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "original_c = df.apply(lambda x: len(x['sentence'].split(' ')), axis=1).reset_index(name='original_words_count')\n",
    "variation_c = df.apply(lambda x: len(x['variation'].split(' ')), axis=1).reset_index(name='variation_words_count')\n",
    "\n",
    "result = pd.concat([df['uuid'], original_c, variation_c], axis=1)\n",
    "\n",
    "\n",
    "with_different_size = result[result.apply(lambda x: x['original_words_count'] != x['variation_words_count'], axis=1)]\n",
    "print(with_different_size.describe())\n",
    "with_different_size\n",
    "original_c\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Compute percentage translated"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "                                     uuid  percentage_translated\n0    013b5afe-6d9f-44ac-b2d5-024795c6752f               0.666667\n1    9767334c-ebbe-417c-9839-81d1be073c25               0.750000\n2    8fd81571-44ba-4bdd-bd66-797f99b11c04               0.800000\n3    4719992d-32ee-490d-8fcc-ca1545c32c50               0.800000\n4    51417618-eb6d-44ec-9a10-07f30d4069a7               0.444444\n..                                    ...                    ...\n148  de48c3e1-0ee0-416b-9a29-935f39ef99eb               0.880000\n149  f933cae9-8de5-4217-a9c2-3d76060eea64               0.954545\n150  8a5855eb-96d0-4488-83be-1aa4cd718ea1               1.000000\n151  966d97fb-4ae8-4b3a-84a9-8deeb502b241               1.000000\n152  e170d97b-f01a-4fa1-9865-531f1bac11df               0.761905\n\n[153 rows x 2 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>uuid</th>\n      <th>percentage_translated</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>013b5afe-6d9f-44ac-b2d5-024795c6752f</td>\n      <td>0.666667</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>9767334c-ebbe-417c-9839-81d1be073c25</td>\n      <td>0.750000</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>8fd81571-44ba-4bdd-bd66-797f99b11c04</td>\n      <td>0.800000</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>4719992d-32ee-490d-8fcc-ca1545c32c50</td>\n      <td>0.800000</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>51417618-eb6d-44ec-9a10-07f30d4069a7</td>\n      <td>0.444444</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>148</th>\n      <td>de48c3e1-0ee0-416b-9a29-935f39ef99eb</td>\n      <td>0.880000</td>\n    </tr>\n    <tr>\n      <th>149</th>\n      <td>f933cae9-8de5-4217-a9c2-3d76060eea64</td>\n      <td>0.954545</td>\n    </tr>\n    <tr>\n      <th>150</th>\n      <td>8a5855eb-96d0-4488-83be-1aa4cd718ea1</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>151</th>\n      <td>966d97fb-4ae8-4b3a-84a9-8deeb502b241</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>152</th>\n      <td>e170d97b-f01a-4fa1-9865-531f1bac11df</td>\n      <td>0.761905</td>\n    </tr>\n  </tbody>\n</table>\n<p>153 rows × 2 columns</p>\n</div>"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "from grimoire.smartimmersion.dataset import TranslationPercentage\n",
    "\n",
    "\n",
    "\n",
    "percentage_tranlated = df.apply(lambda x: TranslationPercentage.percentage_difference(x['sentence'], x['variation']), axis=1).reset_index(name='percentage_translated')\n",
    "\n",
    "result = pd.concat([df['uuid'],  percentage_tranlated['percentage_translated']], axis=1)\n",
    "\n",
    "result\n",
    "\n",
    "\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "pycharm-3da30a04",
   "language": "python",
   "display_name": "PyCharm (machineLearningExperiments)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
