{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Data set look\n",
    "\n",
    "## Statistics"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'SmartImmersion' from 'grimoire.smartimmersion.smartimmersion' (/home/jean/projects/grimoire/grimoire/smartimmersion/smartimmersion/__init__.py)",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mImportError\u001B[0m                               Traceback (most recent call last)",
      "\u001B[0;32m<ipython-input-1-5f24e6351155>\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m      1\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mpandas\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0mpd\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      2\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 3\u001B[0;31m \u001B[0;32mfrom\u001B[0m \u001B[0mgrimoire\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msmartimmersion\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mscoring\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mdataset\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mSourceDataset\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m      4\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      5\u001B[0m \u001B[0mdf\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mSourceDataset\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mload_as_pandas\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/projects/grimoire/grimoire/smartimmersion/__init__.py\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[0;32m----> 1\u001B[0;31m \u001B[0;32mfrom\u001B[0m \u001B[0mgrimoire\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msmartimmersion\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msmartimmersion\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mSmartImmersion\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m",
      "\u001B[0;31mImportError\u001B[0m: cannot import name 'SmartImmersion' from 'grimoire.smartimmersion.smartimmersion' (/home/jean/projects/grimoire/grimoire/smartimmersion/smartimmersion/__init__.py)"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from grimoire.smartimmersion.scoring.dataset import SourceDataset\n",
    "\n",
    "df = SourceDataset().load_as_pandas()\n",
    "\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "df.tail(50)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Does the complexity increase with the word size?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "sentence_size = df.apply(lambda x: len(x['sentence']), axis=1).reset_index(name='sentence_size')\n",
    "merged = pd.concat([df, sentence_size], axis=1)\n",
    "merged.sort_values(by=['sentence_size'])\n",
    "\n",
    "merged.sentence_size = pd.to_numeric(merged.sentence_size)\n",
    "merged.score = pd.to_numeric(merged.score)\n",
    "\n",
    "merged.plot(x='sentence_size', y='score', style='o')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Does the number of words always match the number of translated words?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "original_c = df.apply(lambda x: len(x['sentence'].split(' ')), axis=1).reset_index(name='original_words_count')\n",
    "variation_c = df.apply(lambda x: len(x['variation'].split(' ')), axis=1).reset_index(name='variation_words_count')\n",
    "\n",
    "result = pd.concat([df['uuid'], original_c, variation_c], axis=1)\n",
    "\n",
    "\n",
    "with_different_size = result[result.apply(lambda x: x['original_words_count'] != x['variation_words_count'], axis=1)]\n",
    "print(with_different_size.describe())\n",
    "with_different_size\n",
    "original_c\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute percentage translated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "from grimoire.smartimmersion.dataset import TranslationPercentage\n",
    "\n",
    "\n",
    "\n",
    "percentage_tranlated = df.apply(lambda x: TranslationPercentage.percentage_difference(x['sentence'], x['variation']), axis=1).reset_index(name='percentage_translated')\n",
    "\n",
    "result = pd.concat([df['uuid'],  percentage_tranlated['percentage_translated']], axis=1)\n",
    "\n",
    "result\n",
    "\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "pycharm-3238e124",
   "language": "python",
   "display_name": "PyCharm (grimoire)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
