{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Which words repeat?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jean/projects/grimoire/grimoire/smartimmersion/scoring/dataset.py:60: FutureWarning: pandas.io.json.json_normalize is deprecated, use pandas.json_normalize instead\n",
      "  return json_normalize(result)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<bound method NDFrame.describe of     date_created score                                           sentence  \\\n",
       "0     2020-09-26     1                                can detect synonyms   \n",
       "1     2020-09-26     1  Evangelize the niggers so that they stay forev...   \n",
       "2     2020-09-28   0.9  Most of the batteries in today's electric vehi...   \n",
       "3     2020-09-28   0.4  Each variety has different parameters for ener...   \n",
       "4     2020-09-28   0.8  Musk also intends to build a battery factory a...   \n",
       "..           ...   ...                                                ...   \n",
       "585   2020-09-28     1  Roughly 20 to 25 terawatt hours\" of battery pr...   \n",
       "586   2020-09-28   0.9  Tesla is right at the front of the global race...   \n",
       "587   2020-09-28     1  battery companies of today are like the oil co...   \n",
       "588   2020-09-28   0.5  only around 8 percent of newly registered vehi...   \n",
       "589   2020-09-28   0.9  At that point, it will just be a matter of tim...   \n",
       "\n",
       "                                     uuid  \\\n",
       "0    013b5afe-6d9f-44ac-b2d5-024795c6752f   \n",
       "1    235b1608-32b3-4a76-841f-de065e1892bb   \n",
       "2    6d7c4bb3-eac7-47c1-baed-325000bc492a   \n",
       "3    7bdb3d6b-4bb0-4c54-8b00-ab6a9157bb7e   \n",
       "4    7f740dce-9222-4949-b63e-c317456494e3   \n",
       "..                                    ...   \n",
       "585  2c567923-444f-4f23-8942-1392818e2110   \n",
       "586  6cc5faed-743d-47b5-8637-5a839b63619b   \n",
       "587  391421e8-a5ff-4b71-a794-260917df8fbf   \n",
       "588  b25fd67a-88f2-4c76-bc32-12be1bbfc92d   \n",
       "589  e0181767-9879-47b1-a052-eda03e671cf6   \n",
       "\n",
       "                                             variation  \n",
       "0                                 kann detect Synonyme  \n",
       "1    Evangelisieren das Nigger so Das Sie bleibe im...  \n",
       "2    Die meisten von das Batterien im heute elektri...  \n",
       "3    Jeder Vielfalt hat anders Parameter zum Energi...  \n",
       "4    Musk ebenfalls intends zu build ein Batterie F...  \n",
       "..                                                 ...  \n",
       "585  Roughly 20 zu 25 Terawatt Std von Batterie Pro...  \n",
       "586  Tesla ist Recht at das front von das global Re...  \n",
       "587  Batterie Firmen von heute sind mögen das Öl Fi...  \n",
       "588  nur um 8 percent von neu Eingetragen Fahrzeuge...  \n",
       "589  At Das Punkt, es will just Sein ein matter von...  \n",
       "\n",
       "[590 rows x 5 columns]>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from grimoire.smartimmersion.scoring.dataset import SourceDataset\n",
    "\n",
    "pdf = SourceDataset().load_as_pandas()\n",
    "pdf.dtypes\n",
    "\n",
    "pdf.describe\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3.8/site-packages/ipykernel/ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n",
      "/home/jean/.local/lib/python3.8/site-packages/pyspark/sql/context.py:75: DeprecationWarning: Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - in-memory</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://archlinux:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.0.1-SNAPSHOT</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>pyspark-shell</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x7f32a383f5b0>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "import pyspark\n",
    "from pyspark.sql import SparkSession\n",
    "\n",
    "spark = SparkSession.builder\\\n",
    "    .config('spark.sql.execution.pandas.convertToArrowArraySafely', False) \\\n",
    "    .getOrCreate()\n",
    "            \n",
    "\n",
    "\n",
    "from pyspark.sql.types import *\n",
    "\n",
    "spark\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jean/.local/lib/python3.8/site-packages/pyspark/sql/pandas/conversion.py:291: UserWarning: createDataFrame attempted Arrow optimization because 'spark.sql.execution.arrow.pyspark.enabled' is set to true; however, failed by the reason below:\n",
      "  ('Exception thrown when converting pandas.Series (object) to Arrow Array (string). It can be caused by overflows or other unsafe conversions warned by Arrow. Arrow safe type check can be disabled by using SQL config `spark.sql.execution.pandas.convertToArrowArraySafely`.', ArrowTypeError(\"Expected a string or bytes object, got a 'int' object\"))\n",
      "Attempting non-optimization as 'spark.sql.execution.arrow.pyspark.fallback.enabled' is set to true.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DataFrame[count(1): bigint]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mySchema = StructType([StructField('date_created', StringType(), True)\\\n",
    "                       ,StructField('score', StringType(), True)\\\n",
    "                       ,StructField('sentence', StringType(), True)\\\n",
    "                       ,StructField('uuid', StringType(), True)\\\n",
    "                       ,StructField('variation', StringType(), True)\\\n",
    "                       \n",
    "                      ])\n",
    "df = spark.createDataFrame(pdf, schema=mySchema)\n",
    "df.createOrReplaceTempView(\"dataset\")\n",
    "\n",
    "display(spark.sql('select count(*) from dataset limit 10'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
