{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jean/anaconda3/envs/dbconnect/lib/python3.7/site-packages/ipykernel/ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n",
      "/home/jean/projects/grimoire/grimoire/smartimmersion/scoring/dataset.py:59: FutureWarning: pandas.io.json.json_normalize is deprecated, use pandas.json_normalize instead\n",
      "  return json_normalize(result)\n"
     ]
    }
   ],
   "source": [
    "from grimoire.smartimmersion.scoring.dataset import SourceDataset\n",
    "pdf = SourceDataset().load_as_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "<pyspark.sql.session.SparkSession at 0x7faacefb4210>",
      "text/html": "\n            <div>\n                <p><b>SparkSession - in-memory</b></p>\n                \n        <div>\n            <p><b>SparkContext</b></p>\n\n            <p><a href=\"http://127.0.0.1:4040\">Spark UI</a></p>\n\n            <dl>\n              <dt>Version</dt>\n                <dd><code>v3.0.1</code></dd>\n              <dt>Master</dt>\n                <dd><code>local[*]</code></dd>\n              <dt>AppName</dt>\n                <dd><code>pyspark-shell</code></dd>\n            </dl>\n        </div>\n        \n            </div>\n        "
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import findspark\n",
    "findspark.init(spark_home='/opt/apache-spark')\n",
    "import pyspark\n",
    "from pyspark.sql import SparkSession\n",
    "\n",
    "\n",
    "spark = SparkSession.builder.getOrCreate()\n",
    "\n",
    "\n",
    "\n",
    "from pyspark.sql.types import *\n",
    "\n",
    "spark\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "Py4JError",
     "evalue": "An error occurred while calling o31.pandasRespectSessionTimeZone. Trace:\npy4j.Py4JException: Method pandasRespectSessionTimeZone([]) does not exist\n\tat py4j.reflection.ReflectionEngine.getMethod(ReflectionEngine.java:318)\n\tat py4j.reflection.ReflectionEngine.getMethod(ReflectionEngine.java:326)\n\tat py4j.Gateway.invoke(Gateway.java:274)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n\tat java.lang.Thread.run(Thread.java:748)\n\n",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mPy4JError\u001B[0m                                 Traceback (most recent call last)",
      "\u001B[0;32m<ipython-input-6-e6748207b176>\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m      7\u001B[0m                       ])\n\u001B[1;32m      8\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 9\u001B[0;31m \u001B[0mdf\u001B[0m \u001B[0;34m=\u001B[0m  \u001B[0mspark\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcreateDataFrame\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mpdf\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mschema\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mmySchema\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m     10\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/dbconnect/lib/python3.7/site-packages/pyspark/sql/session.py\u001B[0m in \u001B[0;36mcreateDataFrame\u001B[0;34m(self, data, schema, samplingRatio, verifySchema)\u001B[0m\n\u001B[1;32m    749\u001B[0m             \u001B[0mrequire_minimum_pandas_version\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    750\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 751\u001B[0;31m             \u001B[0;32mif\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_wrapped\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_conf\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mpandasRespectSessionTimeZone\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    752\u001B[0m                 \u001B[0mtimezone\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_wrapped\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_conf\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msessionLocalTimeZone\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    753\u001B[0m             \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/dbconnect/lib/python3.7/site-packages/py4j/java_gateway.py\u001B[0m in \u001B[0;36m__call__\u001B[0;34m(self, *args)\u001B[0m\n\u001B[1;32m   1255\u001B[0m         \u001B[0manswer\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mgateway_client\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msend_command\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mcommand\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1256\u001B[0m         return_value = get_return_value(\n\u001B[0;32m-> 1257\u001B[0;31m             answer, self.gateway_client, self.target_id, self.name)\n\u001B[0m\u001B[1;32m   1258\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   1259\u001B[0m         \u001B[0;32mfor\u001B[0m \u001B[0mtemp_arg\u001B[0m \u001B[0;32min\u001B[0m \u001B[0mtemp_args\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/dbconnect/lib/python3.7/site-packages/pyspark/sql/utils.py\u001B[0m in \u001B[0;36mdeco\u001B[0;34m(*a, **kw)\u001B[0m\n\u001B[1;32m     61\u001B[0m     \u001B[0;32mdef\u001B[0m \u001B[0mdeco\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0ma\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkw\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     62\u001B[0m         \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 63\u001B[0;31m             \u001B[0;32mreturn\u001B[0m \u001B[0mf\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0ma\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkw\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m     64\u001B[0m         \u001B[0;32mexcept\u001B[0m \u001B[0mpy4j\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mprotocol\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mPy4JJavaError\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0me\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     65\u001B[0m             \u001B[0ms\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0me\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mjava_exception\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtoString\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/anaconda3/envs/dbconnect/lib/python3.7/site-packages/py4j/protocol.py\u001B[0m in \u001B[0;36mget_return_value\u001B[0;34m(answer, gateway_client, target_id, name)\u001B[0m\n\u001B[1;32m    330\u001B[0m                 raise Py4JError(\n\u001B[1;32m    331\u001B[0m                     \u001B[0;34m\"An error occurred while calling {0}{1}{2}. Trace:\\n{3}\\n\"\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 332\u001B[0;31m                     format(target_id, \".\", name, value))\n\u001B[0m\u001B[1;32m    333\u001B[0m         \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    334\u001B[0m             raise Py4JError(\n",
      "\u001B[0;31mPy4JError\u001B[0m: An error occurred while calling o31.pandasRespectSessionTimeZone. Trace:\npy4j.Py4JException: Method pandasRespectSessionTimeZone([]) does not exist\n\tat py4j.reflection.ReflectionEngine.getMethod(ReflectionEngine.java:318)\n\tat py4j.reflection.ReflectionEngine.getMethod(ReflectionEngine.java:326)\n\tat py4j.Gateway.invoke(Gateway.java:274)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n\tat java.lang.Thread.run(Thread.java:748)\n\n"
     ]
    }
   ],
   "source": [
    "mySchema = StructType([StructField('date_created', StringType(), True)\\\n",
    "                       ,StructField('score', StringType(), True)\\\n",
    "                       ,StructField('sentence', StringType(), True)\\\n",
    "                       ,StructField('uuid', StringType(), True)\\\n",
    "                       ,StructField('variation', StringType(), True)\\\n",
    "                       \n",
    "                      ])\n",
    "\n",
    "df =  spark.createDataFrame(pdf, schema=mySchema)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "\n",
    "df.groupBy('score').count().orderBy('score').show()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "language": "python",
   "display_name": "Python 3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [
     "%% md\n",
     "\n",
     "\n",
     "##  Which words repeat?\n"
    ],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
