{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "51adebf4-7aba-4a03-b50b-36c2ec974122",
     "showTitle": false,
     "title": ""
    },
    "id": "7STETVppB3Np"
   },
   "source": [
    "![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png)\n",
    "\n",
    "# Automatic Databricks Cluster Creation with John Snow Labs libraries pre-Iinstalled\n",
    "For more details see https://nlp.johnsnowlabs.com/docs/en/jsl/install_advanced#via-json-secrets-file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "d93776cb-0b0a-4430-8d48-5cbb5a5c540a",
     "showTitle": false,
     "title": ""
    },
    "id": "F-6THc9uB3Nw"
   },
   "outputs": [],
   "source": [
    "%pip install -U johnsnowlabs_for_databricks\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "7015f047-b770-44d5-a50f-5dd7d822078d",
     "showTitle": false,
     "title": ""
    },
    "id": "902YNd9iB3Nz"
   },
   "outputs": [],
   "source": [
    "# from johnsnowlabs.auto_install.jsl_home import get_install_suite_from_jsl_home\n",
    "from johnsnowlabs import nlp\n",
    "import json,os\n",
    "\n",
    "cluster_license_string = \"\"\"\n",
    "{\n",
    "    \"SPARK_NLP_LICENSE\": \"\",\n",
    "    \n",
    "    \"SECRET\":\"\",\n",
    "    \"JSL_VERSION\":\"\",\n",
    "    \"PUBLIC_VERSION\":\"\",\n",
    "\n",
    "    \"AWS_ACCESS_KEY_ID\": \"\",\n",
    "    \"AWS_SECRET_ACCESS_KEY\": \"\",\n",
    "    \"AWS_SESSION_TOKEN\": \"\"\n",
    "}\n",
    "\"\"\"\n",
    "if os.path.exists('license.json'): os.remove('license.json')\n",
    "with open('license.json', 'w') as file:\n",
    "    file.write(cluster_license_string)\n",
    "\n",
    "# 2) Setup Databricks Access Token & Host\n",
    "db_token=''\n",
    "db_host=\"\"\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "a8219a9d-8bff-4596-85ec-6477ac599ad8",
     "showTitle": false,
     "title": ""
    },
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0hLEfIbBB3N0",
    "outputId": "8465b13b-15df-49a7-c1bc-cacc37ad7d74"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "📋 Stored John Snow Labs License in /root/.johnsnowlabs/licenses/license_number_0_for_Spark-Healthcare_Spark-OCR.json\n",
      "👷 Setting up  John Snow Labs home in /root/.johnsnowlabs, this might take a few minutes.\n",
      "Downloading 🐍+🚀 Python Library spark_nlp-5.0.2-py2.py3-none-any.whl\n",
      "Downloading 🐍+💊 Python Library spark_nlp_jsl-5.0.2-py3-none-any.whl\n",
      "Downloading 🫘+🚀 Java Library spark-nlp-assembly-5.0.2.jar\n",
      "Downloading 🫘+💊 Java Library spark-nlp-jsl-5.0.2.jar\n",
      "🙆 JSL Home setup in /root/.johnsnowlabs\n",
      "👌 Detected license file /content/license.json\n",
      "👌 Created cluster with id=0909-204233-aj256xun on host=XXXXXXXXXXXXXX/api/\n",
      "Copying /root/.johnsnowlabs/py_installs/spark_nlp_jsl-5.0.2-py3-none-any.whl to remote cluster path dbfs:/johnsnowlabs/py_installs/spark_nlp_jsl-5-py2.py3-none-any.whl\n",
      "Installed 💊Spark-Healthcare Spark NLP for Healthcare ✅\n",
      "Installed nlu ✅\n",
      "Installed spark-nlp-display ✅\n",
      "Installed johnsnowlabs_for_databricks ✅\n",
      "Copying /root/.johnsnowlabs/py_installs/spark_nlp-5.0.2-py2.py3-none-any.whl to remote cluster path dbfs:/johnsnowlabs/py_installs/spark_nlp-5-py2.py3-none-any.whl\n",
      "🚀Spark-NLP Installed Spark NLP! ✅\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=PENDING\n",
      "Cluster-Id=0909-204233-aj256xun not ready, status=RUNNING\n",
      "👌 Cluster-Id 0909-204233-aj256xun is ready!\n"
     ]
    }
   ],
   "source": [
    "runtime = '9.1.x-scala2.12'\n",
    "\n",
    "# For azure you can use 'Standard_DS3_v2'\n",
    "instance_type =  'Standard_DS3_v2'  # Azure\n",
    "#instance_type =  'i3.xlarge'\n",
    "\n",
    "test_cluster_id = nlp.install_to_databricks(\n",
    "    json_license_path='license.json',\n",
    "    spark_version=runtime,\n",
    "    databricks_host=db_host,\n",
    "    databricks_token=db_token,\n",
    "    visual=False,\n",
    "    # spark_env_vars=spark_env_vars,\n",
    "    # clean_cluster=False,\n",
    "    cluster_name=\"mlflow_test\",\n",
    "    node_type_id=instance_type,\n",
    "    driver_node_type_id=instance_type,\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "application/vnd.databricks.v1+notebook": {
   "dashboards": [],
   "language": "python",
   "notebookMetadata": {
    "pythonIndentUnit": 4
   },
   "notebookName": "create-cluster_PUBLIC",
   "widgets": {}
  },
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
