{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "cellMetadata": {
            "byteLimit": 2048000,
            "rowLimit": 10000
          },
          "inputWidgets": {},
          "nuid": "51adebf4-7aba-4a03-b50b-36c2ec974122",
          "showTitle": false,
          "title": ""
        },
        "id": "D6cn8o8xWMuC"
      },
      "source": [
        "![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png)\n",
        "\n",
        "# Automatic AWS Glue Setup with John Snow Labs libraries\n",
        "For more details see https://nlp.johnsnowlabs.com/docs/en/jsl/install_advanced#via-json-secrets-file"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "%pip install -U johnsnowlabs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bbF0E4tqWjh2"
      },
      "source": [
        "### Setup AWS credentials in environment variables or create a boto3 session\n",
        "\n",
        "Refer these for understanding boto3 sessions.\n",
        "1. https://boto3.amazonaws.com/v1/documentation/api/latest/guide/session.html\n",
        "2. https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "VVOepP7CWMuG"
      },
      "outputs": [],
      "source": [
        "# Ask your AWS admin for the AWS access keys if you don't have them\n",
        "\n",
        "import boto3\n",
        "# Creating a boto session. You can also use a profile name instead of the access key and secret key\n",
        "boto_session = boto3.Session(\n",
        "    aws_access_key_id=\"Your AWS Access Key\",\n",
        "    aws_secret_access_key=\"Your AWS Secret Key\",\n",
        "    # aws_session_token=\"Your AWS Session Tokenk\",\n",
        "    region_name=\"Your AWS Region\",\n",
        "    #profile_name=\"Your AWS Profile Name\"\n",
        ")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "cellMetadata": {
            "byteLimit": 2048000,
            "rowLimit": 10000
          },
          "inputWidgets": {},
          "nuid": "7015f047-b770-44d5-a50f-5dd7d822078d",
          "showTitle": false,
          "title": ""
        },
        "id": "l_Gp2WoNWMuH"
      },
      "outputs": [],
      "source": [
        "from johnsnowlabs import nlp\n",
        "import os\n",
        "\n",
        "cluster_license_string = \"\"\" \n",
        "Your John Snow Labs Floating JSON LIcense\n",
        "\"\"\"\n",
        "\n",
        "if os.path.exists('license.json'): os.remove('license.json')\n",
        "with open('license.json', 'w') as file:\n",
        "    file.write(cluster_license_string)\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "cellMetadata": {
            "byteLimit": 2048000,
            "rowLimit": 10000
          },
          "inputWidgets": {},
          "nuid": "a8219a9d-8bff-4596-85ec-6477ac599ad8",
          "showTitle": false,
          "title": ""
        },
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IySo3GM0WMuH",
        "outputId": "b25a7075-c632-47f0-bedc-01ad235ed09a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "✅ Successfully uploaded to s3://aws-glue-assets-189352970232-us-east-1/assets/jars/spark-nlp-assembly-5.0.2.jar\n",
            "👷 Uploading spark_nlp-5.0.2-py2.py3-none-any.whl ...\n",
            "✅ Successfully uploaded to s3://aws-glue-assets-189352970232-us-east-1/assets/packages/spark_nlp-5.0.2-py2.py3-none-any.whl\n",
            "👷 Uploading spark-nlp-jsl-5.0.2.jar ...\n",
            "✅ Successfully uploaded to s3://aws-glue-assets-189352970232-us-east-1/assets/jars/spark-nlp-jsl-5.0.2.jar\n",
            "👷 Uploading spark_nlp_jsl-5.0.2-py3-none-any.whl ...\n",
            "✅ Successfully uploaded to s3://aws-glue-assets-189352970232-us-east-1/assets/packages/spark_nlp_jsl-5.0.2-py3-none-any.whl\n",
            "\n",
            "\n",
            "Add the following lines in the beginning of your Glue notebook job:\n",
            "\n",
            "1. \n",
            "%additional_python_modules johnsnowlabs_by_kshitiz==5.0.7rc2,s3://aws-glue-assets-189352970232-us-east-1/assets/packages/spark_nlp-5.0.2-py2.py3-none-any.whl,s3://aws-glue-assets-189352970232-us-east-1/assets/packages/spark_nlp_jsl-5.0.2-py3-none-any.whl\n",
            "\n",
            "%extra_jars s3://aws-glue-assets-189352970232-us-east-1/assets/jars/spark-nlp-assembly-5.0.2.jar,s3://aws-glue-assets-189352970232-us-east-1/assets/jars/spark-nlp-jsl-5.0.2.jar\n",
            "\n",
            "%%configure \n",
            "{\n",
            "    \"--conf\": \"\"\"spark.jsl.settings.pretrained.cache_folder=s3://aws-glue-assets-189352970232-us-east-1/cache_pretrained/  \n",
            "--conf spark.hadoop.fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider \n",
            "--conf spark.jars.packages=org.apache.hadoop:hadoop-aws:3.2.1,com.amazonaws:aws-java-sdk:1.11.828 \n",
            "--conf spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem \n",
            "--conf spark.hadoop.fs.s3a.path.style.access=true \n",
            "--conf spark.jsl.settings.aws.region=us-east-1\"\"\"\n",
            "}\n",
            "\n",
            "2. \n",
            "%glue_version 4.0\n",
            "%number_of_workers 2 \n",
            "\n",
            "import sys\n",
            "from awsglue.transforms import *\n",
            "from awsglue.utils import getResolvedOptions\n",
            "from pyspark.context import SparkContext\n",
            "from awsglue.context import GlueContext\n",
            "from awsglue.job import Job\n",
            "\n",
            "sc = SparkContext.getOrCreate()\n",
            "glueContext = GlueContext(sc)\n",
            "spark = glueContext.spark_session\n",
            "spark._jvm.com.johnsnowlabs.util.start.registerListenerAndStartRefresh()\n",
            "job = Job(glueContext)\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Uploads jars and packages to s3 bucket\n",
        "# Prints out command to be used in AWS Glue Notebook\n",
        "# Refer this link for more details: https://nlp.johnsnowlabs.com/docs/en/jsl/aws-glue-utils\n",
        "\n",
        "nlp.install_to_glue(boto_session=boto_session, json_license_path=\"license.json\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "application/vnd.databricks.v1+cell": {
          "cellMetadata": {},
          "inputWidgets": {},
          "nuid": "e9460868-f659-498b-9523-ceb9d2505234",
          "showTitle": false,
          "title": ""
        },
        "id": "BiuvYaquWMuI"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "application/vnd.databricks.v1+notebook": {
      "dashboards": [],
      "language": "python",
      "notebookMetadata": {
        "pythonIndentUnit": 4
      },
      "notebookName": "create-cluster_PUBLIC",
      "widgets": {}
    },
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
