{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "1eaaa772a41a4abda538965d76ddff87": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6aaeb75f42a947aa9178130cbb28e43a",
              "IPY_MODEL_15af62b24c7148e8b736d805bc800e62",
              "IPY_MODEL_51ef52743f5446dfabda362253b19b1f"
            ],
            "layout": "IPY_MODEL_aced084f506a4cd682e8da83bdfb1e68"
          }
        },
        "6aaeb75f42a947aa9178130cbb28e43a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0d378bc994a6482d8b13454565176f06",
            "placeholder": "​",
            "style": "IPY_MODEL_9804a7ed7233493cb2a14a04c5eabe31",
            "value": "Preprocessing: 100%"
          }
        },
        "15af62b24c7148e8b736d805bc800e62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4182cae4c4674beb967a92754eecfc34",
            "max": 723,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_488ba2172ee141569980eb6be70d225b",
            "value": 723
          }
        },
        "51ef52743f5446dfabda362253b19b1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3dc7773eeef04a30aa38711140f1d3c7",
            "placeholder": "​",
            "style": "IPY_MODEL_6e431a4e46f648688f0af665fa315ac3",
            "value": " 723/723 [00:17&lt;00:00, 68.52docs/s]"
          }
        },
        "aced084f506a4cd682e8da83bdfb1e68": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0d378bc994a6482d8b13454565176f06": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9804a7ed7233493cb2a14a04c5eabe31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4182cae4c4674beb967a92754eecfc34": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "488ba2172ee141569980eb6be70d225b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3dc7773eeef04a30aa38711140f1d3c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6e431a4e46f648688f0af665fa315ac3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "126b8af600354a48a721d26064c22ed8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d95e4f184c684e118ba47cf119d901c0",
              "IPY_MODEL_a888d53b771e47618995b2d7e1deb40d",
              "IPY_MODEL_4ae36ee20c5143eabb1b63bf993e68ca"
            ],
            "layout": "IPY_MODEL_427a3dcdea7645feb8f3326b2bcf16ee"
          }
        },
        "d95e4f184c684e118ba47cf119d901c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_89b6d8480fa14c19a16130fb36fdc924",
            "placeholder": "​",
            "style": "IPY_MODEL_3d577af888f84735a4add9068b549559",
            "value": "Preprocessing: 100%"
          }
        },
        "a888d53b771e47618995b2d7e1deb40d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f002f053acbb4f0488023618371a3fff",
            "max": 723,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2cd1a6ca97a243a5835815eab1d27d3c",
            "value": 723
          }
        },
        "4ae36ee20c5143eabb1b63bf993e68ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dcee6bd3ccab4abc9b7990a20e11d2f5",
            "placeholder": "​",
            "style": "IPY_MODEL_ef3ef7d5c4dd4240af3a0e1c29823047",
            "value": " 723/723 [00:07&lt;00:00, 85.73docs/s]"
          }
        },
        "427a3dcdea7645feb8f3326b2bcf16ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "89b6d8480fa14c19a16130fb36fdc924": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3d577af888f84735a4add9068b549559": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f002f053acbb4f0488023618371a3fff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2cd1a6ca97a243a5835815eab1d27d3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "dcee6bd3ccab4abc9b7990a20e11d2f5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ef3ef7d5c4dd4240af3a0e1c29823047": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png)\n",
        "\n",
        "<img src=\"https://haystack.deepset.ai/images/haystack-ogimage.png\" width=\"300\"/>            \n",
        "\n",
        "https://haystack.deepset.ai [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/johnsnowlabs/blob/master/notebooks/haystack_with_johnsnowlabs.ipynb)\n",
        "\n",
        "\n",
        "This tutorial showcase how to use [Johnsnowlabs Components with Langchain](https://nlp.johnsnowlabs.com/docs/en/jsl/haystack-utils) for Scalable Pre-Processing and Embedding computation on clusters\n",
        "\n",
        "If you want to scale this, you can re-use this code in a spark-cluster created with [nlp.install_to_databricks()](https://nlp.johnsnowlabs.com/docs/en/jsl/install_advanced#into-a-freshly-created-databricks-cluster-automatically)"
      ],
      "metadata": {
        "id": "LIUmU42lYLMA"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u3eDorlyQ0Sr"
      },
      "outputs": [],
      "source": [
        "! pip install johnsnowlabs\n",
        "from johnsnowlabs import nlp\n",
        "! pip install 'farm-haystack[all]'\n",
        "! pip install tensorflow==2.14\n",
        "nlp.start()\n",
        "\n",
        "# restart session after installing evertything\n",
        "import os\n",
        "os.kill(os.getpid(), 9)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Download some Sample Data and Convert do Haystack Documents"
      ],
      "metadata": {
        "id": "TlnJcqZAmrT3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Download some sample data we use as a mini-db\n",
        "! wget https://raw.githubusercontent.com/langchain-ai/langchain/master/docs/docs/modules/state_of_the_union.txt\n",
        "\n",
        "from haystack import Document\n",
        "def create_documents_from_file(file_path):\n",
        "    # Helper func for reading files as Haystack Document object\n",
        "    # returns a list of Document with one object for every line in file_path\n",
        "    documents = []\n",
        "    with open(file_path, 'r') as file:\n",
        "        for id, line in enumerate(file):\n",
        "            documents.append(Document(content=line.strip(), content_type=\"text\", id=id))\n",
        "    return documents\n"
      ],
      "metadata": {
        "id": "25uCldigJec4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1e8756f1-9567-43db-dc58-700cc103f182"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-11-17 03:16:53--  https://raw.githubusercontent.com/langchain-ai/langchain/master/docs/docs/modules/state_of_the_union.txt\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.110.133, 185.199.108.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 39028 (38K) [text/plain]\n",
            "Saving to: ‘state_of_the_union.txt.3’\n",
            "\n",
            "\rstate_of_the_union.   0%[                    ]       0  --.-KB/s               \rstate_of_the_union. 100%[===================>]  38.11K  --.-KB/s    in 0.01s   \n",
            "\n",
            "2023-11-17 03:16:53 (3.07 MB/s) - ‘state_of_the_union.txt.3’ saved [39028/39028]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Create a Haystack pipe\n",
        "We will add a `JohnSnowLabsHaystackProcessor` and `JohnSnowLabsHaystackEmbedder` for fully distributed computation on spark-clusters.\n",
        "\n",
        "In this simple example we split documents and store their embeddings in the document store for RAG applications"
      ],
      "metadata": {
        "id": "4KR1mozkmvBu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from johnsnowlabs.llm import embedding_retrieval\n",
        "from haystack.nodes import PreProcessor\n",
        "from haystack import Pipeline\n",
        "from haystack.document_stores import InMemoryDocumentStore\n",
        "\n",
        "def get_hay_jsl_pipe(model_name='en.embed_sentence.bert_base_uncased',embed_dim=512):\n",
        "    # small example Haystack pipeline demonstrating JohnSnowLabsHaystackProcessor\n",
        "\n",
        "\n",
        "    # JohnSnowLabsHaystackProcessor support all parameters of JSl DocumentSplitter\n",
        "    processor = embedding_retrieval.JohnSnowLabsHaystackProcessor(\n",
        "        chunk_overlap=2,\n",
        "        chunk_size=20,\n",
        "        explode_splits=True,\n",
        "        keep_seperators=True,\n",
        "        patterns_are_regex=False,\n",
        "        split_patterns=[\"\\n\\n\", \"\\n\", \" \", \"\"],\n",
        "        trim_whitespace=True,\n",
        "    )\n",
        "\n",
        "    # Write some processed data to Doc store, so we can retrieve it later\n",
        "    document_store = InMemoryDocumentStore(embedding_dim=embed_dim)\n",
        "    document_store.write_documents(processor.process(create_documents_from_file(\"state_of_the_union.txt\")))\n",
        "\n",
        "    # If you want to use GPU, make sure you ran nlp.start(hardware_target='gpu') !\n",
        "    retriever = embedding_retrieval.JohnSnowLabsHaystackEmbedder(\n",
        "        embedding_model=model_name,\n",
        "        document_store=document_store,\n",
        "        use_gpu=False,\n",
        "    )\n",
        "\n",
        "    document_store.update_embeddings(retriever)\n",
        "\n",
        "    pipe = Pipeline()\n",
        "    pipe.add_node(component=processor, name=\"Preprocess\", inputs=[\"Query\"])\n",
        "    pipe.add_node(component=retriever, name=\"Embed&Retrieve\", inputs=[\"Query\"])\n",
        "    return pipe\n",
        "\n"
      ],
      "metadata": {
        "id": "HAUSU4TuJp53"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Create & Query the pipe\n",
        "We will get the top K most similar results to our query.\n",
        "You can use [any Sentence Embedding](https://nlp.johnsnowlabs.com/models?task=Embeddings) from John Snow Labs by passing the **nlu_reference** of any Sentence Embedder.\n"
      ],
      "metadata": {
        "id": "mlIgbmMNnFMn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "use_pipe = get_hay_jsl_pipe('en.embed_sentence.use', embed_dim=512)\n",
        "result = use_pipe.run(query=\"Who is the first lady\")\n",
        "for r in result['documents']:\n",
        "  print(r.to_dict())"
      ],
      "metadata": {
        "id": "Au6ZftFXKD7V",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 379,
          "referenced_widgets": [
            "1eaaa772a41a4abda538965d76ddff87",
            "6aaeb75f42a947aa9178130cbb28e43a",
            "15af62b24c7148e8b736d805bc800e62",
            "51ef52743f5446dfabda362253b19b1f",
            "aced084f506a4cd682e8da83bdfb1e68",
            "0d378bc994a6482d8b13454565176f06",
            "9804a7ed7233493cb2a14a04c5eabe31",
            "4182cae4c4674beb967a92754eecfc34",
            "488ba2172ee141569980eb6be70d225b",
            "3dc7773eeef04a30aa38711140f1d3c7",
            "6e431a4e46f648688f0af665fa315ac3"
          ]
        },
        "outputId": "6cba71fe-b04c-46b1-d31e-2e8d40ba2c31"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Spark Session already created, some configs may not take.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Preprocessing:   0%|          | 0/723 [00:00<?, ?docs/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1eaaa772a41a4abda538965d76ddff87"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Spark Session already created, some configs may not take.\n",
            "Warning::Spark Session already created, some configs may not take.\n",
            "Warning::Spark Session already created, some configs may not take.\n",
            "tfhub_use download started this may take some time.\n",
            "Approximate size to download 923.7 MB\n",
            "[OK!]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rUpdating Embedding:   0%|          | 0/21 [00:00<?, ? docs/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Warning::Spark Session already created, some configs may not take.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Documents Processed: 10000 docs [00:08, 1159.41 docs/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'content': 'First Lady and', 'content_type': 'text', 'score': 0.5019346848675483, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '2'}\n",
            "{'content': 'Madam Speaker, Madam', 'content_type': 'text', 'score': 0.5013201007373176, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '0'}\n",
            "{'content': 'Vice President, our', 'content_type': 'text', 'score': 0.5011079601534563, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '1'}\n",
            "{'content': 'Members of Congress', 'content_type': 'text', 'score': 0.5010618177687746, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '4'}\n",
            "{'content': 'Republicans.', 'content_type': 'text', 'score': 0.5007644888755247, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '20'}\n",
            "{'content': 'Supreme Court. My', 'content_type': 'text', 'score': 0.500718500236837, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '7'}\n",
            "{'content': 'Second Gentleman.', 'content_type': 'text', 'score': 0.5007164915232075, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '3'}\n",
            "{'content': 'and the Cabinet.', 'content_type': 'text', 'score': 0.500696245781778, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '5'}\n",
            "{'content': 'fellow Americans.', 'content_type': 'text', 'score': 0.5006815561148572, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '8'}\n",
            "{'content': 'Justices of the', 'content_type': 'text', 'score': 0.5006488399151164, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '6'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "bert_pipe = get_hay_jsl_pipe('en.embed_sentence.bert_base_uncased', embed_dim=768)\n",
        "result = bert_pipe.run(query=\"Who is the first lady\")\n",
        "for r in result['documents']:\n",
        "  print(r.to_dict())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 379,
          "referenced_widgets": [
            "126b8af600354a48a721d26064c22ed8",
            "d95e4f184c684e118ba47cf119d901c0",
            "a888d53b771e47618995b2d7e1deb40d",
            "4ae36ee20c5143eabb1b63bf993e68ca",
            "427a3dcdea7645feb8f3326b2bcf16ee",
            "89b6d8480fa14c19a16130fb36fdc924",
            "3d577af888f84735a4add9068b549559",
            "f002f053acbb4f0488023618371a3fff",
            "2cd1a6ca97a243a5835815eab1d27d3c",
            "dcee6bd3ccab4abc9b7990a20e11d2f5",
            "ef3ef7d5c4dd4240af3a0e1c29823047"
          ]
        },
        "id": "vN9LMKh0FAxI",
        "outputId": "a620b0e5-d3a8-4164-d1b1-670e1a2eb9fa"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Spark Session already created, some configs may not take.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Preprocessing:   0%|          | 0/723 [00:00<?, ?docs/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "126b8af600354a48a721d26064c22ed8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Spark Session already created, some configs may not take.\n",
            "Warning::Spark Session already created, some configs may not take.\n",
            "Warning::Spark Session already created, some configs may not take.\n",
            "sent_bert_base_uncased download started this may take some time.\n",
            "Approximate size to download 392.5 MB\n",
            "[OK!]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rUpdating Embedding:   0%|          | 0/21 [00:00<?, ? docs/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Warning::Spark Session already created, some configs may not take.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Documents Processed: 10000 docs [00:02, 3443.47 docs/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'content': 'First Lady and', 'content_type': 'text', 'score': 0.6348900483227143, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '2'}\n",
            "{'content': 'Vice President, our', 'content_type': 'text', 'score': 0.6224868949875735, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '1'}\n",
            "{'content': 'Second Gentleman.', 'content_type': 'text', 'score': 0.6224488513925066, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '3'}\n",
            "{'content': 'Madam Speaker, Madam', 'content_type': 'text', 'score': 0.621548252770467, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '0'}\n",
            "{'content': 'Supreme Court. My', 'content_type': 'text', 'score': 0.6192054605036265, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '7'}\n",
            "{'content': 'and the Cabinet.', 'content_type': 'text', 'score': 0.6167082973209767, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '5'}\n",
            "{'content': 'Zealand, and many', 'content_type': 'text', 'score': 0.6108719356163572, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '11'}\n",
            "{'content': 'to pay at the gas', 'content_type': 'text', 'score': 0.6108165188987911, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '18'}\n",
            "{'content': 'pump again.', 'content_type': 'text', 'score': 0.6062867537668348, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '19'}\n",
            "{'content': 'Justices of the', 'content_type': 'text', 'score': 0.6051581624724295, 'meta': {}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '6'}\n"
          ]
        }
      ]
    }
  ]
}