{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "af9e2356",
   "metadata": {},
   "source": [
    "# Email Notifier\n",
    "\n",
    "If you are using Seeq's SaaS product, instead of this Data Lab-based notification mechanism, it is recommended that you utilize the built in notification capabilities in Seeq R60+ as described in the following Seeq Knowledge Base article: https://telemetry.seeq.com/support-link/kb/latest/cloud/notifications-on-conditions\n",
    "\n",
    "This Notebook is intended to be scheduled from `Email Notification Scheduler.ipynb`. Before using this Notifier, you will need to configure the connection to the email server. Upon updating the following cell with appropriate credentials, uncomment the line\n",
    "\n",
    "`# test_email()`\n",
    "\n",
    "so that it reads\n",
    "\n",
    "`test_email()`\n",
    "\n",
    "If you see `Success!` in the text that appears below the cell after running it, restore the comment character `#` at the start of the line, click the Save (💾) icon in the toolbar above, and you're ready to go!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ee6ff40",
   "metadata": {},
   "outputs": [],
   "source": [
    "# For sample SMTP setup, see https://support.google.com/mail/answer/7126229\n",
    "smtp_configuration = {\n",
    "    'SMTP Host': 'smtp.mycompany.com',\n",
    "    'SMTP Port': 587,\n",
    "    'Email Address': 'email.sender@mycompany.com',\n",
    "    'Email Password': 'app.password.here'\n",
    "}\n",
    "\n",
    "\n",
    "def test_email():\n",
    "    import smtplib\n",
    "    smtp_host = smtp_configuration['SMTP Host']\n",
    "    smtp_port = smtp_configuration['SMTP Port']\n",
    "    smtp_username = smtp_configuration['Email Address']\n",
    "    smtp_password = smtp_configuration['Email Password']\n",
    "\n",
    "    with smtplib.SMTP(smtp_host, smtp_port) as smtp_client:\n",
    "        smtp_client.starttls()\n",
    "        smtp_client.set_debuglevel(1)\n",
    "        smtp_client.login(smtp_username, smtp_password)\n",
    "        print('Success!')\n",
    "\n",
    "# test_email()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e63f741",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import pathlib\n",
    "import re\n",
    "import smtplib\n",
    "from datetime import datetime, timedelta\n",
    "from email.message import EmailMessage\n",
    "from email.mime.application import MIMEApplication\n",
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd\n",
    "import pytz\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "from seeq import spy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58257d4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "current_job = spy.jobs.pull()\n",
    "current_job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a324e78a",
   "metadata": {},
   "outputs": [],
   "source": [
    "SENT_CAPSULES_PICKLE_SUBFOLDER = Path('_Sent Capsules')\n",
    "SENT_CAPSULES_PICKLE_NAME = Path(\n",
    "    f'sent_capsules_for_condition_{current_job[\"Condition ID\"]}_in_analysis_{current_job[\"Workbook ID\"]}.pkl'\n",
    ")\n",
    "SENT_CAPSULES_PICKLE_PATH = SENT_CAPSULES_PICKLE_SUBFOLDER / SENT_CAPSULES_PICKLE_NAME\n",
    "\n",
    "if not os.path.exists(SENT_CAPSULES_PICKLE_SUBFOLDER):\n",
    "    os.mkdir(SENT_CAPSULES_PICKLE_SUBFOLDER)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4d06985",
   "metadata": {},
   "source": [
    "### Capsule Handlers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eba73ab7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_sent_capsules(sent_capsules_df, newly_sent_capsules_df, polling_range_start):\n",
    "    if sent_capsules_df.empty:\n",
    "        return newly_sent_capsules_df\n",
    "    if newly_sent_capsules_df.empty:\n",
    "        return sent_capsules_df\n",
    "    updated_df = sent_capsules_df.append(newly_sent_capsules_df, ignore_index=True).drop_duplicates('Capsule Start')\n",
    "    return updated_df[updated_df['Capsule Start'] > polling_range_start].sort_values(by='Capsule Start')\n",
    "\n",
    "\n",
    "def get_unsent_capsules(sent_capsules_df, pulled_capsules_df):\n",
    "    if sent_capsules_df.empty or pulled_capsules_df.empty:\n",
    "        return pulled_capsules_df\n",
    "    return pulled_capsules_df[~pulled_capsules_df['Capsule Start'].isin(sent_capsules_df['Capsule Start'])]\n",
    "\n",
    "\n",
    "def store_sent_capsules(capsules_df, to_file_path=SENT_CAPSULES_PICKLE_PATH):\n",
    "    capsules_df.to_pickle(to_file_path)\n",
    "\n",
    "\n",
    "def retrieve_sent_capsules(from_file_path=SENT_CAPSULES_PICKLE_PATH):\n",
    "    if os.path.exists(from_file_path):\n",
    "        return pd.read_pickle(from_file_path)\n",
    "    else:\n",
    "        return pd.DataFrame()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e342527",
   "metadata": {},
   "source": [
    "### Email Builder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "279066c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The template should have substitution fields like {capsule[\"Some Property\"]}, which will be replaced by the associated\n",
    "# property if possible\n",
    "def fill_in_template(template, capsule, job):\n",
    "    import re\n",
    "    capsule_substitutions = set(re.findall(r'({capsule\\[\\\"(.*?)\\\"\\]})', template))\n",
    "    job_substitutions = set(re.findall(r'({job\\[\\\"(.*?)\\\"\\]})', template))\n",
    "    for to_replace, prop in capsule_substitutions:\n",
    "        replacement = str(capsule[prop]) if prop in capsule else '{Capsule property not found}'\n",
    "        if prop in ['Capsule Start', 'Capsule End']:\n",
    "            replacement = capsule[prop].astimezone(pytz.timezone(job['Time Zone'])).isoformat()\n",
    "        template = template.replace(to_replace, replacement)\n",
    "    for to_replace, prop in job_substitutions:\n",
    "        replacement = str(job[prop]) if prop in job else '{Job property not found}'\n",
    "        template = template.replace(to_replace, replacement)\n",
    "    return template\n",
    "\n",
    "\n",
    "def get_attachment_for_topic_document_pdf(topic_document_url):\n",
    "    base_url = spy.client.host[:-4]\n",
    "    screenshots_url = f'{base_url}/screenshots'\n",
    "    match = re.match(r'.*/workbook/(.*?)/worksheet/(.*)', topic_document_url)\n",
    "    workbook_id = match.group(1)\n",
    "    worksheet_id = match.group(2)\n",
    "    presentation_url = f'{base_url}/present/worksheet/{workbook_id}/{worksheet_id}'\n",
    "    pdf_specs = {\n",
    "        \"url\": presentation_url,\n",
    "        \"format\": \"PDF\",\n",
    "        \"orientation\": \"Portrait\",\n",
    "        \"paperSize\": \"Letter\",\n",
    "        \"margin\": \"0.5in\",\n",
    "        \"cancellationGroup\": f\"PDF Export {presentation_url}\"\n",
    "    }\n",
    "    cookies = {\n",
    "        'sq-auth': spy.client.auth_token\n",
    "    }\n",
    "    headers = {\n",
    "        'Content-Type': 'application/json',\n",
    "        'x-sq-csrf': spy.client.csrf_token\n",
    "    }\n",
    "    try:\n",
    "        response = requests.post(screenshots_url, data=json.dumps(pdf_specs), cookies=cookies, headers=headers)\n",
    "        pdf_url = f'{base_url}{response.headers[\"Location\"]}'\n",
    "        pdf_response = requests.get(pdf_url, cookies=cookies, headers=headers)\n",
    "        filename = pathlib.PurePosixPath(pdf_url).name\n",
    "        attachment = MIMEApplication(\n",
    "            pdf_response.content,\n",
    "            Name=filename\n",
    "        )\n",
    "        attachment['Content-Disposition'] = f'attachment; filename=\"{filename}\"'\n",
    "        return attachment\n",
    "    except Exception as ex:\n",
    "        print(ex)\n",
    "        return None\n",
    "\n",
    "\n",
    "def attach_file_to_msg(filename, msg):\n",
    "    from os.path import basename\n",
    "    with open(filename, \"rb\") as the_file:\n",
    "        attachment = MIMEApplication(\n",
    "            the_file.read(),\n",
    "            Name=basename(filename)\n",
    "        )\n",
    "    attachment['Content-Disposition'] = f'attachment; filename=\"{basename(filename)}\"'\n",
    "    msg.attach(attachment)\n",
    "\n",
    "\n",
    "def add_inline_image_with_content_id_to_msg(path_to_image, content_id, msg):\n",
    "    with open(path_to_image, \"rb\") as img_file:\n",
    "        inline_img = MIMEApplication(\n",
    "            img_file.read(),\n",
    "            Name=pathlib.Path(path_to_image).name\n",
    "        )\n",
    "    inline_img['Content-Disposition'] = 'inline'\n",
    "    inline_img['Content-ID'] = content_id\n",
    "    msg.attach(inline_img)\n",
    "\n",
    "\n",
    "def build_email(job, capsule):\n",
    "    msg = EmailMessage()\n",
    "    msg.make_mixed()\n",
    "\n",
    "    alternative_msg = EmailMessage()\n",
    "    alternative_msg.make_alternative()\n",
    "\n",
    "    related_msg = EmailMessage()\n",
    "    related_msg.make_related()\n",
    "\n",
    "    html_content = fill_in_template(job['Html Template'], capsule, job)\n",
    "\n",
    "    soup = BeautifulSoup(html_content, 'html.parser')\n",
    "    text_content = ' '.join([text for text in soup.find_all(text=True)])\n",
    "    text_msg = EmailMessage()\n",
    "    text_msg.set_content(text_content)\n",
    "    alternative_msg.attach(text_msg)\n",
    "\n",
    "    html_msg = EmailMessage()\n",
    "    html_msg.set_content(html_content, subtype='html')\n",
    "\n",
    "    related_msg.attach(html_msg)\n",
    "    add_inline_image_with_content_id_to_msg('./Seeq Data Lab.jpg', 'sdl', related_msg)\n",
    "\n",
    "    alternative_msg.attach(related_msg)\n",
    "    msg.attach(alternative_msg)\n",
    "\n",
    "    if job['Topic Document URL']:\n",
    "        attachment = get_attachment_for_topic_document_pdf(job['Topic Document URL'])\n",
    "        if attachment:\n",
    "            msg.attach(attachment)\n",
    "\n",
    "    msg['Subject'] = fill_in_template(job['Subject Template'], capsule, job)\n",
    "    msg['From'] = smtp_configuration['Email Address']\n",
    "    msg['To'] = job['To']\n",
    "    msg['Cc'] = job['Cc']\n",
    "    msg['Bcc'] = job['Bcc']\n",
    "\n",
    "    # Here one could add support for inserting inline content in the template based on a dictionary of content IDs\n",
    "    # and associated file paths specified by job_details['Inline Content'].  This would be intended for static\n",
    "    # content that the admin/configurer of the Notifications would set up in advance, probably setting the\n",
    "    # default value in the form for the Add-on Tool.\n",
    "\n",
    "    return msg\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50f8b227",
   "metadata": {},
   "outputs": [],
   "source": [
    "def send_emails(job, unsent_capsules):\n",
    "    smtp_host = smtp_configuration['SMTP Host']\n",
    "    smtp_port = smtp_configuration['SMTP Port']\n",
    "    smtp_username = smtp_configuration['Email Address']\n",
    "    smtp_password = smtp_configuration['Email Password']\n",
    "\n",
    "    messages_to_send = list()\n",
    "    sent_capsules = list()\n",
    "    exceptions = list()\n",
    "    for _, capsule in unsent_capsules.iterrows():\n",
    "        messages_to_send.append((capsule, build_email(job, capsule)))\n",
    "    with smtplib.SMTP(smtp_host, smtp_port) as smtp_client:\n",
    "        smtp_client.starttls()\n",
    "        smtp_client.set_debuglevel(1)\n",
    "        smtp_client.login(smtp_username, smtp_password)\n",
    "        for capsule, message in messages_to_send:\n",
    "            try:\n",
    "                smtp_client.send_message(message)\n",
    "                sent_capsules.append(capsule)\n",
    "            except Exception as ex:\n",
    "                exceptions.append((capsule, ex))\n",
    "        smtp_client.quit()\n",
    "    return (sent_capsules, exceptions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1dad2c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "condition = spy.search({'ID': current_job['Condition ID']})  # ID is used to ensure only one Condition in results\n",
    "sent_capsules = retrieve_sent_capsules()\n",
    "\n",
    "lookback_microseconds = int(24 * 60 * 60 * 1000 * 1000 * float(current_job['Lookback Interval']))\n",
    "polling_range_end = datetime.now(pytz.utc)\n",
    "polling_range_start = (polling_range_end - timedelta(microseconds=lookback_microseconds))\n",
    "inception = pd.Timestamp(current_job['Inception'])\n",
    "if inception > polling_range_start:\n",
    "    polling_range_start = inception\n",
    "\n",
    "capsules_starting_in_lookback_interval = spy.pull(condition, start=polling_range_start, end=polling_range_end,\n",
    "                                                  tz_convert='UTC')\n",
    "if capsules_starting_in_lookback_interval.empty or 'Capsule Start' not in capsules_starting_in_lookback_interval:\n",
    "    capsules_starting_in_lookback_interval = pd.DataFrame()\n",
    "else:\n",
    "    capsules_starting_in_lookback_interval = capsules_starting_in_lookback_interval[\n",
    "        capsules_starting_in_lookback_interval['Capsule Start'] > polling_range_start\n",
    "        ]\n",
    "capsules_starting_in_lookback_interval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b0e43af",
   "metadata": {},
   "outputs": [],
   "source": [
    "unsent_capsules = get_unsent_capsules(retrieve_sent_capsules(), capsules_starting_in_lookback_interval)\n",
    "unsent_capsules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cebdb096",
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    emailed_capsules, exceptions = send_emails(current_job, unsent_capsules)\n",
    "    print(f'Emails were sent successfully for {len(emailed_capsules)} capsules:')\n",
    "    print(emailed_capsules)\n",
    "    print(f'Send failed for {len(exceptions)} capsules:')\n",
    "    print(exceptions)\n",
    "except Exception as ex:\n",
    "    emailed_capsules, exceptions = ([], [])\n",
    "    print(f'Something went wrong sending emails: {ex}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cebdb097",
   "outputs": [],
   "source": [
    "if emailed_capsules:\n",
    "    start_list = [capsule['Capsule Start'] for capsule in emailed_capsules]\n",
    "    newly_sent_capsules = capsules_starting_in_lookback_interval[\n",
    "        capsules_starting_in_lookback_interval['Capsule Start'].isin(start_list)\n",
    "    ]\n",
    "    sent_capsules_updated = update_sent_capsules(sent_capsules, newly_sent_capsules, polling_range_start)\n",
    "    store_sent_capsules(sent_capsules_updated)"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a340437",
   "metadata": {},
   "outputs": [],
   "source": [
    "if emailed_capsules:\n",
    "    start_list = [capsule['Capsule Start'] for capsule in emailed_capsules]\n",
    "    newly_sent_capsules = capsules_starting_in_lookback_interval[\n",
    "        capsules_starting_in_lookback_interval['Capsule Start'].isin(start_list)\n",
    "    ]\n",
    "    sent_capsules_updated = update_sent_capsules(sent_capsules, newly_sent_capsules, polling_range_start)\n",
    "    store_sent_capsules(sent_capsules_updated)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.11",
   "language": "python",
   "name": "python311"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
