# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/scraper/ScrapeDomoSupport.ipynb.

# %% ../../nbs/scraper/ScrapeDomoSupport.ipynb 2
from __future__ import annotations
from functools import partial

from selenium.webdriver.common.keys import Keys

import json
from io import StringIO
import datetime as dt
import time

import pandas as pd
from bs4 import BeautifulSoup

import domolibrary_extensions.scraper.routes as desr
import domolibrary_extensions.scraper.Scraper as desc
import domolibrary_extensions.utils.web as deuw

# %% auto 0
__all__ = ['domo_support_link_extractor_fn', 'DomoSupport_ScrapeConfig_SupportAllTickets', 'DomoSupport_ScrapeConfig_Ticket',
           'DomoSupport_ScrapeConfig_TicketComments', 'DomoSupport_ScrapeFactory',
           'domo_support_all_tickets_pagesource', 'domo_support_all_tickets_extract_content',
           'domo_support_link_extractor_fn__custom', 'domo_support_generate_filename',
           'domo_support_ticket_extract_content', 'domo_support_ticket_comments_extract_content']

# %% ../../nbs/scraper/ScrapeDomoSupport.ipynb 3
from domolibrary_extensions.utils.web import (
    extract_links,
    generate_filename_from_url,
    remove_query_params_from_url,
)

# %% ../../nbs/scraper/ScrapeDomoSupport.ipynb 7
def domo_support_all_tickets_pagesource(
    driver: desr.webdriver,  # must pass an authenticated driver
    return_entire_page: bool = True,
    return_soup: bool = False,
    **kwargs,
) -> BeautifulSoup:
    url = f"https://domo-support.domo.com/s/"

    driver.get(url)

    time.sleep(2)

    # browser element
    page_source = desr.get_pagesource(
        driver=driver,
        search_criteria_tuple=(desr.By.CLASS_NAME, "cellContainer"),
        return_soup=False,
        return_entire_page=True,
    )

    new_len_table = 0
    prev_len_table = None

    while new_len_table != prev_len_table:
        scroll_bar = desr.wait_and_return(
            driver=driver,
            search_criteria_tuple=(desr.By.CLASS_NAME, "uiScroller"),
            return_soup=False,
            # min_sleep_time=15,
        )
        scroll_bar.send_keys(Keys.END)

        time.sleep(2)
        print(f"💤📜 sleeping on the scroll {new_len_table} 💤📜")

        soup = BeautifulSoup(page_source, parser="lxml")

        prev_len_table = new_len_table
        new_len_table = len(soup.find_all("td"))

    print(f"🚀 done retrieving table {new_len_table} 🚀")

    return desr.get_pagesource(
        driver=driver,
        search_criteria_tuple=(desr.By.TAG_NAME, "table"),
        return_soup=return_soup,
        return_entire_page=return_entire_page,
    )

# %% ../../nbs/scraper/ScrapeDomoSupport.ipynb 10
def domo_support_all_tickets_extract_content(
    soup,
    return_soup: bool = False,
    return_extractor: bool = False,
    base_url="https://domo-support.domo.com",
) -> pd.DataFrame:
    soup = soup.find("table")

    if return_soup:
        return soup

    df_with_links = pd.read_html(StringIO(str(soup)), extract_links="body")[0]

    df = pd.read_html(StringIO(str(soup)))[0]

    rename_cols = {
        col: col.replace("Column Actions", "")
        .replace("Sorted", "")
        .replace("Sort", "")
        .replace("Ascending", "")
        .replace("Descending", "")
        .strip()
        for col in df.columns
    }

    df.rename(columns=rename_cols, inplace=True)
    df_with_links.rename(columns=rename_cols, inplace=True)

    df_with_links = pd.DataFrame(
        df_with_links["Case Number"].tolist(),
        index=df_with_links.index,
        columns=["Case Number", "URL"],
    )

    df["Case Number"] = df["Case Number"].astype(int)
    df_with_links["Case Number"] = df_with_links["Case Number"].astype(int)

    df = pd.merge(df, df_with_links, on="Case Number", how="inner")

    df["URL"] = base_url + df["URL"]

    print(f"🚀 {len(df.index)} tickets retrieved")

    if return_extractor:
        return df.to_csv(), "content.csv"

    return df

# %% ../../nbs/scraper/ScrapeDomoSupport.ipynb 12
def domo_support_link_extractor_fn__custom(url, base_url, debug_prn: bool = False):
    url = deuw.remove_query_params_from_url(url)

    if debug_prn:
        print(url)

    if not url or "/s/case/" not in url:
        return None

    url = "/".join(url.split("/")[:6])

    if url.endswith("/"):
        url = url[:-1]

    return url


domo_support_link_extractor_fn = partial(
    extract_links,
    custom_link_extractor_fn=partial(
        domo_support_link_extractor_fn__custom, debug_prn=False
    ),
)


def domo_support_generate_filename(
    domo_instance,
    variant=None,
    url=None,
    file_name="index.html",
    download_folder="./SCRAPE",
    **kwargs,
):
    """must apply at scrape_crawler definition"""

    base = f"{download_folder}/{domo_instance}"
    if variant:
        return f"{base}/{variant}/{str(dt.date.today())}/{file_name}"

    return f"{base}/{generate_filename_from_url(url)}/{file_name}"

# %% ../../nbs/scraper/ScrapeDomoSupport.ipynb 14
DomoSupport_ScrapeConfig_SupportAllTickets = desc.ScrapeConfig(
    id=r".*/domo-support.domo.com/s/$",
    link_extractor_fn=domo_support_link_extractor_fn,
    content_extractor_fn=partial(
        domo_support_all_tickets_extract_content, return_extractor=True
    ),
    pagesource_fn=partial(
        domo_support_all_tickets_pagesource, return_entire_page=True, return_soup=False
    ),
    generate_filename_fn=partial(
        domo_support_generate_filename, variant="all_tickets", domo_instance="default"
    ),
)

# %% ../../nbs/scraper/ScrapeDomoSupport.ipynb 17
def domo_support_ticket_extract_content(
    soup,
    return_soup: bool = False,
    return_extractor: bool = False,
    base_url="https://domo-support.domo.com",
) -> pd.DataFrame:

    table_soup = soup.find("div", class_="test-id__record-layout-container")

    if return_soup:
        return table_soup

    obj = {
        row.find("div", class_="slds-form-element__label")
        .text.strip(): row.find("div", class_="slds-form-element__control")
        .text.strip()
        for row in table_soup.find_all("div", class_="slds-form-element")
    }

    if return_extractor:
        return (json.dumps(obj), "ticket_meta.json")

    return obj


DomoSupport_ScrapeConfig_Ticket = desc.ScrapeConfig(
    id=r".*/s/case/*",
    search_criteria_tuple=(desr.By.CLASS_NAME, "slds-form-element"),
    link_extractor_fn=domo_support_link_extractor_fn,
    content_extractor_fn=partial(
        domo_support_ticket_extract_content, return_extractor=True
    ),
    generate_filename_fn=partial(
        domo_support_generate_filename, domo_instance="default"
    ),
)

# %% ../../nbs/scraper/ScrapeDomoSupport.ipynb 20
def domo_support_ticket_comments_extract_content(
    soup,
    return_soup: bool = False,
    return_extractor: bool = False,
    base_url="https://domo-support.domo.com",
):

    # description_class = "test-id__field-value"
    soup = soup.find(class_="listContent")

    if return_soup:
        return soup

    casecomments_ls = []
    for li in soup.find("ul").find_all("li"):
        comment_hdr = li.find(class_="date").text
        comment_body = li.find(class_="commentBody").text

        casecomments_ls.append({"date": comment_hdr, "body": comment_body})

    if return_extractor:
        res = {"case_comments": casecomments_ls}
        return (json.dumps(res), "ticket_comments.json")

    return pd.DataFrame(casecomments_ls)


DomoSupport_ScrapeConfig_TicketComments = desc.ScrapeConfig(
    id=r".*/s/case/*",
    link_extractor_fn=domo_support_link_extractor_fn,
    search_criteria_tuple=(desr.By.CLASS_NAME, "commentBody"),
    content_extractor_fn=partial(
        domo_support_ticket_comments_extract_content, return_extractor=True
    ),
    generate_filename_fn=partial(
        domo_support_generate_filename, domo_instance="default"
    ),
)

# %% ../../nbs/scraper/ScrapeDomoSupport.ipynb 23
DomoSupport_ScrapeFactory = desc.ScrapeFactory(
    factory_configs=[
        DomoSupport_ScrapeConfig_SupportAllTickets,
        DomoSupport_ScrapeConfig_Ticket,
        DomoSupport_ScrapeConfig_TicketComments,
    ]
)
