# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/scraper/Scraper.ipynb.

# %% ../../nbs/scraper/Scraper.ipynb 2
from __future__ import annotations

import os

from dataclasses import dataclass, field
import re

from typing import List, Callable, Any, Tuple

from functools import partial

from bs4 import BeautifulSoup

# from urllib.parse import urljoin, urlparse
from concurrent.futures import ThreadPoolExecutor, as_completed

# from domolibrary_extensions.utils import upsert_folder, convert_str_file_name

import domolibrary_extensions.scraper.DriverGenerator as desd
import domolibrary_extensions.scraper.routes as desr
import domolibrary_extensions.utils.utils as deut
import datetime as dt

from nbdev.showdoc import patch_to, show_doc

# %% auto 0
__all__ = ['ScrapeConfig', 'is_url_match_pattern', 'ScrapeTask_NoDriverProvided', 'ScrapeTask_NoBaseUrl',
           'ScrapeTask_NoScrapeCrawler', 'ScrapeTask', 'Scrape_Factory_NoConfigMatch', 'ScrapeFactory', 'ScrapeCrawler']

# %% ../../nbs/scraper/Scraper.ipynb 3
from domolibrary_extensions.utils.web import (
    extract_links,
    generate_filename_from_url,
    remove_query_params_from_url,
)

# %% ../../nbs/scraper/Scraper.ipynb 9
@dataclass
class ScrapeConfig:
    """class for collating data about how to scrape a page
    pass a list of scrape_config into ScrapeFactory
    """

    id: re.Pattern  # pattern for matching URLs to appropriate config

    search_criteria_tuple: Tuple = None  # from selenium.webdriver.common.by import By

    content_extractor_fn: Callable = None  # function for the subset of HTML to extract

    link_extractor_fn: Callable = partial(
        extract_links
    )  # function for extracting links from soup

    generate_filename_fn: Callable = (
        None  # fuction for generating filename given a url and dowload_folder
    )

    pagesource_fn: Callable = None

    def __id__(self, other):
        return self.id == other.id

# %% ../../nbs/scraper/Scraper.ipynb 13
def is_url_match_pattern(url, scrape_config_pattern: str):
    """test if a url matches a scrape_config_pattern"""

    pattern = re.compile(scrape_config_pattern)

    match_pattern = pattern.match(url)

    if not match_pattern:
        return False

    return True

# %% ../../nbs/scraper/Scraper.ipynb 15
class ScrapeTask_NoDriverProvided(Exception):
    def __init__(self):
        super().__init__("Driver not provided")


class ScrapeTask_NoBaseUrl(Exception):
    def __init__(self):
        super().__init__(
            "no base_url provided to scrape_task. must run as is_test = True"
        )


class ScrapeTask_NoScrapeCrawler(Exception):
    def __init__(self):
        super().__init__(
            "no scrape_crawler provided.  must run scrape_page as is_test = True"
        )


@dataclass
class ScrapeTask:
    """scrape_factory matches a URL to a scrape_config and returns a scrape_task"""

    url: str  # will be assigned after retrieved from scrape_factory

    base_url: str = None
    download_folder: str = "./SCRAPE"

    generate_filename_fn: Callable = None
    content_extractor_fn: Callable = None
    link_extractor_fn: Callable = None

    pagesource_fn: Callable = None

    max_sleep_time: int = 10
    scrape_config: ScrapeConfig = None

    scrape_config: ScrapeConfig = None
    scrape_crawler: ScrapeCrawler = None  # will be optional parent threadpool manager.

    pagesource: str = None

    soup: BeautifulSoup = None

    def __post_init__(self):
        self.generate_filename_fn = (
            self.generate_filename_fn
            or (self.scrape_config and self.scrape_config.generate_filename_fn)
            or partial(generate_filename_from_url, url=self.url, use_separator="/")
        )

        self.pagesource_fn = (
            self.pagesource_fn
            or (self.scrape_config and self.scrape_config.pagesource_fn)
            or partial(
                desr.get_pagesource,
                search_criteria_tuple=(
                    self.scrape_config and self.scrape_config.search_criteria_tuple
                ),
            )
        )

        self.content_extractor_fn = self.content_extractor_fn or (
            self.scrape_config and self.scrape_config.content_extractor_fn
        )

        self.link_extractor_fn = (
            self.link_extractor_fn
            or (self.scrape_config and self.scrape_config.link_extractor_fn)
            or extract_links
        )

    @classmethod
    def _from_factory(
        cls,
        url: str,
        base_url: str,
        scrape_config: ScrapeConfig,
        scrape_crawler=None,
        download_folder: str = "./SCRAPE",
        generate_filename_fn: Callable = None,
    ):
        return cls(
            url=url,
            base_url=base_url,
            download_folder=download_folder,
            scrape_config=scrape_config,
            scrape_crawler=scrape_crawler,
            generate_filename_fn=generate_filename_fn,
        )

    def _download_content(self, file_name, content):

        dir_name = os.path.dirname(file_name)

        if dir_name[-1] != "/":
            dir_name += "/"

        deut.upsert_folder(dir_name)

        with open(file_name, "w", encoding="utf-8") as f:
            f.write(str(content))

        return True

    def _update_parent(self):
        """if config is part of a scrape_crawler / managed threadpool will update crawler's urls_to_visit list."""

        scrape_crawler = self.scrape_crawler

        if not scrape_crawler:
            return False

        scrape_crawler._add_url_to_visited(self.url)

        if self.soup:
            urls_to_visit = self.link_extractor_fn(
                soup=self.soup, base_url=self.base_url
            )

            print(f"{len(urls_to_visit)} exracted from soup")

            [scrape_crawler._add_url_to_visit(url) for url in urls_to_visit]

            print(self.scrape_crawler.urls_to_visit, self.base_url)

        return urls_to_visit

# %% ../../nbs/scraper/Scraper.ipynb 16
@patch_to(ScrapeTask)
def execute(
    self: ScrapeTask,
    driver=None,
    debug_prn: bool = False,
    is_suppress_errors: bool = False,
    is_test: bool = False,
    download_folder: str = None,
):
    """handles executing the scrape_task
    1. get pagesource
    2. download index
    3. download content
    4. update crawler
    - get_links from pagesource
    """

    try:
        self.pagesource = self.pagesource_fn(
            driver=driver,
            url=self.url,
            max_sleep_time=self.max_sleep_time,
            return_entire_page=True,
            return_soup=False,
            debug_prn=debug_prn,
        )

        download_folder = download_folder or self.download_folder

        self.soup = BeautifulSoup(self.pagesource, "lxml")

        # download index
        file_name = self.generate_filename_fn(
            url=self.url, download_folder=download_folder, file_name="index.html"
        )

        self._download_content(file_name=file_name, content=self.soup)

        # download content
        if self.content_extractor_fn:
            content = self.content_extractor_fn(self.soup)
            content_name = "content.html"

            if isinstance(content, tuple):
                content_name = content[1]
                content = content[0]

            content_name = self.generate_filename_fn(
                download_folder=download_folder, file_name=content_name, url=self.url
            )

            self._download_content(
                file_name=content_name,
                content=content,
            )

        print(f"🎉 successfully scraped {self.url} to {file_name}")

        if is_test or not self.scrape_crawler:
            return True

        # update crawler
        # if not self.base_url:
        #     raise ScrapeTask_NoBaseUrl()

        # if not self.scrape_crawler:
        #     raise ScrapeTask_NoScrapeCrawler()

        """if config is part of a scrape_crawler / managed threadpool will update crawler's urls_to_visit list."""
        self._update_parent()

        return True

    except Exception as e:
        if not is_suppress_errors:
            raise (e)
        return f"💀 failed to download {self.url} received errror{e}"

# %% ../../nbs/scraper/Scraper.ipynb 20
class Scrape_Factory_NoConfigMatch(Exception):
    def __init__(self, text):
        super().__init__(
            f"{text} has no pattern match in factory_configs, add an appropriate config or check pattern matches"
        )


@dataclass
class ScrapeFactory:
    """class handles a list of Scrape_Configs and will return the 'correct one' given a URL"""

    factory_configs: List[ScrapeConfig]

    def generate_task_from_url(
        self,
        url,
        download_folder="./SCRAPE",
        base_url=None,
        scrape_crawler=None,
        generate_filename_fn=None,
    ):
        configs = []

        for config in self.factory_configs:
            if is_url_match_pattern(url=url, scrape_config_pattern=config.id):
                configs.append(config)

        if not configs:
            raise Scrape_Factory_NoConfigMatch(text=url)

        return [
            ScrapeTask._from_factory(
                scrape_config=config,
                url=url,
                base_url=base_url,
                download_folder=download_folder,
                scrape_crawler=scrape_crawler,
                generate_filename_fn=generate_filename_fn,
            )
            for config in configs
        ]

# %% ../../nbs/scraper/Scraper.ipynb 22
@dataclass
class ScrapeCrawler:
    """threadpool manager for crawling through a list of urls"""

    base_url: str
    download_folder: str
    visited_urls: set
    urls_to_visit: set

    executor: ThreadPoolExecutor = field(repr=False)
    scrape_factory: ScrapeFactory = field(repr=False)  # list of expected scrape_configs
    driver_generator: desd.DriverGenerator = field(repr=False)

    # def __post_init__(self):

    @classmethod
    def init(
        cls,
        urls_to_visit: List[str],
        base_url: str,
        scrape_factory: ScrapeFactory,
        driver_generator: desd.DriverGenerator = None,
        download_folder: str = "./SCRAPE/",
        driver_path=None,
        max_workers=5,
        urls_visited: List[str] = None,
    ):
        scrape_crawler = cls(
            executor=ThreadPoolExecutor(max_workers=max_workers),
            scrape_factory=scrape_factory,
            base_url=base_url,
            download_folder=download_folder,
            visited_urls=set(),
            urls_to_visit=set(),
            driver_generator=driver_generator
            or desd.DriverGenerator(driver_path=driver_path),
        )

        if urls_visited:
            [scrape_crawler._add_url_to_visited(url) for url in urls_visited]

        if urls_to_visit:
            [scrape_crawler._add_url_to_visit(url) for url in urls_to_visit]

        return scrape_crawler

    def _add_url_to_visit(self, url, debug_prn: bool = False):
        """adds a URL to the list of URLS to visit after testing that the URL has not already been visited"""

        if url not in self.visited_urls and url not in self.urls_to_visit:
            if debug_prn:
                print(f"adding {url} to to_vist list")

            self.urls_to_visit.add(url)
            return self.urls_to_visit

    def _add_url_to_visited(self, url, debug_prn: bool = False):
        if url not in self.visited_urls:
            if debug_prn:
                print(f"adding {url} to visited list")

            self.visited_urls.add(url)
            return self.visited_urls

    def _quit(self):
        """call when the executor queue is empty"""

        self.executor.shutdown(wait=True)
        return f"Done scraping {len(self.visited_urls)} urls"

    def crawl_urls(
        self: ScrapeCrawler,
        is_test: bool = False,
        debug_prn: bool = False,
        is_headless: bool = True,
    ):
        while self.urls_to_visit:
            url = self.urls_to_visit.pop()
            self.visited_urls.add(url)

            driver = self.driver_generator.get_webdriver(is_headless=is_headless)

            tasks = self.scrape_factory.generate_task_from_url(
                url,
                scrape_crawler=self,
                base_url=self.base_url,
                download_folder=self.download_folder,
            )

            future_to_task = {
                self.executor.submit(task.execute, driver=driver, is_test=is_test): task
                for task in tasks
            }
            for future in as_completed(future_to_task):

                try:
                    future.result()

                except Exception as e:
                    print(e)

        return self._quit()
