# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/scraper/Scraper.ipynb.

# %% ../../nbs/scraper/Scraper.ipynb 2
from __future__ import annotations

import os
import re

from dataclasses import dataclass, field
from typing import List, Callable, Tuple, Any


from functools import partial
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor, as_completed

import domolibrary_extensions.scraper.DriverGenerator as desd
import domolibrary_extensions.scraper.routes as desr
import domolibrary_extensions.utils.utils as deut

from nbdev.showdoc import patch_to

# %% auto 0
__all__ = ['ScrapeConfig', 'is_url_match_pattern', 'ScrapeTask_NoDriverProvided', 'ScrapeTask_NoBaseUrl',
           'ScrapeTask_NoScrapeCrawler', 'ScrapeTask', 'Scrape_Factory_NoConfigMatch', 'ScrapeFactory', 'ScrapeCrawler']

# %% ../../nbs/scraper/Scraper.ipynb 3
from domolibrary_extensions.utils.web import (
    extract_links,
    generate_filename_from_url,
    remove_query_params_from_url,
)

# %% ../../nbs/scraper/Scraper.ipynb 9
@dataclass
class ScrapeConfig:
    """class for collating data about how to scrape a page
    pass a list of scrape_config into ScrapeFactory
    """

    id: re.Pattern  # pattern for matching URLs to appropriate config

    search_criteria_tuple: Tuple = None  # from selenium.webdriver.common.by import By

    content_extractor_fn: Callable = None  # function for the subset of HTML to extract

    link_extractor_fn: Callable = partial(
        extract_links
    )  # function for extracting links from soup

    generate_filename_fn: Callable = (
        None  # fuction for generating filename given a url and dowload_folder
    )

    pagesource_fn: Callable = None

    def __id__(self, other):
        return self.id == other.id

# %% ../../nbs/scraper/Scraper.ipynb 13
def is_url_match_pattern(url, scrape_config_pattern: str):
    """test if a url matches a scrape_config_pattern"""

    pattern = re.compile(scrape_config_pattern)

    match_pattern = pattern.match(url)

    if not match_pattern:
        return False

    return True

# %% ../../nbs/scraper/Scraper.ipynb 15
class ScrapeTask_NoDriverProvided(Exception):
    def __init__(self):
        super().__init__("Driver not provided")


class ScrapeTask_NoBaseUrl(Exception):
    def __init__(self):
        super().__init__(
            "no base_url provided to scrape_task. must run as is_test = True"
        )


class ScrapeTask_NoScrapeCrawler(Exception):
    def __init__(self):
        super().__init__(
            "no scrape_crawler provided.  must run scrape_page as is_test = True"
        )


@dataclass
class ScrapeTask:
    """scrape_factory matches a URL to a scrape_config and returns a scrape_task"""

    url: str  # will be assigned after retrieved from scrape_factory

    base_url: str = None
    download_folder: str = "./SCRAPE"

    generate_filename_fn: Callable = None
    content_extractor_fn: Callable = None
    link_extractor_fn: Callable = None

    pagesource_fn: Callable = None

    max_sleep_time: int = 10
    scrape_config: ScrapeConfig = None

    scrape_config: ScrapeConfig = None
    scrape_crawler: Any = None  # will be optional parent threadpool manager.

    pagesource: str = None

    soup: BeautifulSoup = None

    def __post_init__(self):
        self.generate_filename_fn = (
            self.generate_filename_fn
            or (self.scrape_config and self.scrape_config.generate_filename_fn)
            or partial(generate_filename_from_url, url=self.url, use_separator="/")
        )

        self.pagesource_fn = (
            self.pagesource_fn
            or (self.scrape_config and self.scrape_config.pagesource_fn)
            or partial(
                desr.get_pagesource,
                search_criteria_tuple=(
                    self.scrape_config and self.scrape_config.search_criteria_tuple
                ),
            )
        )

        self.content_extractor_fn = self.content_extractor_fn or (
            self.scrape_config and self.scrape_config.content_extractor_fn
        )

        self.link_extractor_fn = (
            self.link_extractor_fn
            or (self.scrape_config and self.scrape_config.link_extractor_fn)
            or extract_links
        )

    @classmethod
    def _from_factory(
        cls,
        url: str,
        base_url: str,
        scrape_config: ScrapeConfig,
        scrape_crawler=None,
        download_folder: str = "./SCRAPE",
        generate_filename_fn: Callable = None,
    ):
        return cls(
            url=url,
            base_url=base_url,
            download_folder=download_folder,
            scrape_config=scrape_config,
            scrape_crawler=scrape_crawler,
            generate_filename_fn=generate_filename_fn,
        )

    def _download_content(self, file_name, content):

        dir_name = os.path.dirname(file_name)

        if dir_name[-1] != "/":
            dir_name += "/"

        deut.upsert_folder(dir_name)

        with open(file_name, "w", encoding="utf-8") as f:
            f.write(str(content))

        return True

    def _update_parent(self):
        """if config is part of a scrape_crawler / managed threadpool will update crawler's urls_to_visit list."""

        scrape_crawler = self.scrape_crawler

        if not scrape_crawler:
            return False

        scrape_crawler._add_url_to_visited(self.url)

        if self.soup:
            urls_to_visit = self.link_extractor_fn(
                soup=self.soup, base_url=self.base_url
            )

            print(f"{len(urls_to_visit)} exracted from soup")

            [scrape_crawler._add_url_to_visit(url) for url in urls_to_visit]

            print(self.scrape_crawler.urls_to_visit, self.base_url)

        return urls_to_visit

# %% ../../nbs/scraper/Scraper.ipynb 16
@patch_to(ScrapeTask)
def execute(
    self: ScrapeTask,
    driver=None,
    debug_prn: bool = False,
    is_suppress_errors: bool = False,
    is_test: bool = False,
    download_folder: str = None,
):
    """handles executing the scrape_task
    1. get pagesource
    2. download index
    3. download content
    4. update crawler
    - get_links from pagesource
    """

    try:
        self.pagesource = self.pagesource_fn(
            driver=driver,
            url=self.url,
            max_sleep_time=self.max_sleep_time,
            return_entire_page=True,
            return_soup=False,
            debug_prn=debug_prn,
        )

        download_folder = download_folder or self.download_folder

        self.soup = BeautifulSoup(self.pagesource, "lxml")

        # download index
        file_name = self.generate_filename_fn(
            url=self.url, download_folder=download_folder, file_name="index.html"
        )

        self._download_content(file_name=file_name, content=self.soup)

        # download content
        if self.content_extractor_fn:
            content = self.content_extractor_fn(self.soup)
            content_name = "content.html"

            if isinstance(content, tuple):
                content_name = content[1]
                content = content[0]

            content_name = self.generate_filename_fn(
                download_folder=download_folder, file_name=content_name, url=self.url
            )

            self._download_content(
                file_name=content_name,
                content=content,
            )

        print(f"🎉 successfully scraped {self.url} to {file_name}")

        if is_test or not self.scrape_crawler:
            return True

        # update crawler
        # if not self.base_url:
        #     raise ScrapeTask_NoBaseUrl()

        # if not self.scrape_crawler:
        #     raise ScrapeTask_NoScrapeCrawler()

        """if config is part of a scrape_crawler / managed threadpool will update crawler's urls_to_visit list."""
        self._update_parent()

        return True

    except Exception as e:
        if not is_suppress_errors:
            raise (e)
        return f"💀 failed to download {self.url} received errror{e}"

# %% ../../nbs/scraper/Scraper.ipynb 20
class Scrape_Factory_NoConfigMatch(Exception):
    def __init__(self, text):
        super().__init__(
            f"{text} has no pattern match in factory_configs, add an appropriate config or check pattern matches"
        )


@dataclass
class ScrapeFactory:
    """class handles a list of Scrape_Configs and will return the 'correct one' given a URL"""

    factory_configs: List[ScrapeConfig]

    def generate_task_from_url(
        self,
        url,
        download_folder="./SCRAPE",
        base_url=None,
        scrape_crawler=None,
        generate_filename_fn=None,
    ):
        configs = []

        for config in self.factory_configs:
            if is_url_match_pattern(url=url, scrape_config_pattern=config.id):
                configs.append(config)

        if not configs:
            raise Scrape_Factory_NoConfigMatch(text=url)

        return [
            ScrapeTask._from_factory(
                scrape_config=config,
                url=url,
                base_url=base_url,
                download_folder=download_folder,
                scrape_crawler=scrape_crawler,
                generate_filename_fn=generate_filename_fn,
            )
            for config in configs
        ]

# %% ../../nbs/scraper/Scraper.ipynb 22
@dataclass
class ScrapeCrawler:
    """threadpool manager for crawling through a list of urls"""

    base_url: str
    download_folder: str
    visited_urls: set
    urls_to_visit: set

    executor: ThreadPoolExecutor = field(repr=False)
    scrape_factory: ScrapeFactory = field(repr=False)  # list of expected scrape_configs
    driver_generator: desd.DriverGenerator = field(repr=False)

    # def __post_init__(self):

    @classmethod
    def init(
        cls,
        urls_to_visit: List[str],
        base_url: str,
        scrape_factory: ScrapeFactory,
        driver_generator: desd.DriverGenerator = None,
        download_folder: str = "./SCRAPE/",
        driver_path=None,
        max_workers=5,
        urls_visited: List[str] = None,
    ):
        scrape_crawler = cls(
            executor=ThreadPoolExecutor(max_workers=max_workers),
            scrape_factory=scrape_factory,
            base_url=base_url,
            download_folder=download_folder,
            visited_urls=set(),
            urls_to_visit=set(),
            driver_generator=driver_generator
            or desd.DriverGenerator(driver_path=driver_path),
        )

        if urls_visited:
            [scrape_crawler._add_url_to_visited(url) for url in urls_visited]

        if urls_to_visit:
            [scrape_crawler._add_url_to_visit(url) for url in urls_to_visit]

        return scrape_crawler

    def _add_url_to_visit(self, url, debug_prn: bool = False):
        """adds a URL to the list of URLS to visit after testing that the URL has not already been visited"""

        if url not in self.visited_urls and url not in self.urls_to_visit:
            if debug_prn:
                print(f"adding {url} to to_vist list")

            self.urls_to_visit.add(url)
            return self.urls_to_visit

    def _add_url_to_visited(self, url, debug_prn: bool = False):
        if url not in self.visited_urls:
            if debug_prn:
                print(f"adding {url} to visited list")

            self.visited_urls.add(url)
            return self.visited_urls

    def _quit(self):
        """call when the executor queue is empty"""

        self.executor.shutdown(wait=True)
        return f"Done scraping {len(self.visited_urls)} urls"

    def crawl_urls(
        self: ScrapeCrawler,
        is_test: bool = False,
        debug_prn: bool = False,
        is_headless: bool = True,
    ):
        while self.urls_to_visit:
            url = self.urls_to_visit.pop()
            self.visited_urls.add(url)

            driver = self.driver_generator.get_webdriver(is_headless=is_headless)

            tasks = self.scrape_factory.generate_task_from_url(
                url,
                scrape_crawler=self,
                base_url=self.base_url,
                download_folder=self.download_folder,
            )

            future_to_task = {
                self.executor.submit(task.execute, driver=driver, is_test=is_test): task
                for task in tasks
            }
            for future in as_completed(future_to_task):

                try:
                    future.result()

                except Exception as e:
                    print(e)

        return self._quit()
