"""supporting functions"""

# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/utils/web.ipynb.

# %% auto 0
__all__ = ['remove_query_params_from_url', 'generate_filename_from_url', 'extract_links']

# %% ../../nbs/utils/web.ipynb 3
from typing import List, Callable, Union
from bs4 import BeautifulSoup
import domolibrary_extensions.utils.files as defi

from urllib.parse import urljoin, urlparse

# %% ../../nbs/utils/web.ipynb 4
from domolibrary_extensions.utils.files import (
    convert_html_to_markdown,
    convert_remove_html_tags,
)

# %% ../../nbs/utils/web.ipynb 5
def remove_query_params_from_url(url):
    u = urlparse(url)
    return urljoin(url, urlparse(url).path)

# %% ../../nbs/utils/web.ipynb 7
def generate_filename_from_url(
    url,
    download_folder=None,
    use_separator="/",
    max_length=None,
    file_name=None,
    suffix=None,
) -> str:
    parsed_url = urlparse(remove_query_params_from_url(url))

    file_path = [str for str in parsed_url[2].split("/") if str][:max_length]

    if download_folder:
        file_path[0:0] = [download_folder]

    if file_name:
        file_path.append(file_name)

    if suffix:
        file_path[-1] = defi.change_file_suffix(file_path[-1], suffix)

    file_path = use_separator.join(file_path)

    return file_path

# %% ../../nbs/utils/web.ipynb 9
def extract_links(
    soup: BeautifulSoup,
    base_url: str = None,
    custom_link_extractor_fn: Callable = None,  # can add custom function for handling URLs
) -> List[Union[str, dict]]:
    """returns a list of urls"""

    links_ls = []

    for link in soup.findAll("a"):
        if not link.has_attr("href"):
            continue

        url = link["href"]

        if custom_link_extractor_fn:
            url = custom_link_extractor_fn(url, base_url)

        if not url:
            continue

        if url.startswith("/") and base_url:
            url = urljoin(base_url, url)

        if base_url and not url.startswith(base_url):
            continue

        if not url or url in links_ls:
            continue

        links_ls.append(url)

    return list(set(links_ls))
