"""
Core functions needed to make the module work.
"""

# import locale
import logging
import re

from typing import List, Optional, Set, Tuple
from urllib.robotparser import RobotFileParser

from .clean import normalize_url, scrub_url
from .filters import (
    basic_filter,
    domain_filter,
    extension_filter,
    is_navigation_page,
    is_not_crawlable,
    lang_filter,
    path_filter,
    type_filter,
    validate_url,
)
from .network import redirection_test
from .settings import BLACKLIST
from .urlutils import (
    extract_domain,
    get_base_url,
    fix_relative_urls,
    is_external,
    is_known_link,
)


LOGGER = logging.getLogger(__name__)

FIND_LINKS_REGEX = re.compile(r"<a [^<>]+?>", re.I)
HREFLANG_REGEX = re.compile(r'hreflang=["\']?([a-z-]+)', re.I)
LINK_REGEX = re.compile(r'href=["\']?([^ ]+?)(["\' >])', re.I)


def check_url(
    url: str,
    strict: bool = False,
    with_redirects: bool = False,
    language: Optional[str] = None,
    with_nav: bool = False,
    trailing_slash: bool = True,
) -> Optional[Tuple[str, str]]:
    """Check links for appropriateness and sanity
    Args:
        url: url to check
        strict: set to True for stricter filtering
        with_redirects: set to True for redirection test (per HTTP HEAD request)
        language: set target language (ISO 639-1 codes)
        with_nav: set to True to include navigation pages instead of discarding them
        trailing_slash: set to False to trim trailing slashes

    Returns:
        A tuple consisting of canonical URL and extracted domain

    Raises:
        ValueError, handled in exception.
    """

    # first sanity check
    # use standard parsing library, validate and strip fragments, then normalize
    try:
        # length test
        if basic_filter(url) is False:
            LOGGER.debug("rejected, basic filter: %s", url)
            raise ValueError

        # clean
        url = scrub_url(url)

        # get potential redirect, can raise ValueError
        if with_redirects:
            url = redirection_test(url)

        # spam & structural elements
        if type_filter(url, strict=strict, with_nav=with_nav) is False:
            LOGGER.debug("rejected, type filter: %s", url)
            raise ValueError

        # internationalization and language heuristics in URL
        if (
            language is not None
            and lang_filter(url, language, strict, trailing_slash) is False
        ):
            LOGGER.debug("rejected, lang filter: %s", url)
            raise ValueError

        # split and validate
        validation_test, parsed_url = validate_url(url)
        if validation_test is False:
            LOGGER.debug("rejected, validation test: %s", url)
            raise ValueError

        # content filter based on extensions
        if extension_filter(parsed_url.path) is False:
            LOGGER.debug("rejected, extension filter: %s", url)
            raise ValueError

        # unsuitable domain/host name
        if domain_filter(parsed_url.netloc) is False:
            LOGGER.debug("rejected, domain name: %s", url)
            raise ValueError

        # strict content filtering
        if strict and path_filter(parsed_url.path, parsed_url.query) is False:
            LOGGER.debug("rejected, path filter: %s", url)
            raise ValueError

        # normalize
        url = normalize_url(parsed_url, strict, language, trailing_slash)

        # domain info: use blacklist in strict mode only
        if strict:
            domain = extract_domain(url, blacklist=BLACKLIST, fast=True)
        else:
            domain = extract_domain(url, fast=True)
        if domain is None:
            LOGGER.debug("rejected, domain name: %s", url)
            return None

    # handle exceptions
    except (AttributeError, ValueError):
        LOGGER.debug("discarded URL: %s", url)
        return None

    return url, domain


def extract_links(
    pagecontent: str,
    url: Optional[str] = None,
    base_url: Optional[str] = None,
    external_bool: bool = False,
    no_filter: bool = False,
    language: Optional[str] = None,
    strict: bool = True,
    trailing_slash: bool = True,
    with_nav: bool = False,
    redirects: bool = False,
    reference: Optional[str] = None,
) -> Set[str]:
    """Filter links in a HTML document using a series of heuristics
    Args:
        pagecontent: whole page in binary format
        url: full URL of the original page
        base_url: deprecated, legacy only
        external_bool: set to True for external links only, False for
                  internal links only
        no_filter: override settings and bypass checks to return all possible URLs
        language: set target language (ISO 639-1 codes)
        strict: set to True for stricter filtering
        trailing_slash: set to False to trim trailing slashes
        with_nav: set to True to include navigation pages instead of discarding them
        with_redirects: set to True for redirection test (per HTTP HEAD request)
        reference: provide a host reference for external/internal evaluation

    Returns:
        A set containing filtered HTTP links checked for sanity and consistency.

    Raises:
        Nothing.
    """
    base_url = base_url or get_base_url(url)
    url = url or base_url
    candidates, validlinks = set(), set()  # type: Set[str], Set[str]
    if not pagecontent:
        return validlinks
    # define host reference
    reference = reference or base_url
    # extract links
    for link in (m[0] for m in FIND_LINKS_REGEX.finditer(pagecontent)):
        if "rel" in link and "nofollow" in link:
            continue
        # https://en.wikipedia.org/wiki/Hreflang
        if no_filter is False and language is not None and "hreflang" in link:
            langmatch = HREFLANG_REGEX.search(link)
            if langmatch and (
                langmatch[1].startswith(language) or langmatch[1] == "x-default"
            ):
                linkmatch = LINK_REGEX.search(link)
                if linkmatch:
                    candidates.add(linkmatch[1])
        # default
        else:
            linkmatch = LINK_REGEX.search(link)
            if linkmatch:
                candidates.add(linkmatch[1])
    # filter candidates
    for link in candidates:
        # repair using base
        if not link.startswith("http"):
            link = fix_relative_urls(url, link)
        # check
        if no_filter is False:
            checked = check_url(
                link,
                strict=strict,
                trailing_slash=trailing_slash,
                with_nav=with_nav,
                with_redirects=redirects,
                language=language,
            )
            if checked is None:
                continue
            link = checked[0]
            # external/internal links
            if external_bool != is_external(
                url=link, reference=reference, ignore_suffix=True
            ):
                continue
        if is_known_link(link, validlinks):
            continue
        validlinks.add(link)
    # return
    LOGGER.info("%s links found – %s valid links", len(candidates), len(validlinks))
    return validlinks


def filter_links(
    htmlstring: str,
    url: Optional[str],
    base_url: Optional[str] = None,
    lang: Optional[str] = None,
    rules: Optional[RobotFileParser] = None,
    external: bool = False,
    strict: bool = False,
    with_nav: bool = True,
) -> Tuple[List[str], List[str]]:
    "Find links in a HTML document, filter them and add them to the data store."
    links, links_priority = [], []
    url = url or base_url
    for link in extract_links(
        pagecontent=htmlstring,
        url=url,
        external_bool=external,
        language=lang,
        strict=strict,
        with_nav=with_nav,
    ):
        # sanity check
        if is_not_crawlable(link) or (
            rules is not None and not rules.can_fetch("*", link)
        ):
            continue
        # store
        if is_navigation_page(link):
            links_priority.append(link)
        else:
            links.append(link)
    return links, links_priority
