import copy
from page_scraper.entities.builders import build_response_meta
from page_scraper.entities.models import ResponseMeta, UrlContext

from page_scraper.core.utils import exclude_tags, process_text, canonical_url, process_link, classify_url

from bs4 import Tag

from page_scraper.infrastructure.fetcher import HttpFetcher

from requests import Response
from bs4 import BeautifulSoup

TAGS_TO_EXCLUDE = ("[class*=comment], [id*=comment], [id*=sidebar], [id*=footer], [id*=header],"
                   "nav, footer, aside, menu, sidebar, head, header, script")

fetcher = HttpFetcher()

def prepare_response_header(response: Response) -> ResponseMeta:
    return build_response_meta(response.headers)

def get_nav_links(soup: BeautifulSoup) -> list[Tag]:
    return [a for a in soup.select("nav a") if a.get("href")]

def get_clean_link(soup: BeautifulSoup) -> list[Tag]:
    cleaned = copy.copy(soup)
    exclude_tags(cleaned, TAGS_TO_EXCLUDE)
    return [
        a for a in cleaned.select("a")
        if a.get("href")
    ]



def prepare_links(domain:str,soup: list[Tag], position:str=None) -> list[UrlContext]:
    result = []
    for url in soup:
        can_url = canonical_url(url.get("href"), domain)
        if not can_url:
            continue

        classified_url = classify_url(can_url)

        if classified_url=="web":
            response = fetcher.head(can_url)
            data = process_link(can_url, response = response)
        else:
            data = process_link(can_url, url_type = classified_url)

        if not data:
            continue

        data.text = process_text(url)

        data.position=position

        result.append(data)

    return result


def get_page_canonical_url(soup:BeautifulSoup) -> Tag | None:
    return soup.find('link', rel="canonical")
