import time
from types import AsyncGeneratorType
from typing import Tuple, List, Union, Dict

import feedparser
from bs4 import BeautifulSoup
from yarl import URL

from feedsearch_crawler.crawler import ItemParser, Request, Response, to_string
from feedsearch_crawler.feed_spider.feed_info import FeedInfo
from feedsearch_crawler.feed_spider.lib import parse_header_links


class FeedInfoParser(ItemParser):
    async def parse_item(
        self, request: Request, response: Response, *args, **kwargs
    ) -> AsyncGeneratorType:
        self.logger.info("Parsing feed %s", response.url)

        content_type = response.headers.get("content-type", "")

        item = FeedInfo(response.url, content_type)

        # Check link headers first for WebSub content discovery
        # https://www.w3.org/TR/websub/#discovery
        if response.headers:
            item.hubs, item.self_url = self.header_links(response.headers)

        if "type" not in kwargs:
            raise ValueError("type keyword argument is required")

        try:
            data_type = kwargs["type"]
            if data_type == "json":
                item.content_type = "application/json"
                self.parse_json(item, response.json)
            elif data_type == "xml":
                self.parse_xml(item, response.data, response.encoding, response.headers)
                if not item.content_type:
                    item.content_type = "text/xml"
        except Exception as e:
            self.logger.exception("Failed to parse feed %s, Error: %s", item, e)

        if item.favicon and self.spider.favicon_data_uri:
            yield self.spider.follow(item.favicon, self.spider.create_data_uri)

        item.content_length = response.content_length
        self.score_item(item, response.history[0])
        yield item

    def parse_xml(
        self, item: FeedInfo, data: str, encoding: str, headers: Dict
    ) -> None:
        """
        Get info from XML (RSS or ATOM) feed.
        """

        # Parse data with feedparser
        # Don't wrap this in try/except, feedparser eats errors and returns bozo instead
        parsed = self.parse_raw_data(data, encoding, headers)
        if not parsed or parsed.get("bozo") == 1:
            if not isinstance(
                parsed.get("bozo_exception"), feedparser.CharacterEncodingOverride
            ):
                item.bozo = 1
                self.logger.warning("No valid feed data for %s", item)
                return

        feed = parsed.get("feed")

        # Only search if no hubs already present from headers
        if not item.hubs:
            item.hubs, item.self_url = self.websub_links(feed)

        if item.hubs and item.self_url:
            item.is_push = True

        item.version = parsed.get("version")
        item.title = self.feed_title(feed)
        item.description = self.feed_description(feed)

    @staticmethod
    def parse_json(item: FeedInfo, data: dict) -> None:
        """
        Get info from JSON feed.

        :param item: FeedInfo object
        :param data: JSON object
        :return: None
        """
        item.version = data.get("version")
        if "https://jsonfeed.org/version/" not in item.version:
            item.bozo = 1
            return

        item.title = data.get("title")
        item.description = data.get("description")

        favicon = data.get("favicon")
        if favicon:
            item.favicon = URL(favicon)

        # Only search if no hubs already present from headers
        if not item.hubs:
            try:
                item.hubs = list(hub.get("url") for hub in data.get("hubs", []))
            except (IndexError, AttributeError):
                pass

        if item.hubs:
            item.is_push = True

    def parse_raw_data(
        self, raw_data: Union[str, bytes], encoding: str = "utf-8", headers: Dict = None
    ) -> Dict:
        """
        Loads the raw RSS/Atom XML data.
        Returns feedparser Dict.
        https://pythonhosted.org/feedparser/

        :param raw_data: RSS/Atom XML feed
        :type raw_data: str
        :param encoding: Character encoding of raw_data
        :type encoding: str
        :param headers: Response headers
        :return: Dict
        """
        if not encoding:
            encoding = "utf-8"

        h = {}
        if headers:
            if isinstance(headers, dict):
                h = headers
            else:
                try:
                    h.update({k.lower(): v for (k, v) in headers.items()})
                except KeyError:
                    pass

            h.pop("content-encoding", None)

        try:
            start = time.perf_counter()

            if isinstance(raw_data, str):
                raw_data: bytes = raw_data.encode(encoding)

            content_length = len(raw_data)

            # We want to pass data into feedparser as bytes, otherwise if we accidentally pass a url string
            # it will attempt a fetch
            data = feedparser.parse(raw_data, response_headers=h)

            dur = int((time.perf_counter() - start) * 1000)
            self.logger.debug("Feed Parse: size=%s dur=%sms", content_length, dur)

            return data
        except Exception as e:
            self.logger.exception("Could not parse RSS data: %s", e)

    def feed_title(self, feed: dict) -> str:
        """
        Get feed title

        :param feed: feed dict
        :return: str
        """
        title = feed.get("title", None)
        if not title:
            return ""
        return self.clean_title(title)

    def clean_title(self, title: str) -> str:
        """
        Cleans title string, and shortens if too long.
        Have had issues with dodgy feed titles.

        :param title: Title string
        :return: str
        """
        try:
            title = BeautifulSoup(title, self.spider.htmlparser).get_text()
            if len(title) > 1024:
                title = title[:1020] + "..."
            return title
        except Exception as ex:
            self.logger.exception("Failed to clean title: %s", ex)
            return ""

    @staticmethod
    def feed_description(feed: dict) -> str:
        """
        Get feed description.

        :param feed: feed dict
        :return: str
        """
        subtitle = feed.get("subtitle", None)
        if subtitle:
            return subtitle
        return feed.get("description", None)

    @staticmethod
    def websub_links(feed: dict) -> Tuple[List[str], str]:
        """
        Returns a tuple containing the hub url and the self url for
        a parsed feed.

        :param feed: An RSS feed parsed by feedparser
        :type feed: dict
        :return: tuple
        """
        links = feed.get("links", [])
        return FeedInfoParser.find_hubs_and_self_links(links)

    @staticmethod
    def header_links(headers: dict) -> Tuple[List[str], str]:
        """
        Attempt to get self and hub links from HTTP headers
        https://www.w3.org/TR/websub/#x4-discovery

        :param headers: Dict of HTTP headers
        :return: None
        """
        link_header = headers.get("Link")
        links: list = []
        if link_header:

            links = parse_header_links(to_string(link_header))
        return FeedInfoParser.find_hubs_and_self_links(links)

    @staticmethod
    def find_hubs_and_self_links(links: List[dict]) -> Tuple[List[str], str]:
        """
        Parses a list of links into self and hubs urls

        :param links: List of parsed HTTP Link Dicts
        :return: Tuple
        """
        hub_urls: List[str] = []
        self_url: str = ""

        if not links:
            return [], ""

        for link in links:
            try:
                if link["rel"] == "hub":
                    href: str = link["href"]
                    hub_urls.append(href)
                elif link["rel"] == "self":
                    self_url = link["href"]
            except KeyError:
                continue

        return hub_urls, self_url

    @staticmethod
    def score_item(item: FeedInfo, original_url: URL):
        score = 0

        url_str = str(item.url)

        # -- Score Decrement --

        if original_url:
            host = original_url.host
            if host.startswith("www."):
                host = host[len("www.") :]

            if host not in item.url.host:
                score -= 20

        # Decrement the score by every extra path in the url
        parts_len = len(item.url.parts)
        if parts_len > 2:
            score -= (parts_len - 2) * 2

        if item.bozo:
            score -= 20
        if not item.description:
            score -= 10
        if "georss" in url_str:
            score -= 10
        if "alt" in url_str:
            score -= 7
        if "comments" in url_str or "comments" in item.title.lower():
            score -= 15

        # -- Score Increment --
        if item.url.scheme == "https":
            score += 10
        if item.is_push:
            score += 10
        if "index" in url_str:
            score += 15

        kw = ["atom", "rss", ".xml", "feed", "rdf"]
        for p, t in zip(range(len(kw) * 2, 0, -2), kw):
            if t in url_str:
                score += p

        item.score = score
