import json
import logging
import mimetypes
import time
from datetime import datetime
from functools import partial
from typing import Any, Dict, List, Optional, Union

import requests
from bs4 import BeautifulSoup
from newspaper import Article, Config
from tqdm import tqdm

from arkeo.cfg import ConfigurationManager
from arkeo.clean import MarkdownCleaner, convert_markdown_to_text, KEY_MARKDOWN
from arkeo.keys import PrimaryKey, format_nav_key, format_nav_path, is_nav_key_format
from arkeo.meta import MetadataExtractor, KEY_METADATA
from arkeo.path import PathManager
from arkeo.utils import StatsLoggingCodes as SL, slog_domain, to_str
from ref.provider import ProviderManager
from utils.access import DriveManager
from utils.alfnu import expand_range
from utils.file import extract_extension_from_filename
from utils.html import (
    get_headers,
    get_headers_by_index,
    DEFAULT_MAX_RETRIES,
    DEFAULT_SLEEP,
    DEFAULT_TIMEOUT,
)
from utils.init import slog
from utils.text import to_type_datetime
from utils.url import extract_domain, is_valid_url


log = logging.getLogger(__name__)

FILE_EXT = "md"
EXT_TEXT = "txt"


class Ingester:
    """loads article from raw html with metadata extraction"""

    def __init__(
        self,
        path_manager: Optional[PathManager] = None,
        timeout: int = DEFAULT_TIMEOUT,
        sleep: int = DEFAULT_SLEEP,
        max_retries: int = DEFAULT_MAX_RETRIES,
        stop_words: Optional[List[str]] = None,
    ):
        self.path = path_manager or PathManager()
        self.drive: DriveManager = self.path.drive

        self.pm = ProviderManager(self.path)
        self.pk = PrimaryKey(self.path)
        self.cfgs = ConfigurationManager(self.path)

        self.md = MarkdownCleaner(self.path, self.pk, self.cfgs)
        self.mx = MetadataExtractor(self.path, self.pk, self.cfgs)

        self._timeout = timeout
        self._sleep = sleep
        self._max_retries = max_retries
        self._stop_words = stop_words

    # public interface methods
    def process(self, source: Union[str, List[str]]) -> None:
        if source and isinstance(source, str) and self.path.dir_data in source:
            keys = self.drive.read_or_none(source).strip().split("\n")
        else:
            keys = expand_range(source)

        for key in tqdm(keys, desc="processing", ncols=80, ascii=True):
            self.process_one(key)

    def process_one(self, source: str) -> bool:
        if is_valid_url(source):
            article = self.load_from_url(source)
        else:
            if not is_nav_key_format(source):
                source = self.pk.get_pk_attribute(source, "nav_key")
            article = self.load_from_file(source)

        if not article or not article.html:
            log.error(f"failed to load: {source}")
            return False

        if not self.parse(article):
            log.error("failed to parse")
            return False
        else:
            log.debug(f"html: {len(article.html)} text: {len(article.text)}")

        if not self.extract_nlp_features(article):
            log.warning("failed to extract NLP features")
        else:
            log.debug(to_str(article, prove_its_useless=True, shorter=True))

        return self._export(article)

    def load_from_file(self, key: Any) -> Optional[Article]:
        """load article from local or google drive html file"""
        if not key:
            log.warning("show me the key")
            return None

        # TODO fix nav
        if isinstance(key, int):
            key = f"{key:03d}"
            log.warning(f"key, numeric: {key}")

        if isinstance(key, str):
            try:
                nav_key = format_nav_path(key)
                file_path = f"{self.path.raw}{nav_key}.html"
                log.info(f"file_path: {file_path}")
                content = self.path.drive.read(file_path)

                article = Article("")
                article.set_html(content)
                article.nav_key = nav_key
                return article
            except Exception as e:
                log.error(f"load from file '{file_path}' failed. {e}")

        return None

    def load_from_url(
        self,
        url: str,
        config: Optional[Config] = None,
        custom_headers: Optional[Dict[str, str]] = None,
    ) -> Optional[Article]:
        """download article from url with retry logic"""
        if self._is_file_download(url):
            # todo: download the non-html file
            article = Article(url)
            article.download_exception_msg = "mime err"
            slog(
                SL.UNSUPPORTED,
                url,
                extract_extension_from_filename(url),
                article.download_exception_msg,
            )
            return article

        headers = custom_headers or get_headers()

        if not config:
            config = Config()
            config.request_timeout = self._timeout

        for attempt in range(self._max_retries + 1):
            try:
                if attempt > 1:
                    headers = get_headers_by_index(0)
                elif attempt > 0:
                    headers = None

                config.headers = headers
                article = Article(url, config=config)
                article.download()

                if not self._validate_article_html(article):
                    error = getattr(article, "download_exception_msg", "")
                    if "forbidden" in error.lower():
                        article.download_exception_msg = f"forbidden: {error}"
                        rc = SL.FORBIDDEN
                    elif "timeout" in error.lower():
                        article.download_exception_msg = f"timeout: {error}"
                        rc = SL.TIMEOUT
                    else:
                        article.download_exception_msg = f"mystery: {error}"
                        rc = SL.ERROR
                    slog_domain(rc, url, headers, error)
                    return article  # hard wall

                time.sleep(self._sleep)
                article.download_exception_msg = ""
                slog_domain(SL.SUCCESS, url, headers)
                log.info(f"successful download ({url}) [headers: {config.headers}]")
                return article

            except requests.exceptions.Timeout:
                article = Article(url)
                article.download_exception_msg = f"timeout (attempt {attempt + 1})"
                if attempt < self._max_retries:
                    time.sleep(self._sleep * (attempt + 1))
                    continue
                slog_domain(SL.TIMEOUT, url, headers, article.download_exception_msg)
                return article

            except Exception as e:
                article = Article(url)
                article.download_exception_msg = f"failure: {str(e)}"
                slog_domain(SL.ERROR, url, headers, e)
                return article

        slog_domain(SL.ERROR, url, headers, "she moves in mysterious ways")
        return article

    def parse(self, article: Article) -> bool:
        """parse article content and extract metadata"""
        try:
            article.parse()
        except Exception as e:
            log.warning(e)

        article.soup = BeautifulSoup(article.html, "html.parser")
        config = self._get_config(article)  # sometimes needs soup

        self._integrate_metadata(article, config[KEY_METADATA])  # compensate newspaper
        self._process_markdown(article, config[KEY_MARKDOWN])  # refine content vs meta

        return True

    def extract_nlp_features(self, article: Article) -> bool:
        """extract nlp features from parsed article"""
        if not article:
            log.error("no article")
            return False

        if not article.is_parsed:
            article.is_parsed = True
            log.warning("manually activated 'parsed' flag")

        article.nlp()

        # todo: override newspaper keywords
        # init: self.kw_yake = yake.KeywordExtractor(n=5,top=10)
        return True

    def write_markdown_or_text(self, article) -> bool:
        """write markdown content or fallback to text if markdown fails"""
        if not self._has_nav_key(article):
            log.warning("requires nav_key")
            return False

        file_path = f"{self.path.corpus}{format_nav_path(article.nav_key)}.{FILE_EXT}"

        if hasattr(article, "markdown") and article.markdown:
            if self.drive.write_or_false(file_path, article.markdown):
                return True
            log.warning("failed to write markdown")

        if article.text:
            if self.drive.write_or_false(
                file_path.replace(FILE_EXT, EXT_TEXT), article.text
            ):
                return True
            log.warning("failed to write text")

        return False

    # content processing methods
    def _integrate_metadata(self, article: Article, config: Dict[str, Any]) -> None:
        """supplement / override 3rd party properties"""
        article_fields = [
            attr
            for attr in dir(article)
            if not attr.startswith("_") and not callable(getattr(article, attr))
        ]

        self.mx.set_slog(partial(slog_domain, url=article.url) if article.url else None)

        soup = getattr(article, "soup", None)  # cya
        updates = self.mx.extract_metadata(
            article_fields,
            config,
            article.html,
            soup,
        )

        processed_keys = []
        for key in updates:
            if key.startswith("_") or (article_fields and key not in article_fields):
                continue

            try:
                log.info(f"{key} ↦ {updates[key]}")

                value = updates[key]
                if isinstance(value, list):
                    value = ", ".join(value)
                setattr(article, key, value)

                processed_keys.append(key)
            except (AttributeError, TypeError) as e:
                log.warning(f"failed to set {key}: {e}")

        for key in processed_keys:
            updates.pop(key)

        # excess into miscellany
        article.additional_data = json.dumps(updates)

    def _process_markdown(self, article: Article, config: Dict[str, Any]) -> None:
        self.md.set_slog(partial(slog_domain, url=article.url) if article.url else None)

        if extracted := self.md.extract_body(article.html, config):
            article.html = extracted

        frontmatter = self._get_frontmatter(article)
        markdown = self.md.convert(
            article.html,
            config,
            frontmatter,
            article.imgs,
        )
        if not markdown:
            return

        article.text = convert_markdown_to_text(markdown) or article.text

        if "meta_img" in frontmatter:
            del frontmatter["meta_img"]
        if "top_image" in frontmatter:
            del frontmatter["top_image"]
        markdown = self.md.format_header_yaml(frontmatter) + markdown
        article.markdown = markdown

    # export
    def _export(self, article: Article) -> bool:
        """export article, set nav_key needed"""
        new_pk = None
        if not self._has_nav_key(article):
            new_pk = self.pk.get_next_pk()
            publish_date = to_type_datetime(article.publish_date)
            article.nav_key = format_nav_key(publish_date, new_pk)

        if not self.write_markdown_or_text(article):
            log.warning(f"failed to export {article.nav_key}: {article.url}")
            return False

        if new_pk:
            self.pk.update_pk(
                new_pk, {"nav_key": article.nav_key, "url": article.canonical_link}
            )

        return True

    # configuration, helpers
    def _get_config(self, article: Article) -> Optional[Dict[str, Any]]:
        config = self.cfgs.get(article.source_url)
        old_domain = article.source_url
        config = self._resolve_canonical_config(article, config)

        old_domain = "" if article.source_url == old_domain else f"'{old_domain}' ↦ "
        log.info(f"source_url: {old_domain}'{article.source_url}'")
        if not article.canonical_link:
            log.error("HTML possibly broken.")
            return None

        return config

    def _resolve_canonical_config(
        self, article: Article, default_config: Dict[str, Any]
    ) -> Dict[str, Any]:
        """extract canonical URL and return domain-specific configuration"""
        """
        url: download url
        canonical_url: provider-defined
        source_url: provider-defined domain, with "https" stripped to without
        """
        if not article.canonical_link or not is_valid_url(article.canonical_link):
            article.canonical_link = self.mx._extract_with_css_selectors(
                article.soup, default_config.get("url", [])
            )

        if not article.canonical_link:
            # for broken html
            article.canonical_link = self._extract_canonical_url(article.soup)

        if not article.canonical_link and is_valid_url(article.url):
            article.canonical_link = article.url

        article.source_url = extract_domain(article.canonical_link)

        if not is_valid_url(article.url):
            article.url = article.canonical_link

        return self.cfgs.get(article.source_url)

    def _extract_canonical_url(self, soup: BeautifulSoup) -> str:
        """extract canonical URL with multiple fallbacks / for broken HTML"""

        # try og:url first
        og_url = soup.find("meta", property="og:url")
        if og_url and og_url.get("content"):
            return og_url.get("content").strip()

        # try canonical link
        canonical = soup.find("link", rel="canonical")
        if canonical and canonical.get("href"):
            return canonical.get("href").strip()

        # try twitter:url
        twitter_url = soup.find("meta", property="twitter:url")
        if twitter_url and twitter_url.get("content"):
            return twitter_url.get("content").strip()

        return ""

    # formatting and generation helpers
    def _get_frontmatter(self, article: Article) -> Dict[str, Any]:
        """extract standard metadata fields from newspaper3k Article"""
        authors = article.authors
        if authors and isinstance(authors, list):
            authors = ", ".join(authors)
        pub_date = article.publish_date
        if pub_date and isinstance(pub_date, datetime):
            pub_date = pub_date.strftime("%B %d, %Y")

        return {
            "title": article.title or "",
            "canonical": article.canonical_link or article.url or "",
            "description": article.meta_description or "",
            "authors": authors or "",
            "date": pub_date or "",
            "tags": article.keywords or "",
            "category": self.pm.get_category(article.source_url),
            "top_image": article.top_image or "",
            "meta_img": article.meta_img or "",
            "source_url": article.source_url or "",
        }

    # validation and classification helpers
    def _has_nav_key(self, article: Article) -> bool:
        return hasattr(article, "nav_key") and article.nav_key

    def _is_file_download(self, url: str) -> bool:
        """check if URL points to file vs HTML"""
        mime_type, _ = mimetypes.guess_type(url)
        if mime_type is None:
            return False
        return not mime_type.startswith(("text/html", "application/xhtml+xml"))

    def _validate_article_html(self, article: Article) -> bool:
        """validate article contains HTML"""
        if not article or not hasattr(article, "html") or not article.html:
            return False

        if hasattr(article, "response") and article.response:
            content_type = article.response.headers.get("content-type", "").lower()
            if self._is_file_download(content_type):
                return False

            content_disposition = article.response.headers.get(
                "content-disposition", ""
            )
            if "attachment" in content_disposition.lower():
                return False

        if article.html.count("\x00") > len(article.html) * 0.01:
            return False

        return True
