from io import BytesIO
from pypdf import PdfReader
import requests
import re
from requests.exceptions import RequestException
from bs4 import BeautifulSoup
from newspaper import Article, ArticleException, Config
from requests_html import HTMLSession
import random
from lxml import html
from urllib.parse import urljoin
import csv

from querent.logging.logger import setup_logger

logger = setup_logger(__name__, "webpage_extractor.log")

USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Safari/605.1.15",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.0",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:72.0) Gecko/20100101 Firefox/72.0",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Safari/605.1.15",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:80.0) Gecko/20100101 Firefox/80.0",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36",
    "Mozilla/5.0 (iPhone; CPU iPhone OS 13_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.4 Mobile/15E148 Safari/604.1",
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36",
]


class WebpageExtractor:
    def __init__(self, num_extracts=3):
        """
        Initialize the WebpageExtractor class.
        """
        self.crawled_urls = set()
        self.num_extracts = num_extracts
        self.data = []

    def extract_with_3k(self, url):
        """
        Extract the text from a webpage using the 3k method.

        Args:
            url (str): The URL of the webpage to extract from.

        Returns:
            str: The extracted text.
        """
        try:
            if url.lower().endswith(".pdf"):
                response = requests.get(url)
                response.raise_for_status()

                with BytesIO(response.content) as pdf_data:
                    reader = PdfReader(pdf_data)
                    content = " ".join(
                        [
                            reader.getPage(i).extract_text()
                            for i in range(reader.getNumPages())
                        ]
                    )

            else:
                config = Config()
                config.browser_user_agent = random.choice(USER_AGENTS)
                config.request_timeout = 10
                session = HTMLSession()

                response = session.get(url)
                response.html.render(timeout=config.request_timeout)
                html_content = response.html.html

                article = Article(url, config=config)
                article.set_html(html_content)
                article.parse()
                content = article.text.replace(
                    '\t', ' ').replace('\n', ' ').strip()

            return content[:1500]

        except ArticleException as ae:
            logger.error(
                f"Error while extracting text from HTML (newspaper3k): {str(ae)}"
            )
            return f"Error while extracting text from HTML (newspaper3k): {str(ae)}"

        except RequestException as re:
            logger.error(
                f"Error while making the request to the URL (newspaper3k): {str(re)}"
            )
            return f"Error while making the request to the URL (newspaper3k): {str(re)}"

        except Exception as e:
            logger.error(
                f"Unknown error while extracting text from HTML (newspaper3k): {str(e)}"
            )
            return ""

    def extract_with_bs4(self, url):
        """
        Extract the text from a webpage using the BeautifulSoup4 method.

        Args:
            url (str): The URL of the webpage to extract from.

        Returns:
            str: The extracted text.
        """
        headers = {"User-Agent": random.choice(USER_AGENTS)}
        try:
            response = requests.get(url, headers=headers, timeout=10)
            if response.status_code == 200:
                soup = BeautifulSoup(response.text, "html.parser")
                for tag in soup(
                    [
                        "script",
                        "style",
                        "nav",
                        "footer",
                        "head",
                        "link",
                        "meta",
                        "noscript",
                    ]
                ):
                    tag.decompose()

                main_content_areas = soup.find_all(
                    ["main", "article", "section", "div"]
                )
                if main_content_areas:
                    main_content = max(main_content_areas,
                                       key=lambda x: len(x.text))
                    content_tags = ["p", "h1", "h2", "h3", "h4", "h5", "h6"]
                    content = " ".join(
                        [
                            tag.text.strip()
                            for tag in main_content.find_all(content_tags)
                        ]
                    )
                else:
                    content = " ".join(
                        [
                            tag.text.strip()
                            for tag in soup.find_all(
                                ["p", "h1", "h2", "h3", "h4", "h5", "h6"]
                            )
                        ]
                    )

                content = re.sub(r"\t", " ", content)
                content = re.sub(r"\s+", " ", content)

                # Store the content of the current URL
                self.crawled_urls.add(url)
                self.store_content(url, content)

                # Recursively crawl internal links
                # self.extract_internal_links_and_crawl(soup)
                return content

            elif response.status_code == 404:
                return f"Error: 404. Url is invalid or does not exist. Try with valid url..."
            else:
                logger.error(f"")
                return f"Error while extracting text from HTML (bs4): {response.status_code} for url - {url}"

        except Exception as e:
            logger.error(
                f"Unknown error while extracting text from HTML (bs4): {str(e)}"
            )
            return ""

    def extract_internal_links_and_crawl(self, soup):
        for link in soup.find_all("a", href=True):
            link_href = link.get("href")
            if (
                link_href.startswith("/")
                or link_href.startswith(".")
                or link_href.startswith("#")
            ):
                link_href = urljoin("https://www.asecuritysite.com", link_href)
            if (
                link_href.startswith("https://www.asecuritysite.com")
                and link_href not in self.crawled_urls
            ):
                self.extract_with_bs4(link_href)

    def store_content(self, url, content):
        self.data.append({"url": url, "content": content})

    def save_to_csv(self):
        csv_file_path = "/home/ansh/Desktop/asecuritysite.csv"
        with open(csv_file_path, "w", newline="", encoding="utf-8") as csvfile:
            fieldnames = ["url", "content"]
            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)

            writer.writeheader()  # Write the header row

            for data_dict in self.data:
                writer.writerow(data_dict)

    def extract_with_lxml(self, url):
        """
        Extract the text from a webpage using the lxml method.

        Args:
            url (str): The URL of the webpage to extract from.

        Returns:
            str: The extracted text.
        """
        try:
            config = Config()
            config.browser_user_agent = random.choice(USER_AGENTS)
            config.request_timeout = 10
            session = HTMLSession()

            response = session.get(url)
            response.html.render(timeout=config.request_timeout)
            html_content = response.html.html

            tree = html.fromstring(html_content)
            paragraphs = tree.cssselect("p, h1, h2, h3, h4, h5, h6")
            content = " ".join(
                [para.text_content()
                 for para in paragraphs if para.text_content()]
            )
            content = content.replace("\t", " ").replace("\n", " ").strip()

            return content

        except ArticleException as ae:
            logger.error(
                "Error while extracting text from HTML (lxml): {str(ae)}")
            return ""

        except RequestException as re:
            logger.error(
                f"Error while making the request to the URL (lxml): {str(re)}")
            return ""

        except Exception as e:
            logger.error(
                f"Unknown error while extracting text from HTML (lxml): {str(e)}"
            )
            return ""

    def extract_links(self, url):
        """
        Extract internal links from a webpage.

        Args:
            url (str): The URL of the webpage to extract links from.

        Returns:
            list: A list of internal links (URLs).
        """
        try:
            headers = {"User-Agent": random.choice(USER_AGENTS)}
            response = requests.get(url, headers=headers, timeout=10)
            if response.status_code == 200:
                soup = BeautifulSoup(response.text, "html.parser")
                links = []
                for link in soup.find_all("a", href=True):
                    link_href = link.get("href")
                    if (
                        link_href.startswith("/")
                        or link_href.startswith(".")
                        or link_href.startswith("#")
                    ):
                        link_href = urljoin(url, link_href)
                    if (
                        link_href.startswith(url)
                        and link_href not in self.crawled_urls
                        and link_href != url
                    ):
                        links.append(link_href)
                return links
            else:
                logger.error(
                    f"Error while extracting links from HTML (bs4): {response.status_code} for url - {url}"
                )
                return []
        except Exception as e:
            logger.error(
                f"Unknown error while extracting links from HTML (bs4): {str(e)}"
            )
            return []
