import requests
import xml.etree.ElementTree as ET
from urllib.parse import urlparse
import pandas as pd
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor, as_completed
import time
import random


class DetailedSitemapParser:
    def __init__(self, sitemap_url: str, max_workers: int = 10):
        self.sitemap_url = sitemap_url
        self.max_workers = max_workers
        self.headers = {
            "User-Agent": (
                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
                "AppleWebKit/537.36 (KHTML, like Gecko) "
                "Chrome/115.0 Safari/537.36"
            )
        }

    # ------------------------ #
    # Fetch sitemap XML
    # ------------------------ #
    def fetch_sitemap(self):
        response = requests.get(self.sitemap_url, timeout=20)
        response.raise_for_status()
        return response.text

    # ------------------------ #
    # Parse sitemap into URL list
    # ------------------------ #
    def parse_urls(self):
        xml_data = self.fetch_sitemap()
        root = ET.fromstring(xml_data)
        namespace = {"ns": "http://www.sitemaps.org/schemas/sitemap/0.9"}

        rows = []

        for url_tag in root.findall("ns:url", namespace):
            loc_tag = url_tag.find("ns:loc", namespace)
            if loc_tag is None:
                continue

            full_url = loc_tag.text.strip()
            path = urlparse(full_url).path.strip("/")

            if path == "":
                category = ""
                slug = ""
            else:
                parts = path.split("/")
                category = parts[0]
                slug = "/".join(parts[1:]) if len(parts) > 1 else ""

            rows.append([full_url, category, slug])

        df = pd.DataFrame(rows, columns=["url", "category", "slug"])
        return df

    # ------------------------ #
    # Extract Title, Meta Title, Meta Description, H1
    # ------------------------ #
    def fetch_page_data(self, idx, url, retries=1):
        try:
            time.sleep(random.uniform(0.05, 0.15))
            response = requests.get(url, headers=self.headers, timeout=8)
            response.raise_for_status()
            html = response.text
        except Exception:
            if retries > 0:
                return self.fetch_page_data(idx, url, retries - 1)
            return idx, {"meta_title": "", "meta_description": "", "h1": ""}

        soup = BeautifulSoup(html, "lxml")

        meta_title = soup.title.string.strip() if soup.title and soup.title.string else ""

        meta_desc_tag = soup.find("meta", attrs={"name": "description"})
        meta_description = meta_desc_tag["content"].strip() if meta_desc_tag else ""

        h1_tag = soup.find("h1")
        h1 = h1_tag.get_text(strip=True) if h1_tag else ""

        return idx, {
            "meta_title": meta_title,
            "meta_description": meta_description,
            "h1": h1
        }

    # ------------------------ #
    # Multi-threaded metadata enrichment (SAFE)
    # ------------------------ #
    def enrich_metadata(self, df):
        urls = df["url"].tolist()
        results = {i: {"meta_title": "", "meta_description": "", "h1": ""} for i in range(len(urls))}

        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            futures = {
                executor.submit(self.fetch_page_data, idx, url): idx
                for idx, url in enumerate(urls)
            }

            for future in as_completed(futures):
                idx, data = future.result()
                results[idx] = data  # Correct mapping by index

        df["meta_title"] = [results[i]["meta_title"] for i in range(len(results))]
        df["meta_description"] = [results[i]["meta_description"] for i in range(len(results))]
        df["h1"] = [results[i]["h1"] for i in range(len(results))]

        return df

    # ------------------------ #
    # MAIN PROCESS
    # ------------------------ #
    def parse(self):
        df = self.parse_urls()
        df = self.enrich_metadata(df)
        return df
