#!/usr/bin/env python3
"""
B2C Domain List Curator
=======================
Harvests, normalizes, and curates B2C/personal email domains from multiple sources.
Creates a clean, deduplicated list ready for production use.
"""

import re
import json
import hashlib
import logging
from pathlib import Path
from datetime import datetime
from typing import Set, Dict, Tuple
import requests
import publicsuffix2

# Configure logging
logging.basicConfig(
    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
log = logging.getLogger(__name__)


class B2CDomainCurator:
    """Curates and maintains a list of B2C email domains."""

    # Source repositories
    SOURCES = {
        "mailchecker": {
            "url": "https://raw.githubusercontent.com/FGRibreau/mailchecker/master/list.txt",
            "description": "Popular disposable email detector (6,700+ domains)",
            "license": "MIT",
        },
        "disposable_email_domains": {
            "url": "https://raw.githubusercontent.com/disposable/disposable-email-domains/master/domains.txt",
            "description": "Comprehensive disposable list (150k+ but noisy)",
            "license": "MIT",
        },
        "free_email_providers": {
            "url": "https://gist.githubusercontent.com/tbrianjones/5992856/raw/93213efb652749e226e69884d6c048e595c1280a/free_email_provider_domains.txt",
            "description": "Common free email providers (550+)",
            "license": "Public",
        },
    }

    # Well-known consumer ISP domains to include
    CONSUMER_ISP_DOMAINS = {
        # US ISPs
        "comcast.net",
        "xfinity.com",
        "cox.net",
        "charter.net",
        "spectrum.net",
        "att.net",
        "sbcglobal.net",
        "verizon.net",
        "aol.com",
        "aim.com",
        # UK ISPs
        "btinternet.com",
        "virginmedia.com",
        "sky.com",
        "talktalk.net",
        "plusnet.com",
        "btopenworld.com",
        "ntlworld.com",
        # Canadian ISPs
        "shaw.ca",
        "telus.net",
        "rogers.com",
        "bell.net",
        "videotron.ca",
        # Australian ISPs
        "bigpond.com",
        "bigpond.net.au",
        "optusnet.com.au",
        "tpg.com.au",
        # Global providers
        "gmail.com",
        "yahoo.com",
        "yahoo.co.uk",
        "yahoo.fr",
        "yahoo.de",
        "hotmail.com",
        "outlook.com",
        "live.com",
        "msn.com",
        "icloud.com",
        "me.com",
        "mac.com",
        "protonmail.com",
        "proton.me",
        "tutanota.com",
        "gmx.com",
        "gmx.net",
        "mail.com",
        "email.com",
        "usa.com",
        "yandex.com",
        "yandex.ru",
        "mail.ru",
        "rambler.ru",
        "qq.com",
        "163.com",
        "126.com",
        "sina.com",
        "sohu.com",
        "naver.com",
        "daum.net",
        "hanmail.net",
    }

    def __init__(self, output_dir: Path = Path("resources")):
        self.output_dir = Path(output_dir)
        self.output_dir.mkdir(parents=True, exist_ok=True)
        self.psl = publicsuffix2.PublicSuffixList()

    def fetch_domain_list(self, source_name: str) -> Set[str]:
        """Fetch domains from a source URL."""
        source = self.SOURCES[source_name]
        log.info(f"Fetching {source_name}: {source['description']}")

        try:
            response = requests.get(source["url"], timeout=30)
            response.raise_for_status()

            # Parse domains (one per line)
            domains = set()
            for line in response.text.strip().split("\n"):
                domain = line.strip().lower()
                if domain and not domain.startswith("#"):
                    domains.add(domain)

            log.info(f"  Retrieved {len(domains):,} domains from {source_name}")
            return domains

        except Exception as e:
            log.error(f"  Failed to fetch {source_name}: {e}")
            return set()

    def normalize_domain(self, domain: str) -> str:
        """Normalize a domain to its public suffix form."""
        domain = domain.lower().strip()

        # Remove wildcards and subdomains
        domain = domain.replace("*.", "").lstrip(".")

        # Get the public suffix (e.g., gmail.com from mail.gmail.com)
        public_suffix = self.psl.get_public_suffix(domain)

        return public_suffix if public_suffix else domain

    def is_valid_domain(self, domain: str) -> bool:
        """Check if a domain is valid."""
        # Basic validation
        if not domain or len(domain) < 4:
            return False

        # Must have at least one dot
        if "." not in domain:
            return False

        # Check against regex pattern
        pattern = r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$"
        return bool(re.match(pattern, domain))

    def curate_domains(
        self, include_disposable: bool = False
    ) -> Tuple[Set[str], Dict[str, any]]:
        """
        Curate the final domain list.

        Args:
            include_disposable: If True, include disposable email domains

        Returns:
            Tuple of (domain_set, metadata_dict)
        """
        log.info("Starting domain curation process...")

        all_domains = set()
        source_counts = {}

        # Fetch from all sources
        for source_name in self.SOURCES:
            if not include_disposable and "disposable" in source_name:
                log.info(f"Skipping {source_name} (disposable domains excluded)")
                continue

            domains = self.fetch_domain_list(source_name)
            source_counts[source_name] = len(domains)
            all_domains.update(domains)

        # Add consumer ISP domains
        log.info(f"Adding {len(self.CONSUMER_ISP_DOMAINS)} known consumer ISP domains")
        all_domains.update(self.CONSUMER_ISP_DOMAINS)

        # Normalize all domains
        log.info("Normalizing domains...")
        normalized = set()

        for domain in all_domains:
            norm_domain = self.normalize_domain(domain)
            if self.is_valid_domain(norm_domain):
                normalized.add(norm_domain)

        log.info(f"Normalized {len(all_domains):,} -> {len(normalized):,} domains")

        # Create metadata
        metadata = {
            "version": datetime.now().strftime("%Y%m%d"),
            "total_domains": len(normalized),
            "sources": source_counts,
            "include_disposable": include_disposable,
            "creation_time": datetime.now().isoformat(),
        }

        return normalized, metadata

    def save_domain_list(self, domains: Set[str], metadata: Dict[str, any]) -> Path:
        """Save the curated domain list and metadata."""
        version = metadata["version"]

        # Save domain list (one per line)
        list_path = self.output_dir / f"b2c_domains_v{version}.txt"
        with open(list_path, "w") as f:
            for domain in sorted(domains):
                f.write(f"{domain}\n")

        # Calculate SHA256
        with open(list_path, "rb") as f:
            sha256 = hashlib.sha256(f.read()).hexdigest()

        metadata["sha256"] = sha256
        metadata["filename"] = list_path.name

        # Save metadata
        manifest_path = self.output_dir / "manifest.json"
        existing_manifest = {}

        if manifest_path.exists():
            with open(manifest_path, "r") as f:
                existing_manifest = json.load(f)

        existing_manifest[version] = metadata

        with open(manifest_path, "w") as f:
            json.dump(existing_manifest, f, indent=2)

        log.info(f"Saved {len(domains):,} domains to {list_path}")
        log.info(f"SHA256: {sha256}")

        return list_path

    def quality_check(self, domains: Set[str]) -> Dict[str, any]:
        """Perform quality checks on the domain list."""
        log.info("Running quality checks...")

        checks = {
            "total_count": len(domains),
            "has_gmail": "gmail.com" in domains,
            "has_yahoo": "yahoo.com" in domains,
            "has_outlook": "outlook.com" in domains,
            "shortest_domain": min(domains, key=len) if domains else "",
            "longest_domain": max(domains, key=len) if domains else "",
            "avg_length": sum(len(d) for d in domains) / len(domains) if domains else 0,
        }

        # Check for suspicious patterns
        suspicious = []
        for domain in domains:
            if len(domain) < 4 or len(domain) > 50:
                suspicious.append(domain)

        checks["suspicious_count"] = len(suspicious)
        checks["suspicious_samples"] = suspicious[:10]

        return checks


def main():
    """Main curation workflow."""
    import argparse

    parser = argparse.ArgumentParser(description="Curate B2C email domain list")
    parser.add_argument("--output-dir", default="resources", help="Output directory")
    parser.add_argument(
        "--include-disposable",
        action="store_true",
        help="Include disposable email domains",
    )
    parser.add_argument(
        "--quality-report", action="store_true", help="Generate quality report"
    )
    args = parser.parse_args()

    curator = B2CDomainCurator(output_dir=Path(args.output_dir))

    # Curate domains
    domains, metadata = curator.curate_domains(
        include_disposable=args.include_disposable
    )

    # Quality check
    if args.quality_report:
        quality = curator.quality_check(domains)
        log.info("Quality Report:")
        for key, value in quality.items():
            log.info(f"  {key}: {value}")

    # Save
    output_path = curator.save_domain_list(domains, metadata)
    log.info(f"✓ Domain curation complete: {output_path}")


if __name__ == "__main__":
    main()
