#!/usr/bin/env python3
"""
Broken Link Finder for Career Center Pages
Crawls target URLs, finds outbound links, checks for 404s
Outputs to CSV with contact information from Hunter.io

Usage: python main.py --input urls.csv --output results
"""

import asyncio
import aiohttp
import csv
import json
import os
import re
import sys
import time
from dataclasses import dataclass, asdict, field
from datetime import datetime
from typing import List, Optional, Tuple, Dict
from urllib.parse import urljoin, urlparse

from bs4 import BeautifulSoup

# Configuration
MAX_CONCURRENT_REQUESTS = 5  # Be polite to servers
REQUEST_TIMEOUT = 30
DELAY_BETWEEN_REQUESTS = 1.0  # seconds
USER_AGENT = "Mozilla/5.0 (compatible; BrokenLinkChecker/1.0; +https://revarta.com)"
PROCESSED_URLS_FILE = "output/processed-urls.json"

# Email pattern for extraction
EMAIL_PATTERN = re.compile(r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}')

@dataclass
class BrokenLink:
    source_url: str
    source_university: str
    broken_url: str
    anchor_text: str
    surrounding_context: str
    status_code: int
    error_message: str
    found_at: str
    contact_email: str = ""

@dataclass
class CrawlResult:
    url: str
    university: str
    total_links: int
    broken_links: List[BrokenLink]
    crawl_time: float


# ============ Processed URL Tracking ============

def load_processed_urls(filepath: str = PROCESSED_URLS_FILE) -> set:
    """Load set of already-processed URLs from JSON file"""
    if not os.path.exists(filepath):
        return set()
    try:
        with open(filepath, 'r') as f:
            data = json.load(f)
            return set(data.get('processed', []))
    except (json.JSONDecodeError, IOError):
        return set()


def save_processed_url(url: str, filepath: str = PROCESSED_URLS_FILE):
    """Add a URL to the processed URLs file"""
    # Ensure directory exists
    output_dir = os.path.dirname(filepath)
    if output_dir:
        os.makedirs(output_dir, exist_ok=True)

    # Load existing data
    if os.path.exists(filepath):
        try:
            with open(filepath, 'r') as f:
                data = json.load(f)
        except (json.JSONDecodeError, IOError):
            data = {'processed': []}
    else:
        data = {'processed': []}

    # Add URL if not already present
    if url not in data['processed']:
        data['processed'].append(url)

    data['last_updated'] = datetime.now().isoformat()

    with open(filepath, 'w') as f:
        json.dump(data, f, indent=2)


# ============ Contact Email Extraction ============

def extract_contact_emails(html: str, base_domain: str) -> List[str]:
    """Extract contact emails from page HTML, prioritizing career-related emails"""
    soup = BeautifulSoup(html, 'html.parser')
    emails = set()

    # 1. Extract from mailto: links (highest priority)
    for anchor in soup.find_all('a', href=True):
        href = anchor['href']
        if href.startswith('mailto:'):
            email = href.replace('mailto:', '').split('?')[0].strip().lower()
            if '@' in email and not email.endswith('.png') and not email.endswith('.jpg'):
                emails.add(email)

    # 2. Extract emails from page text using regex
    page_text = soup.get_text()
    found_emails = EMAIL_PATTERN.findall(page_text)
    for email in found_emails:
        email = email.lower()
        # Filter out image files and common non-contact emails
        if not any(email.endswith(ext) for ext in ['.png', '.jpg', '.gif', '.svg']):
            if not any(skip in email for skip in ['example.com', 'yoursite', 'domain.com']):
                emails.add(email)

    if not emails:
        return []

    # 3. Score and prioritize emails
    def score_email(email: str) -> int:
        score = 0
        email_lower = email.lower()

        # Prioritize career-related emails
        career_keywords = ['career', 'employment', 'job', 'recruit', 'hire', 'talent', 'work']
        for keyword in career_keywords:
            if keyword in email_lower:
                score += 100

        # Prioritize emails from the same domain
        if base_domain in email_lower:
            score += 50

        # Deprioritize generic/personal emails
        generic = ['info@', 'contact@', 'hello@', 'support@', 'help@', 'admin@']
        for prefix in generic:
            if email_lower.startswith(prefix):
                score -= 10

        # Deprioritize common free email providers
        free_providers = ['gmail.com', 'yahoo.com', 'hotmail.com', 'outlook.com']
        for provider in free_providers:
            if provider in email_lower:
                score -= 50

        return score

    # Sort by score (highest first)
    sorted_emails = sorted(emails, key=score_email, reverse=True)
    return sorted_emails


class BrokenLinkFinder:
    def __init__(self, delay: float = DELAY_BETWEEN_REQUESTS):
        self.delay = delay
        self.session: Optional[aiohttp.ClientSession] = None
        self.results: List[BrokenLink] = []
        
    async def __aenter__(self):
        timeout = aiohttp.ClientTimeout(total=REQUEST_TIMEOUT)
        self.session = aiohttp.ClientSession(
            timeout=timeout,
            headers={"User-Agent": USER_AGENT}
        )
        return self
        
    async def __aexit__(self, exc_type, exc_val, exc_tb):
        if self.session:
            await self.session.close()
            
    async def fetch_page(self, url: str) -> Tuple[Optional[str], int, str]:
        """Fetch a page and return (html, status_code, error_message)"""
        try:
            async with self.session.get(url, allow_redirects=True) as response:
                if response.status == 200:
                    html = await response.text()
                    return html, response.status, ""
                else:
                    return None, response.status, f"HTTP {response.status}"
        except asyncio.TimeoutError:
            return None, 0, "Timeout"
        except aiohttp.ClientError as e:
            return None, 0, str(e)
        except Exception as e:
            return None, 0, f"Error: {str(e)}"
            
    async def check_link(self, url: str) -> Tuple[int, str]:
        """Check if a link is broken. Returns (status_code, error_message)"""
        try:
            async with self.session.head(url, allow_redirects=True) as response:
                return response.status, ""
        except asyncio.TimeoutError:
            return 0, "Timeout"
        except aiohttp.ClientError as e:
            return 0, str(e)
        except Exception as e:
            return 0, f"Error: {str(e)}"
            
    def extract_links(self, html: str, base_url: str) -> List[dict]:
        """Extract all outbound links from HTML with context"""
        soup = BeautifulSoup(html, 'html.parser')
        links = []
        base_domain = urlparse(base_url).netloc
        
        for anchor in soup.find_all('a', href=True):
            href = anchor['href']
            
            # Skip internal links, javascript, mailto, tel, etc.
            if href.startswith('#'):
                continue
            if href.startswith('javascript:'):
                continue
            if href.startswith('mailto:'):
                continue
            if href.startswith('tel:'):
                continue
                
            # Resolve relative URLs
            full_url = urljoin(base_url, href)
            parsed = urlparse(full_url)
            
            # Skip if not HTTP/HTTPS
            if parsed.scheme not in ('http', 'https'):
                continue
                
            # Skip internal links (same domain)
            if parsed.netloc == base_domain:
                continue
            # Also skip .edu links within same institution
            if '.edu' in parsed.netloc and base_domain.split('.')[-2] in parsed.netloc:
                continue
                
            # Get anchor text
            anchor_text = anchor.get_text(strip=True)[:200]
            
            # Get surrounding context
            parent = anchor.parent
            context = ""
            if parent:
                context = parent.get_text(strip=True)[:300]
                
            links.append({
                'url': full_url,
                'anchor_text': anchor_text,
                'context': context
            })
            
        return links
        
    async def crawl_page(self, url: str, university: str) -> CrawlResult:
        """Crawl a single page and find broken links"""
        print(f"[{datetime.now().strftime('%H:%M:%S')}] Crawling: {url}")
        start_time = time.time()
        broken_links = []
        
        # Fetch the page
        html, status, error = await self.fetch_page(url)
        
        if not html:
            print(f"  ⚠️  Failed to fetch page: {error}")
            return CrawlResult(
                url=url,
                university=university,
                total_links=0,
                broken_links=[],
                crawl_time=time.time() - start_time
            )
            
        # Extract links
        links = self.extract_links(html, url)
        print(f"  📎 Found {len(links)} outbound links")

        # Extract contact emails from the page
        base_domain = urlparse(url).netloc
        contact_emails = extract_contact_emails(html, base_domain)
        contact_email = contact_emails[0] if contact_emails else ""
        if contact_email:
            print(f"  📧 Found contact: {contact_email}")

        # Check each link
        for i, link_data in enumerate(links):
            link_url = link_data['url']

            # Add delay between requests
            if i > 0:
                await asyncio.sleep(self.delay)

            status_code, error_msg = await self.check_link(link_url)

            # Consider 4xx and 5xx as broken
            is_broken = status_code >= 400 or status_code == 0

            if is_broken:
                print(f"  ❌ Broken ({status_code}): {link_url[:60]}...")
                broken_link = BrokenLink(
                    source_url=url,
                    source_university=university,
                    broken_url=link_url,
                    anchor_text=link_data['anchor_text'],
                    surrounding_context=link_data['context'],
                    status_code=status_code,
                    error_message=error_msg,
                    found_at=datetime.now().isoformat(),
                    contact_email=contact_email
                )
                broken_links.append(broken_link)
            else:
                print(f"  ✓ OK ({status_code}): {link_url[:60]}...")
                
        crawl_time = time.time() - start_time
        print(f"  ⏱️  Completed in {crawl_time:.1f}s, found {len(broken_links)} broken links")
        
        return CrawlResult(
            url=url,
            university=university,
            total_links=len(links),
            broken_links=broken_links,
            crawl_time=crawl_time
        )


def load_targets(filepath: str) -> List[dict]:
    """Load target URLs from CSV"""
    targets = []
    with open(filepath, 'r', encoding='utf-8') as f:
        reader = csv.DictReader(f)
        for row in reader:
            targets.append({
                'url': row['Target URL'],
                'university': row['University'],
                'tier': row.get('Tier', ''),
                'category': row.get('Category', '')
            })
    return targets


def save_results(broken_links: List[BrokenLink], output_path: str):
    """Save results to CSV"""
    if not broken_links:
        print("No broken links found.")
        return

    # Ensure output directory exists
    output_dir = os.path.dirname(output_path)
    if output_dir:
        os.makedirs(output_dir, exist_ok=True)

    fieldnames = [
        'source_university',
        'source_url',
        'broken_url',
        'anchor_text',
        'surrounding_context',
        'status_code',
        'error_message',
        'found_at',
        'contact_email'
    ]
    
    with open(output_path, 'w', newline='', encoding='utf-8') as f:
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        writer.writeheader()
        for link in broken_links:
            writer.writerow(asdict(link))
            
    print(f"\n✅ Results saved to: {output_path}")
    print(f"   Total broken links found: {len(broken_links)}")


async def main():
    import argparse

    parser = argparse.ArgumentParser(description='Find broken links on career center pages')
    parser.add_argument('--input', '-i', default='broken-link-targets.csv', help='Input CSV with target URLs')
    parser.add_argument('--output', '-o', default='output/broken-links-results.csv', help='Output CSV path')
    parser.add_argument('--delay', '-d', type=float, default=1.0, help='Delay between requests (seconds)')
    parser.add_argument('--limit', '-l', type=int, default=None, help='Limit number of pages to crawl')
    parser.add_argument('--force', '-f', action='store_true', help='Re-process URLs even if already processed')
    args = parser.parse_args()

    # Load targets
    if not os.path.exists(args.input):
        print(f"Error: Input file not found: {args.input}")
        sys.exit(1)

    targets = load_targets(args.input)
    print(f"\n🎯 Loaded {len(targets)} target URLs")

    # Filter out already-processed URLs (unless --force)
    if not args.force:
        processed_urls = load_processed_urls()
        original_count = len(targets)
        targets = [t for t in targets if t['url'] not in processed_urls]
        skipped = original_count - len(targets)
        if skipped > 0:
            print(f"   ⏭️  Skipping {skipped} already-processed URLs")

    if args.limit:
        targets = targets[:args.limit]
        print(f"   (Limited to {args.limit} pages)")

    if not targets:
        print("\n✅ No new URLs to process!")
        return

    # Crawl pages
    all_broken_links = []

    async with BrokenLinkFinder(delay=args.delay) as finder:
        for target in targets:
            result = await finder.crawl_page(target['url'], target['university'])
            all_broken_links.extend(result.broken_links)

            # Mark URL as processed
            save_processed_url(target['url'])

            # Add delay between pages
            await asyncio.sleep(args.delay * 2)

    # Save results
    save_results(all_broken_links, args.output)

    # Summary
    print(f"\n📊 Summary:")
    print(f"   Pages crawled: {len(targets)}")
    print(f"   Broken links found: {len(all_broken_links)}")

    # Group by university
    by_university = {}
    for link in all_broken_links:
        uni = link.source_university
        if uni not in by_university:
            by_university[uni] = []
        by_university[uni].append(link)

    print(f"\n   By university:")
    for uni, links in sorted(by_university.items(), key=lambda x: -len(x[1])):
        print(f"   - {uni}: {len(links)} broken links")


if __name__ == "__main__":
    asyncio.run(main())
