"""Fallback strategies for blocked sites — tested & working alternatives."""
import re
import requests
import xml.etree.ElementTree as ET
import urllib.parse

UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"

# Tested results 2026-02-19:
# ✅ Google News RSS — works for Bloomberg, Reuters, Indeed, Glassdoor  
# ✅ Startpage — works for Crunchbase, BestBuy, Target, SimilarWeb, G2, Redfin
# ❌ Walmart typeahead — connection refused
# ❌ Yelp HTML — 403 blocked

FALLBACKS = {
    # News/Finance → Google News RSS
    "bloomberg.com": {"strategy": "google_news_rss", "query": "bloomberg markets today"},
    "reuters.com": {"strategy": "google_news_rss", "query": "reuters breaking news"},
    # Jobs → Google News RSS with site filter
    "indeed.com": {"strategy": "google_news_rss_site"},
    "glassdoor.com": {"strategy": "google_news_rss_site"},
    # E-commerce → Brave Search
    "walmart.com": {"strategy": "brave_search"},
    "bestbuy.com": {"strategy": "brave_search"},
    "target.com": {"strategy": "brave_search"},
    # Business data → Brave Search
    "crunchbase.com": {"strategy": "brave_search"},
    "similarweb.com": {"strategy": "brave_search"},
    "g2.com": {"strategy": "brave_or_gnews", "query": "g2 software reviews"},
    # Real estate → Brave, fallback to Google News
    "redfin.com": {"strategy": "brave_or_gnews", "query": "redfin homes for sale"},
    "zillow.com": {"strategy": "brave_or_gnews", "query": "zillow real estate listings"},
    # Directories → Brave, fallback to Google News
    "yellowpages.com": {"strategy": "brave_or_gnews", "query": "local business directory"},
    # SEO → Brave Search
    "semrush.com": {"strategy": "brave_search"},
    # Reviews → Brave, fallback to Google News
    "capterra.com": {"strategy": "brave_or_gnews", "query": "capterra software reviews"},
    "g2.com": {"strategy": "brave_or_gnews", "query": "g2 software reviews"},
}


def get_fallback(domain):
    domain = domain.lower().removeprefix("www.")
    return FALLBACKS.get(domain)


def execute_fallback(domain, url, client=None):
    """Execute fallback for a blocked domain. Returns standard result dict."""
    domain = domain.lower().removeprefix("www.")
    fb = FALLBACKS.get(domain)
    if not fb:
        return _fail(domain, "No fallback available")

    strategy = fb["strategy"]
    try:
        if strategy == "google_news_rss":
            return _google_news_rss(fb.get("query", domain.replace(".com", "")), domain, client=client)
        elif strategy == "google_news_rss_site":
            query = _extract_query(url, domain)
            return _google_news_rss(query, domain, site_filter=domain, client=client)
        elif strategy == "brave_search":
            query = _extract_query(url, domain)
            return _brave_search(query, domain, client=client)
        elif strategy == "brave_or_gnews":
            query = fb.get("query", _extract_query(url, domain))
            result = _brave_search(query, domain, client=client)
            if result.get("success"):
                return result
            # Brave rate limited — fall back to Google News RSS
            return _google_news_rss(query, domain, client=client)
        elif strategy == "startpage":
            query = _extract_query(url, domain)
            return _startpage_search(query, domain, client=client)
        else:
            return _fail(domain, f"Unknown strategy: {strategy}")
    except Exception as e:
        return _fail(domain, str(e))


def _fail(domain, error):
    return {"success": False, "data": {}, "source": "fallback", "fallback_used": True,
            "original_site": domain, "error": error}


def _extract_query(url, domain):
    """Extract a meaningful search query from URL."""
    # Try query params
    m = re.search(r'[?&](?:q|query|search_terms|find_desc)=([^&]+)', url)
    if m:
        return urllib.parse.unquote_plus(m.group(1))
    # Use path segments
    path = urllib.parse.urlparse(url).path.strip("/")
    if path:
        # Take last meaningful segment
        parts = [p for p in path.split("/") if p and p not in ("www", "search", "review", "overview")]
        if parts:
            return parts[-1].replace("-", " ").replace("_", " ")
    return domain.replace(".com", "")


def _fetch(client, url, timeout=10):
    """ALWAYS route through proxy — every request = revenue."""
    if client:
        return client.fetch(url, timeout=timeout)
    raise RuntimeError("No client provided — cannot make request without proxy")


def _brave_search(query, domain, client=None):
    """Search Brave — works for Crunchbase, BestBuy, Target, SimilarWeb, G2, Redfin, Zillow, etc."""
    import time
    search = f"site:{domain} {query}"
    brave_url = f"https://search.brave.com/search?q={urllib.parse.quote(search)}"

    # Rate limit: Brave returns 429 if too fast
    time.sleep(1)
    resp = _fetch(client, brave_url)
    if resp.status_code == 429:
        time.sleep(3)
        resp = _fetch(client, brave_url)
    if resp.status_code != 200:
        return _fail(domain, f"Brave HTTP {resp.status_code}")

    results = []
    # Brave uses data-type="web" blocks with <a href> and <span> for titles
    matches = re.findall(
        r'data-type="web".*?<a\s+href="(https?://[^"]+)"[^>]*>.*?<span[^>]*>([^<]+)</span>',
        resp.text, re.S
    )
    for url_match, title in matches[:10]:
        title = title.strip()
        if title and len(title) > 2:
            results.append({"title": title, "url": url_match})

    # Also try to extract snippets
    snippets = re.findall(
        r'data-type="web".*?<div class="snippet-description[^"]*"[^>]*>(.*?)</div>',
        resp.text, re.S
    )
    for i, snip in enumerate(snippets[:10]):
        clean = re.sub(r'<[^>]+>', '', snip).strip()
        if i < len(results) and clean:
            results[i]["snippet"] = clean[:200]

    return {
        "success": len(results) > 0,
        "data": {"query": query, "count": len(results), "results": results},
        "source": "brave-search",
        "fallback_used": True,
        "original_site": domain,
        "error": None if results else "No results found"
    }


def _google_news_rss(query, domain, site_filter=None, client=None):
    """Fetch Google News RSS — tested working for Bloomberg, Reuters, Indeed, Glassdoor."""
    search = f"site:{site_filter} {query}" if site_filter else query
    rss_url = f"https://news.google.com/rss/search?q={urllib.parse.quote(search)}&hl=en-US&gl=US&ceid=US:en"
    
    resp = _fetch(client, rss_url)
    if resp.status_code != 200:
        return _fail(domain, f"RSS HTTP {resp.status_code}")

    articles = []
    try:
        root = ET.fromstring(resp.content)
        for item in root.iter("item"):
            title = item.find("title")
            link = item.find("link")
            pub = item.find("pubDate")
            source = item.find("source")
            if title is not None and title.text:
                articles.append({
                    "title": title.text,
                    "url": link.text if link is not None else "",
                    "published": pub.text if pub is not None else "",
                    "source": source.text if source is not None else "",
                })
            if len(articles) >= 15:
                break
    except ET.ParseError:
        return _fail(domain, "RSS parse error")

    return {
        "success": len(articles) > 0,
        "data": {"query": query, "count": len(articles), "articles": articles},
        "source": "google-news-rss",
        "fallback_used": True,
        "original_site": domain,
        "error": None if articles else "No articles found"
    }


def _startpage_search(query, domain, client=None):
    """Search Startpage — tested working for Crunchbase, BestBuy, Target, SimilarWeb, G2, Redfin."""
    search = f"site:{domain} {query}" if domain else query
    sp_url = f"https://www.startpage.com/do/search?q={urllib.parse.quote(search)}"
    
    resp = _fetch(client, sp_url)
    if resp.status_code != 200:
        return _fail(domain, f"Startpage HTTP {resp.status_code}")

    results = []
    # Parse Startpage results — h3 tags contain titles
    h3s = re.findall(r'<h3[^>]*>([^<]+)</h3>', resp.text)
    # Also try to get URLs
    links = re.findall(r'<a[^>]*href="(https?://[^"]*' + re.escape(domain) + r'[^"]*)"', resp.text)
    
    for i, title in enumerate(h3s[:10]):
        title = title.strip()
        if title and len(title) > 3:
            results.append({
                "title": title,
                "url": links[i] if i < len(links) else "",
            })

    # If no h3s matched, try broader pattern
    if not results:
        for m in re.finditer(r'class="[^"]*result[^"]*"[^>]*>.*?<a[^>]*href="([^"]+)"[^>]*>([^<]+)', resp.text, re.S):
            results.append({"title": m.group(2).strip(), "url": m.group(1)})
            if len(results) >= 10:
                break

    return {
        "success": len(results) > 0,
        "data": {"query": query, "count": len(results), "results": results},
        "source": "startpage-search",
        "fallback_used": True,
        "original_site": domain,
        "error": None if results else "No results found"
    }
