"""Main IPLoop client."""

import time
import logging
import requests

from .fingerprint import chrome_fingerprint
from .retry import retry_request, new_session_id
from .support import SupportClient
from .exceptions import AuthError, ProxyError, TimeoutError
from .cookies import CookieManager
from .tls_fingerprint import tls_request, get_engine as tls_engine

logger = logging.getLogger("iploop")


class StickySession:
    """A session that reuses the same proxy IP."""

    def __init__(self, client, session_id, country=None, city=None, lifetime=None):
        self._client = client
        self.session_id = session_id
        self.country = country or client._country
        self.city = city or client._city
        self.lifetime = lifetime

    def fetch(self, url, **kwargs):
        kwargs.setdefault("country", self.country)
        kwargs.setdefault("city", self.city)
        kwargs["session"] = self.session_id
        kwargs["sesstype"] = "sticky"
        if self.lifetime:
            kwargs["lifetime"] = self.lifetime
        kwargs["_no_rotate"] = True
        return self._client.fetch(url, **kwargs)

    def get(self, url, **kwargs):
        return self.fetch(url, method="GET", **kwargs)

    def post(self, url, data=None, json=None, **kwargs):
        kwargs["method"] = "POST"
        kwargs["data"] = data or json
        return self.fetch(url, **kwargs)

    def put(self, url, data=None, json=None, **kwargs):
        kwargs["method"] = "PUT"
        kwargs["data"] = data or json
        return self.fetch(url, **kwargs)

    def delete(self, url, **kwargs):
        kwargs["method"] = "DELETE"
        return self.fetch(url, **kwargs)


class IPLoop:
    """Residential proxy SDK — one-liner web fetching through millions of real IPs."""

    def __init__(self, api_key, country=None, city=None, debug=False, auto_render=False, cookie_path=None):
        if not api_key:
            raise AuthError("API key is required")
        self.api_key = api_key
        self.base_proxy = "proxy.iploop.io:8880"
        self.api_base = "https://gateway.iploop.io:9443"
        self._country = country
        self._city = city
        self.auto_render = auto_render
        self._renderer = None
        self._support = SupportClient(api_key, self.api_base)
        self._stats = {"requests": 0, "success": 0, "errors": 0, "total_time": 0}
        self._cookies = CookieManager(persist_path=cookie_path)
        self._tls_engine = tls_engine()

        if debug:
            logging.basicConfig(level=logging.DEBUG)
            logger.setLevel(logging.DEBUG)

    @property
    def stats(self):
        """Get request statistics."""
        avg = self._stats["total_time"] / max(self._stats["requests"], 1)
        return {
            **self._stats,
            "avg_time": round(avg, 2),
            "success_rate": round(self._stats["success"] / max(self._stats["requests"], 1) * 100, 1)
        }

    def _build_proxy_auth(self, country=None, city=None, session=None, render=False,
                          sesstype=None, lifetime=None):
        parts = [self.api_key]
        c = country or self._country
        if c:
            parts.append(f"country-{c.lower()}")
        ci = city or self._city
        if ci:
            parts.append(f"city-{ci.lower()}")
        if sesstype:
            parts.append(f"sesstype-{sesstype}")
        if session:
            parts.append(f"session-{session}")
        if lifetime:
            parts.append(f"lifetime-{lifetime}")
        if render:
            parts.append("render-1")
        return "-".join(parts)

    def _proxy_url(self, country=None, city=None, session=None, render=False,
                   sesstype=None, lifetime=None):
        auth = self._build_proxy_auth(country=country, city=city, session=session,
                                      render=render, sesstype=sesstype, lifetime=lifetime)
        return {
            "http": f"http://user:{auth}@{self.base_proxy}",
            "https": f"http://user:{auth}@{self.base_proxy}",
        }

    def fetch(self, url, country=None, city=None, session=None, render=False,
              headers=None, method="GET", data=None, timeout=15, retries=3,
              _no_rotate=False, sesstype=None, lifetime=None):
        """Fetch a URL through residential proxy with auto-retry and smart headers."""
        c = country or self._country or "US"
        # Phase 9: auto-apply 14-header Chrome fingerprint
        h = chrome_fingerprint(c)
        if headers:
            h.update(headers)

        def do_request(attempt):
            sid = session if _no_rotate else (session or new_session_id())
            proxies = self._proxy_url(country=country, city=city, session=sid, render=render,
                                      sesstype=sesstype, lifetime=lifetime)
            req_start = time.time()
            # Use TLS fingerprint engine if available (curl_cffi/tls_client)
            resp = tls_request(method, url, headers=h, proxies=proxies,
                               data=data, timeout=timeout, cookies=self._cookies.jar)
            elapsed = time.time() - req_start
            # Save cookies from response
            if hasattr(resp, 'cookies'):
                for cookie in resp.cookies:
                    self._cookies.jar.set_cookie(cookie)
                self._cookies.save()
            logger.debug("%-3s %s → %d (%.2fs) country=%s session=%s tls=%s",
                         method, url, resp.status_code, elapsed, c, sid, self._tls_engine)
            return resp

        self._stats["requests"] += 1
        start = time.time()
        try:
            resp = retry_request(do_request, retries=retries)
            self._stats["success"] += 1
            self._stats["total_time"] += time.time() - start
            return resp
        except requests.exceptions.Timeout:
            self._stats["errors"] += 1
            raise TimeoutError(f"All {retries} retries timed out for {url}")
        except (requests.exceptions.ConnectionError, requests.exceptions.ProxyError) as e:
            self._stats["errors"] += 1
            raise ProxyError(f"Proxy connection failed after {retries} retries: {e}")
        except Exception:
            self._stats["errors"] += 1
            raise

    def get(self, url, **kwargs):
        """GET request through proxy."""
        return self.fetch(url, method="GET", **kwargs)

    def post(self, url, data=None, json=None, **kwargs):
        """POST request through proxy."""
        import json as json_mod
        if json is not None:
            kwargs.setdefault("headers", {})["Content-Type"] = "application/json"
            data = json_mod.dumps(json)
        return self.fetch(url, method="POST", data=data, **kwargs)

    def put(self, url, data=None, json=None, **kwargs):
        """PUT request through proxy."""
        import json as json_mod
        if json is not None:
            kwargs.setdefault("headers", {})["Content-Type"] = "application/json"
            data = json_mod.dumps(json)
        return self.fetch(url, method="PUT", data=data, **kwargs)

    def delete(self, url, **kwargs):
        """DELETE request through proxy."""
        return self.fetch(url, method="DELETE", **kwargs)

    def session(self, session_id=None, country=None, city=None, lifetime=None):
        """Create a sticky session that reuses the same proxy IP.
        
        Args:
            session_id: Custom session ID (auto-generated if not provided)
            country: Target country code (e.g. "US", "DE")
            city: Target city (e.g. "miami", "london")
            lifetime: Session lifetime in minutes (e.g. 30)
        """
        return StickySession(self, session_id or new_session_id(), country, city, lifetime=lifetime)

    def batch(self, max_workers=10):
        """Create a batch fetcher for concurrent requests. Safe up to 25 workers."""
        from .concurrent import BatchFetcher
        return BatchFetcher(self, max_workers)

    def fingerprint(self, country="US"):
        """Get Chrome desktop fingerprint headers for a country."""
        return chrome_fingerprint(country)

    def usage(self):
        """Check bandwidth usage and quota."""
        return self._support.usage()

    def status(self):
        """Check service status."""
        return self._support.status()

    def ask(self, question):
        """Ask the support API a question."""
        return self._support.ask(question)

    def countries(self):
        """List available proxy countries."""
        return self._support.countries()

    def request_preset(self, url, data_needed=None):
        """Request a new site preset. We'll add it and notify you when ready.
        
        Args:
            url: The URL/domain you need scraped
            data_needed: What data you need (e.g. "prices, titles, reviews")
        """
        msg = f"PRESET_REQUEST: {url}"
        if data_needed:
            msg += f" | NEED: {data_needed}"
        try:
            return self._support.ask(msg)
        except Exception:
            logger.debug(f"Preset request failed for {url}")
            return {"status": "queued_locally", "url": url}

    def check_updates(self):
        """Check if a newer SDK version is available."""
        try:
            import requests as _req
            resp = _req.get("https://pypi.org/pypi/iploop/json", timeout=5)
            if resp.status_code == 200:
                latest = resp.json()["info"]["version"]
                from importlib.metadata import version
                current = version("iploop")
                if latest != current:
                    return {"update_available": True, "current": current, "latest": latest,
                            "install": f"pip install iploop=={latest}"}
                return {"update_available": False, "current": current}
        except Exception:
            return {"update_available": False, "error": "Could not check"}

    # ── Unified scrape() API ─────────────────────────────────

    SITE_MAP = {
        "youtube.com": "youtube", "youtu.be": "youtube",
        "reddit.com": "reddit", "old.reddit.com": "reddit",
        "amazon.com": "amazon", "amazon.co.uk": "amazon",
        "linkedin.com": "linkedin",
        "ebay.com": "ebay",
        "twitter.com": "twitter", "x.com": "twitter",
        "tiktok.com": "tiktok",
        "instagram.com": "instagram",
        "google.com": "google",
        "nasdaq.com": "nasdaq", "finance.yahoo.com": "nasdaq",
        "en.wikipedia.org": "wikipedia", "wikipedia.org": "wikipedia",
        "news.ycombinator.com": "hackernews",
        "github.com": "github",
        "coingecko.com": "coingecko",
        "wttr.in": "weather",
        "open.spotify.com": "spotify",
        "stackoverflow.com": "stackoverflow",
        "npmjs.com": "npm", "registry.npmjs.org": "npm",
        "pypi.org": "pypi",
        "imdb.com": "imdb", "www.imdb.com": "imdb",
        "xkcd.com": "xkcd",
        "api.exchangerate-api.com": "exchangerate",
        "api.spacexdata.com": "spacex",
        "pokeapi.co": "pokeapi",
        "store.steampowered.com": "steam", "steampowered.com": "steam",
        "goodreads.com": "goodreads", "www.goodreads.com": "goodreads",
        "archive.org": "archiveorg",
        "trustpilot.com": "trustpilot",
        "airbnb.com": "airbnb",
        "remoteok.com": "remoteok",
        "cnn.com": "cnn",
        "aliexpress.com": "aliexpress",
    }

    def _detect_site_or_shopify(self, url):
        """Detect site, with Shopify auto-detection for unknown domains."""
        site = self._detect_site(url)
        if site:
            return site
        # Check if it's a Shopify store
        if "/products.json" in url or "/collections.json" in url:
            return "shopify"
        from .sites.shopify import Shopify as ShopifyPreset
        if ShopifyPreset.is_shopify(url):
            return "shopify"
        return None

    def _detect_site(self, url):
        """Detect site from URL domain."""
        from urllib.parse import urlparse
        hostname = urlparse(url).hostname or ""
        hostname = hostname.lower().removeprefix("www.")
        # Try exact match first, then parent domain
        if hostname in self.SITE_MAP:
            return self.SITE_MAP[hostname]
        # Try matching suffix (e.g. m.youtube.com)
        for domain, site in self.SITE_MAP.items():
            if hostname == domain or hostname.endswith("." + domain):
                return site
        return None

    def scrape(self, url, country=None, **kwargs):
        """
        One-liner scrape: give a URL, get structured data back.

        Returns dict: {"success": bool, "site": str, "data": dict, "source": str, "error": str|None}
        """
        site = self._detect_site_or_shopify(url)
        try:
            if site == "youtube":
                return self._scrape_youtube(url)
            elif site == "reddit":
                return self._scrape_reddit(url)
            elif site == "nasdaq":
                return self._scrape_stocks(url)
            elif site == "google":
                return self._scrape_google(url, country)
            elif site == "amazon":
                return self._scrape_amazon(url, country)
            elif site == "linkedin":
                return self._scrape_linkedin(url, country)
            elif site == "ebay":
                if "_nkw=" in url or "/itm/" in url or "/sch/" in url:
                    result = self.ebay.search(url.split("_nkw=")[-1] if "_nkw=" in url else "electronics", extract=True)
                    products = result.get("products", [])
                    if products:
                        return {"success": True, "site": "ebay", "data": {"products": products, "count": len(products)}, "source": result.get("source", "http"), "error": None}
                # Homepage or non-search URL — use generic
                return self._scrape_generic(url, country)
            elif site == "twitter":
                return self._scrape_twitter(url, country)
            elif site == "tiktok":
                return self._scrape_tiktok(url, country)
            elif site == "instagram":
                return {"success": False, "site": "instagram", "data": {}, "source": "none", "error": "Login wall - requires auth cookies"}
            elif site in ("wikipedia", "hackernews", "github", "coingecko", "weather", "spotify", "stackoverflow", "npm", "pypi", "imdb", "xkcd", "exchangerate", "spacex", "pokeapi", "steam", "goodreads", "archiveorg", "trustpilot", "airbnb", "remoteok", "cnn", "aliexpress", "shopify"):
                try:
                    preset = getattr(self, site)
                    result = {**preset.extract(url), "site": site}
                    # Check if preset returned meaningful data
                    if result.get("success"):
                        return result
                    data = result.get("data", {})
                    if data and len(str(data)) > 50:
                        return result
                except Exception:
                    pass
                # Preset returned weak data — try generic scraper
                return self._scrape_generic(url, country)
            else:
                return self._scrape_generic(url, country)
        except Exception as e:
            return {"success": False, "site": site or "unknown", "data": {}, "source": "error", "error": str(e)}

    def _scrape_youtube(self, url):
        """YouTube via noembed oEmbed API."""
        resp = self.fetch(f"https://noembed.com/embed?url={url}", timeout=10)
        if resp.status_code == 200:
            d = resp.json()
            if d.get("title"):
                return {"success": True, "site": "youtube", "data": {
                    "title": d.get("title"), "author": d.get("author_name"),
                    "thumbnail": d.get("thumbnail_url"),
                }, "source": "oEmbed", "error": None}
        return {"success": False, "site": "youtube", "data": {}, "source": "oEmbed", "error": "oEmbed lookup failed"}

    def _scrape_reddit(self, url):
        """Reddit via .json API — try direct then proxy."""
        import re
        # Normalize: strip /hot /new /top etc for .json append
        clean = re.sub(r'/(hot|new|top|rising|controversial)/?$', '', url.rstrip("/"))
        json_url = clean + ".json"
        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}
        try:
            resp = self.fetch(json_url, country="US", headers=headers, timeout=15)
        except Exception as e:
            return {"success": False, "site": "reddit", "data": {}, "source": "json_api", "error": str(e)}
        if resp.status_code == 200:
            try:
                data = resp.json()
            except Exception:
                return {"success": False, "site": "reddit", "data": {}, "source": "json_api", "error": "Invalid JSON"}
            posts = []
            listing = data if isinstance(data, dict) else data[0] if isinstance(data, list) else {}
            if isinstance(listing, dict):
                children = listing.get("data", {}).get("children", [])
                for c in children[:25]:
                    p = c.get("data", {})
                    posts.append({"title": p.get("title", ""), "score": p.get("score", 0),
                                  "author": p.get("author", ""), "comments": p.get("num_comments", 0)})
            return {"success": len(posts) > 0, "site": "reddit", "data": {"posts": posts, "count": len(posts)}, "source": "json_api", "error": None}
        return {"success": False, "site": "reddit", "data": {}, "source": "json_api", "error": f"HTTP {resp.status_code}"}

    def _scrape_stocks(self, url):
        """Stocks via Yahoo Finance chart API."""
        import re
        # Extract symbol from URL
        m = re.search(r'/quote/([A-Z0-9.^]+)', url, re.IGNORECASE)
        symbol = m.group(1) if m else url.rstrip("/").split("/")[-1].upper()
        api_url = f"https://query1.finance.yahoo.com/v8/finance/chart/{symbol}?interval=1d&range=1d"
        resp = self.fetch(api_url, timeout=10, headers={"User-Agent": "Mozilla/5.0"})
        if resp.status_code == 200:
            chart = resp.json().get("chart", {}).get("result", [])
            if chart:
                meta = chart[0].get("meta", {})
                price = meta.get("regularMarketPrice")
                prev = meta.get("previousClose", 0)
                change = round(price - prev, 2) if price and prev else None
                pct = round(change / prev * 100, 2) if change and prev else None
                return {"success": True, "site": "stocks", "data": {
                    "symbol": symbol, "price": price, "change": change,
                    "change_percent": pct, "currency": meta.get("currency"),
                    "exchange": meta.get("exchangeName"),
                }, "source": "yahoo_api", "error": None}
        return {"success": False, "site": "stocks", "data": {"symbol": symbol}, "source": "yahoo_api", "error": "API lookup failed"}

    def _scrape_google(self, url, country=None):
        """Google via Startpage fallback with proxy."""
        import re, urllib.parse
        m = re.search(r'[?&]q=([^&]+)', url)
        query = urllib.parse.unquote_plus(m.group(1)) if m else url
        sp_url = f"https://www.startpage.com/do/search?q={urllib.parse.quote_plus(query)}"
        resp = self.fetch(sp_url, country=country or "US", timeout=15)
        results = []
        if resp.status_code == 200:
            for m in re.finditer(r'<a[^>]*class="[^"]*result[^"]*"[^>]*href="(https?://[^"]+)"[^>]*>.*?<h[23][^>]*>(.*?)</h[23]>', resp.text, re.DOTALL | re.IGNORECASE):
                title = re.sub(r'<[^>]+>', '', m.group(2)).strip()
                if title:
                    results.append({"title": title, "url": m.group(1)})
        return {"success": len(results) > 0, "site": "google", "data": {"query": query, "results": results[:10], "count": len(results)}, "source": "startpage", "error": None if results else "No results parsed"}

    def _scrape_amazon(self, url, country=None):
        """Amazon product page via proxy with Brave Search fallback."""
        import re, urllib.parse
        # Try direct proxy first
        resp = self.fetch(url, country=country or "US", timeout=20)
        data = {}
        source = "http_proxy"
        if resp.status_code == 200 and len(resp.text) > 10000:
            t = re.search(r'id="productTitle"[^>]*>([^<]+)', resp.text)
            if t: data["title"] = t.group(1).strip()
            p = re.search(r'class="a-price-whole"[^>]*>([^<]+)', resp.text)
            if p: data["price"] = p.group(1).strip()
            # Try og:title fallback
            if not data.get("title"):
                og = re.search(r'property="og:title"\s+content="([^"]+)"', resp.text)
                if og: data["title"] = og.group(1).strip()
        # Fallback: extract ASIN and try multiple methods
        if not data.get("title"):
            asin_m = re.search(r'/(?:dp|gp/product)/([A-Z0-9]{10})', url)
            asin = asin_m.group(1) if asin_m else None
            if asin:
                data["asin"] = asin
                # Method 1: Amazon mobile site (less aggressive bot detection)
                try:
                    mobile_url = f"https://www.amazon.com/dp/{asin}"
                    mr = self.fetch(mobile_url, country=country or "US", timeout=20, headers={
                        "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",
                        "Accept": "text/html", "Accept-Language": "en-US,en;q=0.9"
                    })
                    if mr.status_code == 200 and len(mr.text) > 10000:
                        t = re.search(r'id="productTitle"[^>]*>([^<]+)', mr.text)
                        if not t: t = re.search(r'id="title"[^>]*>([^<]+)', mr.text)
                        if not t: t = re.search(r'<title>([^<]+?)(?:\s*[-:]\s*Amazon)', mr.text)
                        if t:
                            data["title"] = t.group(1).strip()
                            source = "http_proxy_mobile"
                        p = re.search(r'class="a-price-whole"[^>]*>([^<]+)', mr.text)
                        if not p: p = re.search(r'\$(\d{1,5}\.\d{2})', mr.text)
                        if p: data["price"] = p.group(1).strip().rstrip('.')
                except Exception:
                    pass
                # Method 2: DuckDuckGo with exact ASIN URL
                if not data.get("title"):
                    try:
                        ddg_url = f"https://html.duckduckgo.com/html/?q=amazon.com%2Fdp%2F{asin}"
                        dr = self.fetch(ddg_url, timeout=10, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"})
                        if dr.status_code == 200:
                            # Match results containing this ASIN
                            results = re.finditer(r'class="result__a"[^>]*href="[^"]*' + asin + r'[^"]*"[^>]*>(.+?)</a>', dr.text)
                            for rm in results:
                                title = re.sub(r'<[^>]+>', '', rm.group(1)).strip()
                                title = re.sub(r'&amp;', '&', title).strip()
                                title = re.sub(r'^Amazon\.com\s*[:\-]\s*', '', title)
                                title = re.sub(r'\s*[-–|]\s*Amazon\.com.*$', '', title)
                                title = title.strip(' -.:')
                                if len(title) > 8 and 'All Departments' not in title and 'Low prices' not in title:
                                    data["title"] = title
                                    source = "duckduckgo"
                                    break
                            price_m = re.search(r'\$(\d{1,5}\.\d{2})', dr.text)
                            if price_m:
                                data["price"] = price_m.group(1)
                    except Exception:
                        pass
        return {"success": bool(data.get("title")), "site": "amazon", "data": data, "source": source, "error": None if data.get("title") else "Could not extract product data"}

    def _scrape_linkedin(self, url, country=None):
        """LinkedIn via proxy - extract from title/meta."""
        import re
        resp = self.fetch(url, country=country or "US", timeout=20)
        data = {}
        if resp.status_code == 200:
            t = re.search(r'<title>([^<]*)</title>', resp.text)
            if t: data["title"] = t.group(1).strip()
            emp = re.search(r'([\d,]+)\s+employees', resp.text, re.IGNORECASE)
            if emp: data["employees"] = emp.group(1)
            desc = re.search(r'<meta name="description" content="([^"]*)"', resp.text)
            if desc: data["description"] = desc.group(1)[:200]
        return {"success": bool(data.get("title")), "site": "linkedin", "data": data, "source": "http_proxy", "error": None}

    def _scrape_twitter(self, url, country=None):
        """Twitter/X via proxy - limited to title/meta."""
        import re
        resp = self.fetch(url, country=country or "US", timeout=15)
        data = {}
        if resp.status_code == 200:
            t = re.search(r'<title>([^<]*)</title>', resp.text)
            if t: data["title"] = t.group(1).strip()
            data["page_size"] = len(resp.text)
        return {"success": bool(data.get("title")), "site": "twitter", "data": data, "source": "http_proxy", "error": None}

    def _scrape_tiktok(self, url, country=None):
        """TikTok via proxy - title and og:description."""
        import re
        resp = self.fetch(url, country=country or "US", timeout=15)
        data = {}
        if resp.status_code == 200:
            t = re.search(r'<title>([^<]*)</title>', resp.text)
            if t: data["title"] = t.group(1).strip()
            og = re.search(r'<meta property="og:description" content="([^"]*)"', resp.text)
            if og: data["description"] = og.group(1)
        return {"success": bool(data.get("title")), "site": "tiktok", "data": data, "source": "http_proxy", "error": None}

    def _scrape_generic(self, url, country=None):
        """Advanced generic scraper — works on any website."""
        import re, json as _json
        source = "http"
        
        # Try multiple countries for resilience
        countries = [country or "US", "GB", "DE"]
        resp = None
        for c in countries:
            try:
                resp = self.fetch(url, country=c, timeout=20)
                # Check if we got real content (not blocked/empty)
                if resp.status_code == 200 and len(resp.text) > 5000:
                    break
                if resp.status_code == 403:
                    continue  # Try next country
            except Exception:
                continue
        
        if resp is None or resp.status_code == 403:
            return self._generic_brave_fallback(url)

        html = resp.text if hasattr(resp, 'text') else ""
        data = {}

        # 1. Title
        t = re.search(r'<title>([^<]*)</title>', html, re.I)
        if t: data["title"] = t.group(1).strip()

        # 2. Meta tags (description, keywords, author)
        for tag in ("description", "keywords", "author"):
            m = re.search(rf'<meta\s+name="{tag}"\s+content="([^"]*)"', html, re.I)
            if not m:
                m = re.search(rf'<meta\s+content="([^"]*)"\s+name="{tag}"', html, re.I)
            if m: data[tag] = m.group(1)[:500]

        # 3. OpenGraph tags (og:title, og:description, og:image, og:type, og:url)
        og = {}
        for m in re.finditer(r'<meta\s+(?:property|name)="og:(\w+)"\s+content="([^"]*)"', html, re.I):
            og[m.group(1)] = m.group(2)
        for m in re.finditer(r'<meta\s+content="([^"]*)"\s+(?:property|name)="og:(\w+)"', html, re.I):
            og[m.group(2)] = m.group(1)
        if og: data["opengraph"] = og
        if not data.get("title") and og.get("title"):
            data["title"] = og["title"]

        # 4. JSON-LD structured data
        ld_blocks = re.findall(r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>', html, re.S | re.I)
        if ld_blocks:
            parsed_ld = []
            for block in ld_blocks[:5]:
                try:
                    parsed_ld.append(_json.loads(block))
                except Exception:
                    pass
            if parsed_ld:
                data["json_ld"] = parsed_ld

        # 5. Headings (h1, h2)
        h1s = re.findall(r'<h1[^>]*>(.*?)</h1>', html, re.S | re.I)
        h2s = re.findall(r'<h2[^>]*>(.*?)</h2>', html, re.S | re.I)
        clean = lambda s: re.sub(r'<[^>]+>', '', s).strip()
        if h1s: data["h1"] = [clean(h) for h in h1s[:5] if clean(h)]
        if h2s: data["h2"] = [clean(h) for h in h2s[:10] if clean(h)]

        # 6. Main content extraction (article, main, or body text)
        main_text = ""
        for tag in ("article", "main", r'div[^>]*class="[^"]*content[^"]*"'):
            m = re.search(rf'<{tag}[^>]*>(.*?)</{tag.split("[")[0]}>', html, re.S | re.I)
            if m:
                main_text = re.sub(r'<[^>]+>', ' ', m.group(1))
                main_text = re.sub(r'\s+', ' ', main_text).strip()[:5000]
                break
        if not main_text:
            body = re.search(r'<body[^>]*>(.*?)</body>', html, re.S | re.I)
            if body:
                clean_html = re.sub(r'<(script|style|nav|header|footer)[^>]*>.*?</\1>', '', body.group(1), flags=re.S | re.I)
                main_text = re.sub(r'<[^>]+>', ' ', clean_html)
                main_text = re.sub(r'\s+', ' ', main_text).strip()[:5000]
        if main_text: data["text"] = main_text

        # 7. Links (internal + external)
        from urllib.parse import urlparse
        domain = urlparse(url).netloc
        links = re.findall(r'<a[^>]+href="([^"]*)"[^>]*>(.*?)</a>', html, re.S | re.I)
        internal, external = [], []
        for href, text in links[:100]:
            link_text = re.sub(r'<[^>]+>', '', text).strip()
            if not link_text or len(link_text) < 2: continue
            if href.startswith('/') or domain in href:
                internal.append({"url": href, "text": link_text[:100]})
            elif href.startswith('http'):
                external.append({"url": href, "text": link_text[:100]})
        if internal: data["internal_links"] = internal[:20]
        if external: data["external_links"] = external[:10]

        # 8. Images with alt text
        imgs = re.findall(r'<img[^>]+src="([^"]*)"[^>]*alt="([^"]*)"', html, re.I)
        if imgs: data["images"] = [{"src": s, "alt": a} for s, a in imgs[:15] if a]

        # 9. Prices (e-commerce detection)
        prices = re.findall(r'[\$€£¥]\s*[\d,]+\.?\d*', html)
        if prices: data["prices"] = list(set(prices[:10]))

        # 10. Clean HTML (for AI agents — no scripts/styles/nav, just content)
        clean_html = re.sub(r'<(script|style|nav|header|footer|aside|iframe|noscript)[^>]*>.*?</\1>', '', html, flags=re.S | re.I)
        clean_html = re.sub(r'<!--.*?-->', '', clean_html, flags=re.S)
        clean_html = re.sub(r'\s+', ' ', clean_html).strip()
        if clean_html: data["clean_html"] = clean_html[:50000]  # 50KB max

        # 11. Page metadata
        data["page_size"] = len(html)
        data["clean_html_size"] = len(data.get("clean_html", ""))
        data["status_code"] = getattr(resp, 'status_code', None)
        data["url"] = url

        # Smart success detection — not just "has title" but "has REAL content"
        title = data.get("title", "").strip()
        text = data.get("text", "")
        has_real_title = bool(title) and title not in ("&nbsp;", "") and "access denied" not in title.lower() and "blocked" not in title.lower() and "captcha" not in title.lower()
        has_real_content = len(text) > 200  # Less than 200 chars = probably empty/blocked
        has_structured = bool(data.get("json_ld"))
        success = (has_real_title and has_real_content) or has_structured

        # Auto-try Playwright if installed and content looks empty/blocked
        if not success:
            try:
                from .renderer import render_url, PLAYWRIGHT_AVAILABLE
                if PLAYWRIGHT_AVAILABLE:
                    logger.debug(f"HTTP got weak content ({len(text)} chars), trying Playwright...")
                    rendered_html = render_url(self.api_key, url)
                    if rendered_html and len(rendered_html) > len(html):
                        return self._scrape_generic_from_html(url, rendered_html, source="playwright")
            except ImportError:
                pass
            except Exception as e:
                logger.debug(f"Playwright fallback failed: {e}")

        # Still failed? Try Brave Search
        if not success:
            brave_result = self._generic_brave_fallback(url)
            if not brave_result.get("success"):
                # All fallbacks exhausted — auto-request preset
                try:
                    self.request_preset(url)
                except Exception:
                    pass
                brave_result["hint"] = "Site not yet supported. We've been notified and will add support soon. Run client.check_updates() to check for new presets."
                brave_result["preset_requested"] = True
            return brave_result

        return {"success": success, "site": "generic", "data": data, "source": source, "error": None}

    def _scrape_generic_from_html(self, url, html, source="http"):
        """Parse pre-fetched HTML (used by Playwright fallback)."""
        import re, json as _json
        data = {}
        t = re.search(r'<title>([^<]*)</title>', html, re.I)
        if t: data["title"] = t.group(1).strip()
        for tag in ("description", "keywords", "author"):
            m = re.search(rf'<meta\s+name="{tag}"\s+content="([^"]*)"', html, re.I)
            if not m: m = re.search(rf'<meta\s+content="([^"]*)"\s+name="{tag}"', html, re.I)
            if m: data[tag] = m.group(1)[:500]
        ld_blocks = re.findall(r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>', html, re.S | re.I)
        if ld_blocks:
            parsed_ld = []
            for block in ld_blocks[:5]:
                try: parsed_ld.append(_json.loads(block))
                except: pass
            if parsed_ld: data["json_ld"] = parsed_ld
        body = re.search(r'<body[^>]*>(.*?)</body>', html, re.S | re.I)
        if body:
            clean = re.sub(r'<(script|style|nav|header|footer|aside)[^>]*>.*?</\1>', '', body.group(1), flags=re.S | re.I)
            text = re.sub(r'<[^>]+>', ' ', clean)
            text = re.sub(r'\s+', ' ', text).strip()[:5000]
            if text: data["text"] = text
        clean_html = re.sub(r'<(script|style|nav|header|footer|aside|iframe|noscript)[^>]*>.*?</\1>', '', html, flags=re.S | re.I)
        clean_html = re.sub(r'<!--.*?-->', '', clean_html, flags=re.S)
        clean_html = re.sub(r'\s+', ' ', clean_html).strip()[:50000]
        if clean_html: data["clean_html"] = clean_html
        data["page_size"] = len(html)
        data["url"] = url
        success = bool(data.get("title") or data.get("text") or data.get("json_ld"))
        return {"success": success, "site": "generic", "data": data, "source": source, "error": None}

    def _generic_brave_fallback(self, url):
        """Fallback: search Brave for the URL to get basic info."""
        try:
            from urllib.parse import urlparse
            domain = urlparse(url).netloc
            path = urlparse(url).path
            query = f"site:{domain} {path}" if path and path != "/" else domain
            brave_url = f"https://api.search.brave.com/res/v1/web/search?q={query}&count=3"
            # Try without proxy
            import requests as _req
            resp = _req.get(brave_url, headers={"Accept": "application/json", "Accept-Encoding": "gzip",
                "X-Subscription-Token": self._brave_key} if hasattr(self, '_brave_key') else {}, timeout=10)
            if resp.status_code == 200:
                results = resp.json().get("web", {}).get("results", [])
                if results:
                    r = results[0]
                    return {"success": True, "site": "generic", "data": {
                        "title": r.get("title", ""),
                        "description": r.get("description", ""),
                        "url": r.get("url", url),
                    }, "source": "brave_search", "error": None}
            return {"success": False, "site": "generic", "data": {"url": url}, "source": "failed", "error": "Could not fetch or search"}
        except Exception as e:
            return {"success": False, "site": "generic", "data": {"url": url}, "source": "failed", "error": str(e)}

    # ── Site-specific presets ──────────────────────────────────

    @property
    def twitter(self):
        from .sites.twitter import Twitter
        return Twitter(self)

    @property
    def google(self):
        from .sites.google import Google
        return Google(self)

    @property
    def amazon(self):
        from .sites.amazon import Amazon
        return Amazon(self)

    @property
    def instagram(self):
        from .sites.instagram import Instagram
        return Instagram(self)

    @property
    def tiktok(self):
        from .sites.tiktok import TikTok
        return TikTok(self)

    @property
    def youtube(self):
        from .sites.youtube import YouTube
        return YouTube(self)

    @property
    def reddit(self):
        from .sites.reddit import Reddit
        return Reddit(self)

    @property
    def ebay(self):
        from .sites.ebay import eBay
        return eBay(self)

    @property
    def nasdaq(self):
        from .sites.nasdaq import Nasdaq
        return Nasdaq(self)

    @property
    def linkedin(self):
        from .sites.linkedin import LinkedIn
        return LinkedIn(self)

    @property
    def wikipedia(self):
        from .sites.wikipedia import Wikipedia
        return Wikipedia(self)

    @property
    def hackernews(self):
        from .sites.hackernews import HackerNews
        return HackerNews(self)

    @property
    def github(self):
        from .sites.github import GitHub
        return GitHub(self)

    @property
    def coingecko(self):
        from .sites.coingecko import CoinGecko
        return CoinGecko(self)

    @property
    def weather(self):
        from .sites.weather import Weather
        return Weather(self)

    @property
    def spotify(self):
        from .sites.spotify import Spotify
        return Spotify(self)

    @property
    def stackoverflow(self):
        from .sites.stackoverflow import StackOverflow
        return StackOverflow(self)

    @property
    def npm(self):
        from .sites.npm import NPM
        return NPM(self)

    @property
    def pypi(self):
        from .sites.pypi import PyPI
        return PyPI(self)

    @property
    def imdb(self):
        from .sites.imdb import IMDb
        return IMDb(self)

    @property
    def xkcd(self):
        from .sites.xkcd import XKCD
        return XKCD(self)

    @property
    def exchangerate(self):
        from .sites.exchangerate import ExchangeRate
        return ExchangeRate(self)

    @property
    def spacex(self):
        from .sites.spacex import SpaceX
        return SpaceX(self)

    @property
    def pokeapi(self):
        from .sites.pokeapi import PokeAPI
        return PokeAPI(self)

    @property
    def steam(self):
        from .sites.steam import Steam
        return Steam(self)

    @property
    def goodreads(self):
        from .sites.goodreads import Goodreads
        return Goodreads(self)

    @property
    def archiveorg(self):
        from .sites.archiveorg import ArchiveOrg
        return ArchiveOrg(self)

    @property
    def trustpilot(self):
        from .sites.trustpilot import Trustpilot
        return Trustpilot(self)

    @property
    def airbnb(self):
        from .sites.airbnb import Airbnb
        return Airbnb(self)

    @property
    def remoteok(self):
        from .sites.remoteok import RemoteOK
        return RemoteOK(self)

    @property
    def cnn(self):
        from .sites.cnn import CNN
        return CNN(self)

    @property
    def aliexpress(self):
        from .sites.aliexpress import AliExpress
        return AliExpress(self)

    @property
    def shopify(self):
        from .sites.shopify import Shopify
        return Shopify(self)

    def smart_scrape(self, url, country=None, **kwargs):
        """
        Smart scrape: for known-blocked sites, skip proxy and go straight to fallback.
        For other sites, try primary extractor first, then fallback if it fails.

        Returns: {"success": bool, "data": dict, "source": str, "fallback_used": bool, "original_site": str, "error": str|None}
        """
        from urllib.parse import urlparse
        from .sites.blocked_fallbacks import get_fallback, execute_fallback

        hostname = (urlparse(url).hostname or "").lower().removeprefix("www.")

        # Known-blocked sites → skip proxy, go straight to fallback
        if get_fallback(hostname):
            fb_result = execute_fallback(hostname, url, client=self)
            fb_result["original_site"] = hostname
            return fb_result

        # Normal sites → try primary extractor
        result = self.scrape(url, country=country, **kwargs)
        result.setdefault("fallback_used", False)
        result.setdefault("original_site", result.get("site", "unknown"))
        return result

    def render_fetch(self, url, country=None, city=None, wait_for=None, wait_time=None, **kwargs):
        """
        Fetch a URL using Playwright renderer with proxy support.
        
        Args:
            url: URL to render
            country: Country for proxy
            city: City for proxy
            wait_for: CSS selector to wait for
            wait_time: Time to wait in seconds
            **kwargs: Additional arguments
            
        Returns:
            Rendered HTML content
        """
        try:
            from .renderer import render_url
            
            # Build proxy auth string
            c = country or self._country or "US"
            ci = city or self._city
            
            # Use the renderer with proper proxy auth
            html = render_url(
                api_key=self.api_key,
                url=url,
                wait_for=wait_for,
                wait_time=wait_time
            )
            
            return html
            
        except ImportError:
            raise ProxyError(
                "Playwright not installed. Run: pip install iploop[render] or pip install playwright"
            )
        except Exception as e:
            logger.error(f"Render fetch failed: {e}")
            raise ProxyError(f"Render fetch failed: {e}")

    def close(self):
        """Clean up resources."""
        if self._renderer:
            import asyncio
            try:
                loop = asyncio.get_event_loop()
                if loop.is_running():
                    # Schedule cleanup
                    loop.create_task(self._renderer.close())
                else:
                    loop.run_until_complete(self._renderer.close())
            except:
                pass
            self._renderer = None
