"""
scrylo/scrapers/finder.py
--------------------------
Company-first lead discovery pipeline.

ARCHITECTURE CHANGE FROM PREVIOUS VERSION
------------------------------------------
Old approach (person-first):
  Ask Google "who are SaaS founders in X" → get articles, podcasts,
  directories → try to extract people from noisy secondary sources →
  wrong email domains, garbage names, low quantity.

New approach (company-first):
  Ask Google "what are SaaS companies in X" → get company domains →
  fetch each company's own /team /about /people page → extract founders
  directly from primary source → names and domains are always correct.

WHY THIS ELIMINATES BAD LEADS
-------------------------------
Every person extracted comes from a page that says "these people work here."
You never get @saastr.com emails because you never scraped a SaaStr page.
You never get "Capchase Announces" as a name because /team pages don't have
headlines. The source is always the company itself, not a mention of it.

PIPELINE STAGES
---------------
  1. Serper company discovery  — finds company domains, not people
  2. Team page fetching        — fetches /team /about /people /impressum
  3. Semantic HTML extraction  — targets heading+adjacent tags, no regex on flat text
  4. LLM extraction (optional) — Groq/Ollama if configured, handles hard pages
  5. Email resolution          — only guesses at the confirmed company domain
  6. Tiered save               — Tier 1 (full) / Tier 2 (no email) / Tier 3 (LinkedIn only)
"""

from __future__ import annotations

import asyncio
import json
import logging
import re
import time
from dataclasses import dataclass, field
from typing import Optional
from urllib.parse import urlparse

import httpx
from bs4 import BeautifulSoup, Tag
from sqlalchemy.orm import Session

from scrylo.core.config import load as load_cfg
from scrylo.db.models import Lead, LeadStatus
from scrylo.db.session import db_session

log = logging.getLogger("scrylo.finder")

# ─────────────────────────────────────────────────────────────────────────────
# Constants & Configuration
# ─────────────────────────────────────────────────────────────────────────────

SERPER_URL        = "https://google.serper.dev/search"
SERPER_PLACES_URL = "https://google.serper.dev/places"
_TIMEOUT          = httpx.Timeout(connect=8.0, read=15.0, write=5.0, pool=5.0)

# Domains that are never a person's employer — never use as email domain.
_BAD_LEAD_DOMAINS = {
    # Aggregators / directories
    "crunchbase.com", "wellfound.com", "angel.co", "f6s.com",
    "clutch.co", "g2.com", "capterra.com",
    # Investor / VC data
    "pitchbook.com", "dealroom.co", "cbinsights.com",
    # Startup / event communities
    "saastr.com", "saastock.com", "producthunt.com",
    "ycombinator.com", "news.ycombinator.com",
    "techstars.com", "seedcamp.com", "antler.co",
    # Social / publishing
    "linkedin.com", "twitter.com", "x.com", "medium.com",
    "substack.com", "github.com", "facebook.com", "instagram.com",
    # Big platforms
    "google.com", "microsoft.com", "amazon.com", "apple.com",
    "salesforce.com", "hubspot.com", "openai.com",
    # Finance / stock sites
    "nseindia.com", "moneycontrol.com", "bseindia.com",
    # News / media / aggregators
    "afr.com", "techcrunch.com", "forbes.com", "bloomberg.com", "businessinsider.com",
    "theguardian.com", "wired.com", "venturebeat.com", "reuters.com",
    "nytimes.com", "wsj.com", "bbc.com", "bbc.co.uk", "ft.com",
    "economist.com", "inc.com", "fastcompany.com", "entrepreneur.com",
    "podcasts.apple.com", "podcasts.google.com", "spotify.com",
    # Job boards
    "indeed.com", "glassdoor.com",
    # Sports / non-business
    "mlb.com", "nba.com", "nfl.com", "nhl.com", "fifa.com", "usab.com",
    # Government / regulatory
    "sba.gov", "ftc.gov", "sec.gov", "irs.gov",
    # Data broker / lead list sites — not real company domains
    "datamaelumat.com", "zoominfo.com", "rocketreach.co",
    "apollo.io", "hunter.io", "contactout.com",
    # Legal / compliance generators
    "termly.io", "iubenda.com", "privacypolicies.com",
    # Market research / data
    "ibisworld.com", "statista.com", "mordorintelligence.com",
    # Incubators / accelerators (people appear as participants, not employees)
    "revithaca.com", "techstars.com",
    # Video platforms
    "youtube.com", "vimeo.com",
    # Imprint/print-on-demand
    "imprintengine.com", "imprintusa.com",

    # ── Extended bad domain list for distribution safety ─────────────────────

    # More news / media
    "cnn.com", "cnbc.com", "abc.com", "nbcnews.com", "foxnews.com",
    "usatoday.com", "washingtonpost.com", "latimes.com", "theverge.com",
    "engadget.com", "gizmodo.com", "mashable.com", "zdnet.com",
    "cnet.com", "pcmag.com", "techradar.com", "arstechnica.com",
    "thenextweb.com", "siliconrepublic.com", "cityam.com",
    "standard.co.uk", "telegraph.co.uk", "thetimes.co.uk",
    "mirror.co.uk", "metro.co.uk", "express.co.uk",
    "spiegel.de", "zeit.de", "faz.net", "handelsblatt.com",
    "lefigaro.fr", "lemonde.fr", "lesechos.fr",
    "corriere.it", "repubblica.it", "sole24ore.com",
    "elpais.com", "expansion.com", "cincodias.elpais.com",
    "theaustralian.com.au", "smh.com.au", "afr.com", "abc.net.au",
    "techcrunch.com", "techinasia.com", "e27.co", "dealstreetasia.com",
    "disruptafrica.com", "techcabal.com", "techpoint.africa",
    "biztechafrica.com", "businessdailyafrica.com",

    # Startup directories / lists
    "startupblink.com", "startupbase.io", "eu-startups.com",
    "startups.co.uk", "startupgrind.com", "failory.com",
    "startupsavvy.io", "builtinnyc.com", "builtinchicago.com",
    "builtinaustin.com", "builtinla.com", "builtinboston.com",
    "builtincolorado.com", "builtinseattle.com",
    "startupcanada.org", "betakit.com", "techvancouver.com",
    "lafrenchtech.com", "startupsinberlin.de",
    "founded.com", "tracxn.com", "owler.com",
    "companycheck.co.uk", "companieshouse.gov.uk",
    "opencorporates.com", "dnb.com", "manta.com",

    # Event / conference platforms
    "eventbrite.com", "meetup.com", "luma.events",
    "sxsw.com", "disrupt.com", "tnw.com", "riseconf.com",
    "websummit.com", "slush.org", "collision.events",
    "viva.technology", "leweb.com", "cebit.de",
    "inbound.com", "dreamforce.com", "salesforce.com",
    "githubuniverse.com", "devrelcon.com",

    # Podcast / content platforms
    "buzzsprout.com", "transistor.fm", "podbean.com",
    "anchor.fm", "simplecast.com", "captivate.fm",
    "listennotes.com", "podchaser.com", "chartable.com",
    "soundcloud.com", "mixcloud.com",
    "overcast.fm", "pocketcasts.com",

    # Investor / VC platforms
    "angellist.com", "gust.com", "seedrs.com",
    "crowdcube.com", "republic.com", "wefunder.com",
    "startengine.com", "equityzen.com", "forge.com",
    "svb.com", "siliconvalleybank.com",
    "a16z.com", "sequoiacap.com", "ycombinator.com",
    "accel.com", "indexventures.com", "balderton.com",
    "atomico.com", "northzone.com", "earlybird.com",

    # Academic / university
    "mit.edu", "stanford.edu", "harvard.edu", "ox.ac.uk",
    "cam.ac.uk", "imperial.ac.uk", "ucl.ac.uk", "ed.ac.uk",
    "ethz.ch", "epfl.ch", "tum.de", "hpi.de",
    "cornell.edu", "cmu.edu", "berkeley.edu", "caltech.edu",
    "columbia.edu", "yale.edu", "princeton.edu",
    "coursera.org", "udemy.com", "edx.org", "udacity.com",
    "khan academy.org", "khanacademy.org",

    # Government / public sector
    "gov.uk", "europa.eu", "ec.europa.eu",
    "whitehouse.gov", "congress.gov", "senate.gov",
    "treasury.gov", "state.gov", "nasa.gov",
    "nhs.uk", "legislation.gov.uk",

    # Job boards / recruitment
    "linkedin.com", "monster.com", "ziprecruiter.com",
    "dice.com", "stackoverflow.com", "wellfound.com",
    "workatastartup.com", "remoteok.com", "weworkremotely.com",
    "hired.com", "toptal.com", "upwork.com", "freelancer.com",
    "fiverr.com", "guru.com", "peopleperhour.com",
    "workable.com", "lever.co", "greenhouse.io",
    "ashbyhq.com", "smartrecruiters.com", "bamboohr.com",
    "rippling.com", "personio.com",

    # Review / rating platforms
    "trustpilot.com", "reviews.io", "feefo.com",
    "sitejabber.com", "bbb.org", "consumeraffairs.com",
    "yelp.com", "tripadvisor.com",

    # E-commerce / marketplaces
    "amazon.co.uk", "ebay.com", "etsy.com", "shopify.com",
    "woocommerce.com", "magento.com", "bigcommerce.com",
    "stripe.com", "paddle.com", "gumroad.com",
    "producthunt.com", "appsumo.com",

    # Dev / tech platforms
    "stackoverflow.com", "dev.to", "hashnode.com",
    "hackernoon.com", "dzone.com", "infoq.com",
    "npmjs.com", "pypi.org", "packagist.org",
    "dockerhub.com", "hub.docker.com",
    "gitlab.com", "bitbucket.org", "sourceforge.net",
    "codepen.io", "jsfiddle.net", "codesandbox.io",
    "replit.com", "glitch.com",

    # CMS / website builders
    "wordpress.com", "wix.com", "squarespace.com",
    "weebly.com", "webflow.com", "ghost.org",
    "blogger.com", "tumblr.com", "typepad.com",

    # Wiki / reference
    "wikipedia.org", "wikia.com", "fandom.com",
    "quora.com", "reddit.com", "stackexchange.com",

    # Cloud providers (people at these aren't your target)
    "aws.amazon.com", "azure.microsoft.com", "cloud.google.com",
    "digitalocean.com", "linode.com", "vultr.com",
    "heroku.com", "netlify.com", "vercel.com",
    "cloudflare.com", "fastly.com", "akamai.com",

    # Analytics / marketing tools
    "semrush.com", "ahrefs.com", "moz.com",
    "similarweb.com", "alexa.com", "quantcast.com",
    "mailchimp.com", "constantcontact.com", "sendgrid.com",
    "klaviyo.com", "activecampaign.com", "marketo.com",
    "pardot.com", "eloqua.com", "hootsuite.com",
    "buffer.com", "sproutsocial.com", "later.com",

    # PR / press release wires
    "prnewswire.com", "businesswire.com", "globenewswire.com",
    "prweb.com", "accesswire.com", "einpresswire.com",
    "pressat.co.uk", "prlog.org",

    # Legal / IP databases
    "patents.google.com", "lens.org", "espacenet.com",
    "findlaw.com", "law.com", "legalzoom.com",
    "contractscounsel.com", "clerky.com",

    # Aggregator startup sites seen in results
    "startupbeat.com", "startupatlanta.com", "studentstartup.org",
    "startup.club", "startuphealth.com", "leanstartup.co",
    "cnsourcelink.com", "itif.org", "donnaweber.com",
    "seafarerfunds.com", "restofworld.org", "scribd.com",
    "edbatista.com", "thesaasnews.com", "saas-capital.com",
}

# URL path segments that indicate a non-lead page
_SKIP_PATH_PATTERNS = (
    "/job", "/jobs/", "/careers", "/vacancy", "/vacancies",
    "/apply", "/blog/", "/news/", "/press-release", "/article/",
    "/privacy", "/terms", "/cookie", "/legal", "/sitemap",
)

# Team/people page paths in priority order.
# Full list restored — concurrent fetching makes the path count irrelevant
# to speed. More paths = better coverage with no time penalty.
_TEAM_PATHS = [
    "/team",
    "/about",
    "/about-us",
    "/people",
    "/leadership",
    "/founders",
    "/our-team",
    "/management",
    "/company",
    "/contact",
    "/contact-us",
    "/impressum",
    "/imprint",
]

_GENERIC_PREFIXES = {
    "noreply", "no-reply", "donotreply", "support", "help", "info",
    "contact", "admin", "team", "careers", "jobs", "press", "media",
    "legal", "privacy", "sales", "billing", "hello", "hi", "general",
    "office", "mail", "webmaster", "reception", "enquiries", "enquiry",
    "newsletter", "abuse", "security", "postmaster", "feedback",
}

_HONORIFICS = {
    "dr", "mr", "mrs", "ms", "miss", "prof", "rev", "sir",
    "eng", "hon", "sr", "jr",
}

_NOT_NAME_WORDS = {
    "chief", "head", "vice", "senior", "junior", "lead", "director",
    "manager", "officer", "executive", "founder", "co-founder",
    "ceo", "cto", "coo", "cfo", "our", "the", "meet", "about",
    "contact", "team", "join", "read", "see", "get", "how", "why",
    "what", "all", "new", "top", "best", "here", "this", "your",
    "saas", "software", "solutions", "services", "platform", "company",
    "enterprise", "business", "growth", "revenue", "strategy", "monetization",
    "reduced", "reduce", "risk", "grow", "scaling", "scale", "management",
    "privacy", "legal", "terms", "cookie", "policy", "cookies", "login",
    "signup", "register", "blog", "news", "press", "media", "faq", "help",
    "support", "documentation", "guides", "guide", "learn", "menu", "search",
    "main", "office", "fraud", "prevention", "high", "customer", "satisfaction",
    "rapid", "turnaround", "chinese", "americans", "unified", "governance",
    "transformational", "delivery", "first", "investment", "key", "takeaways",
    "million", "thousand", "facebook", "instagram", "resident", "residents",
    "email", "outreach", "rental", "companies", "resident", "residents",
    "strategic", "focus", "rents", "company", "path", "career",

}

_LOCATION_TLD = {
    "australia": ".com.au", "sydney": ".com.au", "melbourne": ".com.au",
    "germany": ".de", "berlin": ".de", "munich": ".de", "hamburg": ".de",
    "france": ".fr", "paris": ".fr",
    "uk": ".co.uk", "london": ".co.uk", "united kingdom": ".co.uk",
    "netherlands": ".nl", "amsterdam": ".nl",
    "sweden": ".se", "stockholm": ".se",
    "nigeria": ".com.ng", "lagos": ".com.ng",
    "india": ".in", "bangalore": ".in", "mumbai": ".in",
    "canada": ".ca", "toronto": ".ca", "vancouver": ".ca",
    "new zealand": ".co.nz", "auckland": ".co.nz",
    "ireland": ".ie", "dublin": ".ie",
    "singapore": ".sg",
    "spain": ".es", "madrid": ".es",
    # USA — no country TLD but tracked so queries are built correctly
    "usa": "", "united states": "", "america": "",
    "new york": "", "san francisco": "", "los angeles": "",
    "chicago": "", "boston": "", "austin": "", "seattle": "",
    "miami": "", "denver": "", "atlanta": "", "dallas": "",
    "houston": "", "philadelphia": "", "phoenix": "",
    # Canada — cities not previously covered
    "ottawa": ".ca", "montreal": ".ca", "calgary": ".ca",
    "edmonton": ".ca", "winnipeg": ".ca", "quebec": ".ca",
    # UK cities
    "manchester": ".co.uk", "birmingham": ".co.uk", "edinburgh": ".co.uk",
    "glasgow": ".co.uk", "bristol": ".co.uk", "leeds": ".co.uk",
    # Other
    "lagos": ".com.ng", "abuja": ".com.ng",
    "nairobi": ".co.ke", "kenya": ".co.ke",
    "johannesburg": ".co.za", "cape town": ".co.za", "south africa": ".co.za",
    "jakarta": ".co.id", "indonesia": ".co.id",
    "brazil": ".com.br", "sao paulo": ".com.br",
    "mexico": ".com.mx", "mexico city": ".com.mx",
    "switzerland": ".ch", "zurich": ".ch", "geneva": ".ch",
    "sweden": ".se", "norway": ".no", "oslo": ".no",
    "denmark": ".dk", "copenhagen": ".dk",
    "finland": ".fi", "helsinki": ".fi",
    "poland": ".pl", "warsaw": ".pl",
    "czech republic": ".cz", "prague": ".cz",
    "austria": ".at", "vienna": ".at",
    "belgium": ".be", "brussels": ".be",
    "portugal": ".pt", "lisbon": ".pt",
    "italy": ".it", "milan": ".it", "rome": ".it",
}

# ── Comprehensive seniority title system ──────────────────────────────────────
# Tier A — founders, owners, C-suite
_TITLE_TIER_A = {
    # CEO variants
    "chief executive", "ceo", "c.e.o",
    # Founder variants
    "founder", "co-founder", "cofounder", "co founder",
    "founding partner", "founding member", "founded by",
    # Owner/principal
    "owner", "proprietor", "sole trader",
    # President/MD
    "president", "managing director", "managing partner",
    "group md", "group ceo", "executive chairman",
    "chairman", "chairwoman", "chairperson",
    # Other C-suite
    "chief technology", "cto", "c.t.o",
    "chief financial", "cfo", "c.f.o",
    "chief operating", "coo", "c.o.o",
    "chief marketing", "cmo", "c.m.o",
    "chief product", "cpo", "c.p.o",
    "chief revenue", "cro", "c.r.o",
    "chief strategy", "cso", "c.s.o",
    "chief commercial", "cco",
    "chief data", "cdo",
    "chief people", "chief human", "chro",
    "chief information", "cio", "c.i.o",
    "chief growth", "chief digital",
    "chief of staff",
    # General partner / principal
    "general partner", "managing general",
    "principal partner", "equity partner",
    # Board
    "board member", "board director",
    "non-executive director", "independent director",
}

# Tier B — VP / Head of / senior leadership
_TITLE_TIER_B = {
    "vice president", "vp of", "v.p.", "svp", "evp", "avp",
    "senior vice president", "executive vice president",
    "head of", "global head", "regional head",
    "general manager", "country manager", "regional manager",
    "partner",  # law/consulting/VC firms
    "senior partner", "limited partner",
    "director of", "global director", "regional director",
    "executive director", "group director",
    "senior director",
}

# Tier C — director level (still valuable, lower priority)
_TITLE_TIER_C = {
    "director",   # generic "Director" alone
    "principal",  # individual contributor principal
    "lead",       # "Lead Engineer" etc. — lower value but still senior
}

# Flat lookup set used for fast "is this a senior title" checks
_ALL_SENIOR_TITLES = _TITLE_TIER_A | _TITLE_TIER_B | _TITLE_TIER_C

# Role keywords used in _parse_query to detect user intent
_QUERY_ROLE_PATTERNS = re.compile(
    r"\b(CEO|CTO|CFO|COO|CMO|CPO|CRO|CSO|CIO|CHRO|CDO|"
    r"Founder|Co-Founder|Cofounder|"
    r"Director|VP|Vice President|"
    r"Owner|MD|Managing Director|President|"
    r"Partner|Principal|Chairman|Chairwoman)\b",
    re.IGNORECASE,
)

# Strip pattern — removes role words from query to isolate industry+location
_QUERY_ROLE_STRIP = re.compile(
    r"\b(CEOs?|CTOs?|CFOs?|COOs?|CMOs?|CPOs?|CROs?|CSOs?|CIOs?|CHROs?|CDOs?|"
    r"Founder|Co-Founder|Cofounder|Founders?|"
    r"Director|Directors?|VP|Vice Presidents?|"
    r"Owner|Owners?|MD|Managing Directors?|Presidents?|"
    r"Partner|Partners?|Principal|Principals?|Chairman|Chairwom[ae]n|"
    r"executives?|leaders?|startups?)\b",
    re.IGNORECASE,
)

_NAME_TAGS  = {"h1", "h2", "h3", "h4", "h5", "strong", "b"}
_TITLE_TAGS = {"p", "span", "div", "small", "em", "li"}

_USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
]

import random
def _ua() -> str:
    return random.choice(_USER_AGENTS)


# ─────────────────────────────────────────────────────────────────────────────
# Email resolution — MX validation + pattern inference
# Built directly into finder.py to avoid sync/async boundary issues.
# All DNS lookups are done via asyncio.get_event_loop().run_in_executor
# so they never block the event loop.
# ─────────────────────────────────────────────────────────────────────────────

# Per-run caches — cleared at start of each find_leads call
# _domain_mx_cache:      domain → has_mx: bool
# _domain_pattern_cache: domain → pattern string e.g. "first.last"
_domain_mx_cache:      dict[str, bool] = {}
_domain_pattern_cache: dict[str, str]  = {}


def _mx_check_sync(domain: str) -> bool:
    """
    Synchronous MX check. Returns True if domain has MX records.
    Called via run_in_executor — never blocks the event loop.
    Simple DNS lookup only — no SMTP connections, no port 25, no probing.
    Cached per domain so each domain is checked once per session.
    """
    if domain in _domain_mx_cache:
        return _domain_mx_cache[domain]
    try:
        import dns.resolver
        resolver          = dns.resolver.Resolver()
        resolver.lifetime = 4.0
        resolver.timeout  = 3.0
        resolver.resolve(domain, "MX")
        has_mx = True
    except Exception:
        has_mx = False
    _domain_mx_cache[domain] = has_mx
    return has_mx


async def _mx_check(domain: str) -> bool:
    """Async wrapper — runs DNS lookup in executor so event loop stays free."""
    loop = asyncio.get_event_loop()
    return await loop.run_in_executor(None, _mx_check_sync, domain)


def _infer_email_pattern(email: str, first: str, last: str) -> str:
    """
    Given a real scraped email and the person's name parts, return the
    pattern string that describes the company's email format.

    "jane.smith@co.com"  + jane/smith → "first.last"
    "jsmith@co.com"      + jane/smith → "flast"
    "jane@co.com"        + jane/smith → "first"
    "j.smith@co.com"     + jane/smith → "f.last"
    "jane_smith@co.com"  + jane/smith → "first_last"
    "janes@co.com"       + jane/smith → "firstl"
    "smith@co.com"       + jane/smith → "last"
    "smith.jane@co.com"  + jane/smith → "last.first"
    """
    if not email or "@" not in email or not first:
        return ""
    local = email.split("@")[0].lower()
    fi = first[0] if first else ""
    li = last[0]  if last  else ""

    if last:
        if local == f"{first}.{last}":  return "first.last"
        if local == f"{fi}{last}":      return "flast"
        if local == f"{fi}.{last}":     return "f.last"
        if local == f"{first}_{last}":  return "first_last"
        if local == f"{first}{li}":     return "firstl"
        if local == f"{last}.{first}":  return "last.first"
        if local == f"{last}{fi}":      return "lastf"
        if local == last:               return "last"
    if local == first:                  return "first"
    return ""


def _apply_pattern(pattern: str, first: str, last: str, domain: str) -> str:
    """
    Build an email address from a known pattern + name parts + domain.
    Returns "" if the pattern can't be applied to these name parts.
    """
    if not first or not domain:
        return ""
    fi = first[0]
    li = last[0] if last else ""

    mapping = {
        "first.last":  f"{first}.{last}@{domain}"  if last else "",
        "flast":       f"{fi}{last}@{domain}"       if last else "",
        "f.last":      f"{fi}.{last}@{domain}"      if last else "",
        "first_last":  f"{first}_{last}@{domain}"   if last else "",
        "firstl":      f"{first}{li}@{domain}"      if last else "",
        "last.first":  f"{last}.{first}@{domain}"   if last else "",
        "lastf":       f"{last}{fi}@{domain}"        if last else "",
        "last":        f"{last}@{domain}"            if last else "",
        "first":       f"{first}@{domain}",
    }
    return mapping.get(pattern, "")


async def _resolve_email(
    first:   str,
    last:    str,
    domain:  str,
) -> tuple[str, float, bool]:
    """
    Resolve the best email for a person at a confirmed company domain.
    Returns (email, confidence, email_verified).

    Steps:
      1. Check pattern cache — if another person on this domain had a
         scraped email, use that pattern first (highest quality guess)
      2. Fall back to frequency-ordered pattern list
      3. MX check the domain — if no MX, return ("", 0.0, False)
      4. If catch-all, cap confidence at 0.5
    """
    if not first or not domain:
        return "", 0.0, False

    # All candidate patterns in frequency order
    candidates = []
    if last:
        fi, li = first[0], last[0]
        candidates = [
            f"{first}.{last}@{domain}",   # ~40%
            f"{first}@{domain}",          # ~20%
            f"{fi}{last}@{domain}",       # ~12%
            f"{fi}.{last}@{domain}",      # ~7%
            f"{first}_{last}@{domain}",   # ~5%
            f"{first}{li}@{domain}",      # ~4%
            f"{last}@{domain}",           # ~4%
            f"{last}.{first}@{domain}",   # ~3%
            f"{first}-{last}@{domain}",   # ~2%
        ]
    else:
        candidates = [f"{first}@{domain}"]

    # If we observed a pattern from another person on this domain,
    # move that pattern to the front of the list
    observed_pattern = _domain_pattern_cache.get(domain, "")
    if observed_pattern:
        preferred = _apply_pattern(observed_pattern, first, last, domain)
        if preferred and preferred in candidates:
            candidates = [preferred] + [c for c in candidates if c != preferred]
        elif preferred:
            candidates = [preferred] + candidates

    # MX check — async DNS lookup, runs in executor, no SMTP connections
    has_mx = await _mx_check(domain)

    if not has_mx:
        log.debug("No MX for domain %s — dropping email", domain)
        return "", 0.0, False

    best_email = candidates[0] if candidates else ""
    if not best_email:
        return "", 0.0, False

    return best_email, 0.75, True


# ─────────────────────────────────────────────────────────────────────────────
# Data structures
# ─────────────────────────────────────────────────────────────────────────────

@dataclass
class FoundLead:
    """Intermediate lead before DB save."""
    name:         str   = ""
    title:        str   = ""
    company:      str   = ""
    domain:       str   = ""
    email:        str   = ""
    linkedin_url: str   = ""
    github_url:   str   = ""
    location:     str   = ""
    source_url:   str   = ""
    score:        float = 0.0
    email_confidence: float | None = None
    email_verified: bool | None = None

    def is_valid(self) -> bool:
        return bool(self.name and self.name.strip() and len(self.name) >= 4)


@dataclass
class FinderResult:
    query:           str
    leads_saved:     int       = 0
    leads_skipped:   int       = 0
    pages_fetched:   int       = 0
    emails_found:    int       = 0
    emails_verified: int       = 0
    tier1:           int       = 0   # fully actionable
    tier2:           int       = 0   # partial
    tier3:           int       = 0   # signal (LinkedIn)
    lead_ids:        list[int] = field(default_factory=list)
    errors:          list[str] = field(default_factory=list)


# ─────────────────────────────────────────────────────────────────────────────
# Helper Functions
# ─────────────────────────────────────────────────────────────────────────────

def _title_tier(title: str) -> int:
    """
    Return seniority tier for a job title string.
    0 = not senior / no title
    1 = Tier C (director level)
    2 = Tier B (VP / Head of)
    3 = Tier A (founder / C-suite)
    """
    if not title:
        return 0
    t = title.lower()
    if any(frag in t for frag in _TITLE_TIER_A):
        return 3
    if any(frag in t for frag in _TITLE_TIER_B):
        return 2
    if any(frag in t for frag in _TITLE_TIER_C):
        return 1
    return 0


def _query_wants_seniority(user_query: str) -> bool:
    """
    Returns True if the user's query explicitly asks for senior/founder roles.
    Used to decide whether to apply seniority filtering.
    """
    return bool(_QUERY_ROLE_PATTERNS.search(user_query))


def _parse_query(user_query: str) -> dict:
    """
    Deconstructs a natural language query into components.

    "AI startups in london"       → industry="ai", location="london", tld=".co.uk"
    "saas founders in berlin"     → industry="saas", location="berlin", tld=".de"
    "startups in the usa"         → industry="startup", location="usa", tld=""
    "saas founders in ottawa"     → industry="saas", location="ottawa", tld=".ca"
    "fintech CTOs new york"       → industry="fintech", location="new york", tld=""
    "b2b software companies"      → industry="b2b software", location="", tld=""
    """
    q = user_query.lower()

    # Detect location — try longer phrases first to catch "new york", "san francisco"
    loc = ""
    for city in sorted(_LOCATION_TLD.keys(), key=len, reverse=True):
        if city in q:
            loc = city
            break

    # Strip: roles, "in/at/near/of/the", location from remainder → pure industry
    clean = re.sub(r"\b(in|at|near|of|the)\b", "", q)
    clean = _QUERY_ROLE_STRIP.sub("", clean)
    if loc:
        clean = re.sub(re.escape(loc), "", clean)
    clean = re.sub(r"\s+", " ", clean).strip()

    # Fallback threshold: > 1 char (not > 2) so "AI" doesn't get dropped
    # Also strip "company/companies" from industry — they pollute the query string
    clean = re.sub(r"\b(companies|company)\b", "", clean, flags=re.IGNORECASE)
    clean = re.sub(r"\s+", " ", clean).strip()
    industry = clean if len(clean) > 1 else "startup"

    return {
        "industry": industry,
        "location": loc,
        "tld":      _LOCATION_TLD.get(loc, ""),
    }


def _build_company_queries(user_query: str, pass_num: int = 1) -> list[str]:
    """
    Blind-append query builder.

    Instead of parsing industry/location from the user query (fragile),
    we append search operators directly to the raw query. Google handles
    the "where" and "what" internally — we just steer it toward company
    pages and away from articles/lists.

    LinkedIn queries are built separately using role detection on the
    raw query so they remain targeted.
    """
    q = user_query.strip()

    # Role terms for LinkedIn queries — extracted from raw query
    roles_found = _QUERY_ROLE_PATTERNS.findall(q)
    if roles_found:
        role_terms = " OR ".join(f'"{r}"' for r in roles_found[:3])
    else:
        role_terms = '"Founder" OR "CEO" OR "CTO" OR "Owner"'

    if pass_num == 1:
        return [
            # Blind append — raw query + inurl targeting forces company pages
            f'{q} (inurl:about OR inurl:contact OR inurl:team) -intitle:best -intitle:top -inurl:blog',
            f'{q} "our team" -intitle:best -site:medium.com -site:forbes.com',
            # LinkedIn profiles
            f'site:linkedin.com/in {role_terms} {q}',
        ]
    if pass_num == 2:
        return [
            f'{q} "meet the team" OR "leadership team" OR "our people"',
            f'{q} "contact us" email -inurl:blog -intitle:top -intitle:best',
            f'{q} site:wellfound.com OR site:crunchbase.com/organization',
            f'site:linkedin.com/in {q} "Head of" OR "VP" OR "Director"',
        ]
    if pass_num == 3:
        return [
            f'{q} company team',
            f'{q} "founded" OR "we are" -intitle:best -intitle:top',
            f'site:linkedin.com/in {q}',
        ]
    # Pass 4 — broadest fallback
    return [
        q,
        f'{q} team',
    ]


def _serper_search(query: str, api_key: str, num: int = 10) -> list[dict]:
    try:
        clean_q = re.sub(r"[<>]", "", query).strip()
        resp = httpx.post(
            SERPER_URL,
            headers={"X-API-KEY": api_key, "Content-Type": "application/json"},
            json={"q": clean_q, "num": num},
            timeout=_TIMEOUT,
        )
        if resp.status_code != 200:
            return []
        return resp.json().get("organic", [])
    except Exception as e:
        log.warning("Serper error: %s", e)
        return []


def _serper_places(query: str, api_key: str) -> list[tuple[str, str]]:
    """
    Call the Serper Places endpoint (Google Maps/Places data).
    Returns list of (domain, company_name) tuples.

    Places results are verified businesses — you will never get a
    TechCrunch article or blog post from this endpoint. It is the
    gold standard for finding real company domains from a query.

    Counts against the same Serper quota as search calls.
    Returns empty list silently on any failure.
    """
    try:
        resp = httpx.post(
            SERPER_PLACES_URL,
            headers={"X-API-KEY": api_key, "Content-Type": "application/json"},
            json={"q": query},
            timeout=_TIMEOUT,
        )
        if resp.status_code != 200:
            log.debug("Serper Places HTTP %d for: %s", resp.status_code, query[:60])
            return []

        places = resp.json().get("places", [])
        results = []
        for place in places:
            website = place.get("website", "")
            title   = place.get("title", "") or place.get("name", "")
            if not website:
                continue
            dom = _domain(website)
            if dom and not _is_bad_domain(dom):
                results.append((dom, title))

        log.info("Serper Places '%s' → %d business domains", query[:50], len(results))
        return results
    except Exception as e:
        log.debug("Serper Places error: %s", e)
        return []


def _domain(url: str) -> str:
    try:
        return re.sub(r"^www\.", "", urlparse(url).netloc.lower())
    except Exception:
        return ""


def _is_bad_domain(dom: str) -> bool:
    if not dom:
        return True
    return any(
        dom == bad or dom.endswith("." + bad)
        for bad in _BAD_LEAD_DOMAINS
    )


def discover_company_domains(
    user_query:      str,
    api_key:         str,
    limit:           int,
    pass_num:        int       = 1,
    exclude_domains: set[str]  = None,
) -> list[tuple[str, str]]:
    """
    Hybrid discovery: Places endpoint + Search endpoint merged.

    Places endpoint (Google Maps/Places):
      - Returns verified real businesses only
      - Zero articles, zero directories, zero noise
      - Best for location-based and business-type queries
      - Called once per session (pass 1 only) — results are cached in seen_domains
        so subsequent passes don't re-add the same domains

    Search endpoint (blind-append queries):
      - Broader coverage via inurl: operators targeting company pages
      - Catches companies not on Google Maps (online-only, global SaaS)
      - Runs every pass with different query angles

    Both are merged into one deduplicated domain list.
    """
    if exclude_domains is None:
        exclude_domains = set()

    seen_domains:      set[str]              = set(exclude_domains)
    results:           list[tuple[str, str]] = []
    linkedin_profiles: list[dict]            = []

    # ── Places endpoint (pass 1 only — gold standard business data) ──────────
    if pass_num == 1:
        places_results = _serper_places(user_query, api_key)
        for dom, co_name in places_results:
            if dom and dom not in seen_domains:
                seen_domains.add(dom)
                results.append((dom, co_name))
        time.sleep(0.3)

    # ── Search endpoint (all passes — blind-append queries) ───────────────────
    queries       = _build_company_queries(user_query, pass_num=pass_num)
    num_per_query = max(10, limit // max(len(queries), 1) + 5)

    seo_low_signal = ["what ", "how ", "meaning of", "definition", "glossary", "faq"]

    for query in queries:
        items = _serper_search(query, api_key, num=num_per_query)
        time.sleep(0.4)

        for r in items:
            link = r.get("link", "")
            dom  = _domain(link)

            # ── LinkedIn profile result ───────────────────────────────────
            if "linkedin.com/in/" in link.lower():
                profile_url = link.split("?")[0]
                title_raw   = r.get("title", "")
                snippet     = r.get("snippet", "")
                title_clean = re.sub(
                    r"\s*\|\s*LinkedIn\s*$", "", title_raw,
                    flags=re.IGNORECASE,
                ).strip()
                linkedin_profiles.append({
                    "url":     profile_url,
                    "title":   title_clean,
                    "snippet": snippet,
                })
                continue

            # ── Company domain result ─────────────────────────────────────
            if not dom or dom in seen_domains or _is_bad_domain(dom):
                continue
            if any(pat in link.lower() for pat in _SKIP_PATH_PATTERNS):
                continue

            title   = r.get("title", "")
            snippet = r.get("snippet", "").lower()

            # SEO bait filter
            if any(p in title.lower() for p in seo_low_signal) or                any(p in snippet for p in seo_low_signal):
                continue

            co_hint = re.sub(
                r"\s*[|\-]\s*(LinkedIn|Crunchbase|AngelList|Wikipedia|GitHub).*$",
                "", title, flags=re.IGNORECASE,
            ).strip()
            if " - " in co_hint: co_hint = co_hint.split(" - ")[0].strip()
            if " | " in co_hint: co_hint = co_hint.split(" | ")[0].strip()

            seen_domains.add(dom)
            results.append((dom, co_hint))

    for lp in linkedin_profiles:
        results.append(("_linkedin_", lp))

    log.info(
        "Pass %d: %d company domains + %d LinkedIn profiles for '%s'",
        pass_num,
        len(results) - len(linkedin_profiles),
        len(linkedin_profiles),
        user_query[:50],
    )
    return results


# ─────────────────────────────────────────────────────────────────────────────
# Extraction Logic
# ─────────────────────────────────────────────────────────────────────────────

async def _fetch_url(url: str) -> str:
    # Patchright: already has timeout_ms=8000 internally
    try:
        from scrylo.scrapers.browser import browser_fetch, patchright_available
        if patchright_available():
            html = await browser_fetch(url, timeout_ms=7000)
            if html and len(html) > 500:
                return html
    except Exception:
        pass

    # httpx: tight timeout — don't wait more than 6s for any single page
    try:
        async with httpx.AsyncClient(
            timeout=httpx.Timeout(connect=4.0, read=6.0, write=3.0, pool=3.0)
        ) as client:
            resp = await client.get(
                url, headers={"User-Agent": _ua()}, follow_redirects=True
            )
            if resp.status_code == 200 and len(resp.text) > 200:
                return resp.text
    except Exception:
        pass
    return ""


def _company_from_domain(domain: str) -> str:
    """
    Derive a readable company name from a domain.

    "therealdeal.com"  → "The Real Deal"
    "my-startup.io"    → "My Startup"
    "acmecorp.co.uk"   → "Acmecorp"
    "hubspot.com"      → "Hub Spot"

    Strategy:
      1. Strip TLD(s) including compound ones
      2. Split on hyphens/underscores (explicit word boundaries)
      3. For single-word slugs, try greedy common-word decomposition
    """
    name = re.sub(
        r"\.(com\.au|co\.uk|co\.nz|com\.ng|com|net|org|io|co|ai|de|fr|"
        r"se|nl|sg|ie|es|in|ca|app|dev|tech|xyz|info|biz)$",
        "", domain, flags=re.IGNORECASE,
    )
    if not name:
        return domain

    # Hyphen/underscore — straightforward split
    if "-" in name or "_" in name:
        return name.replace("-", " ").replace("_", " ").title().strip()

    # Single word slug — greedy common-word decomposition
    _COMMON = {
        "the", "real", "deal", "tech", "hub", "spot", "sales", "force",
        "stack", "flow", "base", "app", "web", "cloud", "net", "soft",
        "ware", "data", "labs", "lab", "works", "work", "group", "media",
        "wire", "hive", "mind", "box", "bay", "bridge", "link", "sync",
        "bright", "smart", "quick", "fast", "easy", "open", "free", "pro",
        "core", "edge", "wave", "spark", "vault", "pulse", "scale", "grow",
        "path", "mark", "map", "flat", "black", "white", "blue", "green",
        "red", "new", "fresh", "lead", "next", "home", "shop", "pay",
        "point", "line", "gate", "port", "yard", "side", "light", "space",
    }

    slug  = name.lower()
    words = []
    i     = 0
    while i < len(slug):
        matched = False
        for length in range(min(10, len(slug) - i), 1, -1):
            candidate = slug[i:i + length]
            if candidate in _COMMON:
                words.append(candidate)
                i += length
                matched = True
                break
        if not matched:
            words.append(slug[i:])
            break

    return " ".join(words).title().strip() if words else name.title()


# People signal words — a page must contain at least one to be worth parsing
_PEOPLE_SIGNALS = {
    "founder", "ceo", "cto", "cfo", "coo", "director", "president",
    "team", "leadership", "management", "about", "people", "our story",
    "head of", "vice president", "managing director", "owner",
}


async def _fetch_one_path(domain: str, path: str) -> tuple[str, str]:
    """Fetch a single path for a domain. Returns (url, html) or (url, '')."""
    url  = f"https://{domain}{path}"
    html = await _fetch_url(url)
    return url, html


async def fetch_team_pages(domain: str) -> list[tuple[str, str]]:
    """
    Fetch team/people pages for a domain concurrently.

    All paths are fetched in parallel using asyncio.gather with a semaphore
    limiting simultaneous connections to 5 — so all 13 paths fire at once
    instead of sequentially. Total time per domain is the slowest single
    request, not the sum of all requests.

    Returns up to 2 pages that pass the people-signal content gate,
    prioritised by path order (team > about > impressum etc.).
    """
    sem = asyncio.Semaphore(5)

    async def _guarded(path: str) -> tuple[str, str]:
        async with sem:
            return await _fetch_one_path(domain, path)

    # Fire all path fetches concurrently
    fetched = await asyncio.gather(
        *[_guarded(path) for path in _TEAM_PATHS],
        return_exceptions=True,
    )

    results = []
    for item in fetched:
        if isinstance(item, Exception):
            continue
        url, html = item
        if not html or len(html) < 500:
            continue
        preview = html[:3000].lower()
        if any(signal in preview for signal in _PEOPLE_SIGNALS):
            results.append((url, html))
        if len(results) >= 2:
            break

    return results


# Additional words that produce garbage names from nav/UI elements
# These slip through _NOT_NAME_WORDS because they start with capitals
# Country, state, city names that can appear as two-word Title Case strings
# and pass _is_person_name incorrectly e.g. "Ivory Coast", "Los Angeles"
_GEOGRAPHY_WORDS = {
    # Countries / territories
    "states", "kingdom", "republic", "coast", "islands", "island",
    "emirates", "africa", "europe", "asia", "america", "americas",
    "zealand", "guinea", "leone", "verde", "marino", "lucia",
    "tobago", "kitts", "nevis", "grenadines", "herzegovina",
    # US cities / regions
    "angeles", "francisco", "antonio", "diego", "orleans",
    "francisco", "jose", "vegas", "worth", "paso",
    # Sports teams (second word)
    "orioles", "braves", "diamondbacks", "yankees", "dodgers",
    "giants", "mets", "cubs", "sox", "astros", "cardinals",
    "nationals", "phillies", "brewers", "pirates", "reds",
    "padres", "rockies", "marlins", "rays", "rangers", "angels",
    "athletics", "mariners", "twins", "tigers", "indians", "royals",
    "bluejays", "jays",
    # Caribbean / territory names
    "rico", "leone", "faso",
}

_NAV_GARBAGE = {
    # Original nav/UI labels
    "page", "found", "modal", "header", "quick", "links", "link",
    "password", "recovery", "right", "where", "belong", "belongs",
    "core", "activities", "activity", "selected", "awards", "award",
    "other", "options", "option", "site", "find", "us", "on",
    "cookie", "cookies", "notice", "accessibility", "assistance",
    "directory", "visit", "library", "facilities", "articles",
    "features", "social", "network", "networks", "loading", "error",
    "close", "open", "click", "skip", "next", "back", "more",
    "view", "show", "hide", "sign", "log", "reset",
    # Navigation sections seen in results
    "sales", "creators", "gifting", "managers", "navigation",
    "guidelines", "coverage", "foundation", "generator", "empowers",
    "disclaimer", "impressum", "commitments", "expectations",
    # Multi-word nav labels (first word catches the pair)
    "for", "who", "corporate", "community", "primary", "global",
    "online", "solid", "rev",
    # Geography mistaken for names
    "coast", "republic", "angeles", "angeles",
    "orioles", "braves", "diamondbacks", "yankees", "dodgers",
    # Data / list product names
    "maelumat", "coverage", "database", "dataset", "list",
    # Design / product UI labels
    "design", "product", "development", "engineering", "analytics",
    "operations", "infrastructure", "architecture", "implementation",
    # Marketing / SEO garbage
    "positive", "reviews", "hire", "mobile", "developers", "designer",
    "daylight", "energy", "across", "always", "active", "automation", "testing",
}


# Valid name-part characters — only letters, hyphens and apostrophes.
# No colons, digits, underscores, brackets, or other punctuation.
# Apostrophes allowed for O'Brien, hyphens for Mary-Jane.
_NAME_PART_RE = re.compile(r"^[\w][\w\'\-]*$", re.UNICODE)

# Suffix patterns that identify non-name words.
# Tradeoff: rejects "Xiaoying" (Chinese -ing name) but eliminates
# "Opening", "Building", "Serving", "Sourcing" etc.
#
# FIRST word: gerunds (-ing), past-participles (-ed), adverbs (-ly),
#             plural occupational nouns, and noun suffixes
_FIRST_WORD_REJECT_SUFFIX = re.compile(
    r"(ing|ings|ed|ly|ers|ors|ees|ists|ants|ents|"
    r"tion|sion|ment|ness|ity|ance|ence|ship|ture|ism|ics|age)$",
    re.IGNORECASE,
)

# LAST word: same patterns — "Proudly Serving", "Deeper Dive" etc.
# NOTE: "-er" intentionally excluded — too many real surnames end in -er
# (Müller, Kanfer, Schneider, Fisher, Turner etc.)
# Those are caught by _COMMON_NOUN_LAST_WORDS instead.
_LAST_WORD_REJECT_SUFFIX = re.compile(
    r"(ing|ings|ed|ly|ers|ors|ees|ists|ants|ents|"
    r"tion|sion|ment|ness|ity|ance|ence|ship|ture|ism|ics|age)$",
    re.IGNORECASE,
)

# Common English nouns/terms that never appear as surnames.
# Catches words that slip through suffix checks.
_COMMON_NOUN_LAST_WORDS = {
    # Food / consumer
    "burger", "burgers", "pizza", "store", "stores", "shop", "shops",
    "market", "markets", "mall", "cafe", "diner", "bar", "bars",
    # Business descriptors
    "gen", "pro", "plus", "max", "lite", "basic", "premium",
    "enterprise", "startup", "agency", "agencies", "firm", "firms",
    "models", "model", "template", "templates",
    # Content / media
    "posts", "post", "rules", "rule", "hours", "hour", "winds", "wind",
    "updates", "update", "magazine", "expertise", "mix", "mixes",
    "soccer", "voices", "voice", "form", "forms", "mode", "modes",
    "news", "views", "view", "tips", "tip", "guide", "guides",
    "hub", "lab", "labs", "studio", "studios", "media", "press",
    "group", "groups", "club", "clubs", "zone", "zones",
    "watch", "wire", "board", "boards", "space", "spaces",
    "world", "globe", "sphere", "stream", "streams", "feed", "feeds",
    "brief", "briefs", "report", "reports", "digest", "review", "reviews",
    "weekly", "daily", "monthly", "times", "journal", "journals",
    "today", "now", "live", "direct", "central", "connect",
    "network", "networks", "online", "digital",
    # Actions / verbs used as section titles
    "dive", "dives", "deep", "intro", "overview", "summary",
    "spotlight", "showcase", "case", "cases", "demo", "demos",
    "search", "explore", "discover", "browse", "find",
    "hire", "apply", "join", "start", "launch", "grow",
    # Abstract / descriptive
    "beauty", "style", "fashion", "lifestyle", "wellness",
    "culture", "vision", "mission", "impact", "story", "stories",
    "insight", "insights", "trend", "trends", "data", "analytics",
    "demand", "supply", "leads", "lead", "funnel", "pipeline",
    # Tech product / AI buzzwords used as section labels
    "layer", "layers", "agent", "agents", "node", "nodes",
    "module", "modules", "engine", "engines", "framework",
    "platform", "platforms", "stack", "stacks", "suite", "suites",
    "touchpoint", "touchpoints", "checkpoint", "checkpoints",
    "endpoint", "endpoints", "workflow", "workflows",
    "pipeline", "pipelines", "gateway", "gateways",
    "interface", "interfaces", "connector", "connectors",
    "dashboard", "dashboards", "portal", "portals",
    "toolkit", "toolkits", "playbook", "playbooks",
    # Product descriptors as last word
    "core", "base", "edge", "cloud", "mesh", "grid",
    "vault", "forge", "pulse", "signal", "beacon",
    "bridge", "link", "links", "path", "paths",
    "point", "points", "node", "hub", "hubs",
    # Business / ops / section labels not caught above
    "ops", "mail", "project", "projects",
    "approach", "approaches", "relations", "affairs",
    "blogs", "blog", "articles", "article",
    "policy", "policies", "terms", "conditions", "guidelines",
    "packages", "package", "features", "feature",
    "benefits", "benefit", "resources", "resource",
    "tools", "tool", "careers", "career", "roles", "role",
    "events", "event", "notices", "notice", "alerts", "alert",
}

# Words that are definitively not name parts — combined blocklist
# Used as a secondary check after structural validation.
_ALL_NON_NAME = (
    _NOT_NAME_WORDS
    | _NAV_GARBAGE
    | _GEOGRAPHY_WORDS
    | {
        # Common English words that appear in nav/section labels
        "we", "embrace", "change", "create", "employment", "opportunities",
        "middle", "upper", "lower", "school", "college", "university",
        "specialisation", "specialization", "areas", "area", "sector",
        "experience", "practical", "peer", "groups", "group",
        "chat", "connect", "explore", "discover", "build", "launch",
        "grow", "scale", "learn", "start", "join", "apply",
        "services", "service", "products", "solutions", "solution",
        "overview", "summary", "intro", "introduction", "welcome",
        "partners", "clients", "customers", "investors", "advisors",
        "portfolio", "ventures", "holdings", "associates", "consulting",
        "technology", "technologies", "systems", "digital", "global",
        "international", "national", "regional", "local",
        "industries", "industry", "sectors", "markets", "market",
        "insights", "resources", "events", "event", "conference",
        "workshops", "workshop", "training", "course", "program",
        "pricing", "plans", "plan", "features", "feature",
        "careers", "career", "jobs", "job", "hiring", "talent",
        "culture", "values", "value", "mission", "vision", "purpose",
        "impact", "results", "success", "case", "stories", "story",
        # Marketing/Signal words
        "marketing", "platform", "discovery", "enrichment", "humans",
        "people", "team", "posts", "updates", "privacy", "policy",
        "terms", "legal", "contact", "support", "help", "guides",
        "documentation", "menu", "search", "login", "signup", "register",
        "about", "values", "mission", "vision", "client", "agency",
        "creative", "strategy", "growth", "revenue", "sales", "support",
        "service", "services", "reviews", "solutions", "solution", "hire",
        "office", "fraud", "prevention", "satisfaction", "turnaround", "americans",
        "partner", "analysis", "situation", "google", "premium", "premier",
        # Adjectives / quantifiers that start section labels not names
        "multiple", "single", "dual", "triple", "unified", "integrated",
        "automated", "intelligent", "adaptive", "advanced", "enhanced",
        "agentic", "generative", "conversational", "predictive", "proactive",
        "real", "virtual", "hybrid", "omni", "cross", "inter",
        "next", "future", "smart", "dynamic", "scalable", "modular",
        "end", "full", "self", "zero", "low", "high", "ultra",
        # Additional adjectives seen as first words in bad names
        "latest", "recent", "current", "agile", "lean", "digital",
        "modern", "leading", "trusted", "proven", "award", "award-winning",
        "investor", "investors", "outsource", "outsourced", "dedicated",
        "comprehensive", "complete", "expert", "experts", "specialist",
        "guardian", "premier", "premium", "professional", "professionals",
    }
)

# ─────────────────────────────────────────────────────────────────────────────
# Statistical Name Filter (Lite NLP)
# ─────────────────────────────────────────────────────────────────────────────

_name_filter_cache: dict[str, set[str]] = {}

def _load_name_filter() -> dict[str, set[str]]:
    if _name_filter_cache:
        return _name_filter_cache
    
    import json as _json
    from pathlib import Path
    
    # Path relative to this file: scrylo/scrapers/finder.py -> scrylo/data/name_filter.json
    data_path = Path(__file__).parent.parent / "data" / "name_filter.json"
    
    try:
        with data_path.open("r", encoding="utf-8") as f:
            data = _json.load(f)
            _name_filter_cache["names"] = set(data.get("common_names", []))
            _name_filter_cache["garbage"] = set(data.get("garbage_prefixes", []))
    except Exception as e:
        log.debug("Could not load name_filter.json: %s", e)
        _name_filter_cache["names"] = set()
        _name_filter_cache["garbage"] = set()
        
    return _name_filter_cache


def _is_person_name(text: str) -> bool:
    """
    True if text looks like a real human name.
    Uses structural checks + Lite NLP (common names/garbage prefixes).
    """
    text = text.strip()
    if not text or len(text) > 60:
        return False

    # 1. Structural rejection
    if text[-1] in ".!?:;,": return False
    if text.isupper():       return False

    words = text.split()
    if not (2 <= len(words) <= 4):
        return False

    # 2. Strip leading honorific for analysis
    analysis_words = words[:]
    if analysis_words and re.sub(r"[^a-z]", "", analysis_words[0].lower()) in _HONORIFICS:
        analysis_words = analysis_words[1:]
    if not analysis_words:
        return False

    # 3. Lite NLP — check first word against statistical dictionaries
    filters    = _load_name_filter()
    first_word = analysis_words[0].lower().rstrip("'-")
    
    # High confidence: explicitly a name
    is_common_name = first_word in filters["names"]
    
    # High risk: explicitly a verb/adjective that isn't a name
    is_garbage     = first_word in filters["garbage"]
    
    if is_garbage and not is_common_name:
        return False

    # 4. Suffix checks on first and last words — culturally neutral
    # These catch gerunds, past participles, plural occupational nouns
    # without relying on any name dictionary.
    first_word = analysis_words[0].lower().rstrip("'-")
    last_word  = analysis_words[-1].lower().rstrip("'-")

    # First word suffix: "Opening", "Building", "Related", "Raised",
    # "Manufacturers", "Certified", "Editors" → all rejected
    if _FIRST_WORD_REJECT_SUFFIX.search(first_word):
        return False

    # Last word suffix: "Administrators", "Contributors", "Leaders" → rejected
    if _LAST_WORD_REJECT_SUFFIX.search(last_word):
        return False

    # Last word common noun: "Posts", "Rules", "Hours", "Winds" → rejected
    if last_word in _COMMON_NOUN_LAST_WORDS:
        return False

    # 5. Word-by-word validation
    for word in analysis_words:
        # Structure check
        if not _NAME_PART_RE.match(word):
            return False
        if not (2 <= len(word) <= 20):
            return False
        if not word[0].isupper():
            return False

        # Internal uppercase check (MacLeod ok, SaaS bad)
        internal_uppers = [c for c in word[1:] if c.isupper()]
        if len(internal_uppers) > 1:
            return False
        if len(internal_uppers) == 1:
            upper_pos = next(i for i, c in enumerate(word[1:], 1) if c.isupper())
            if word[upper_pos - 1] not in ("-", "'"):
                return False

        # Semantic check (Blocklist)
        w = word.lower().rstrip("'-")
        if w in _ALL_NON_NAME:
            return False

    return True


def _extract_emails_from_html(html: str, domain: str = "") -> list[str]:
    import html as _html_mod
    text = _html_mod.unescape(html)
    text = re.sub(r"\[at\]|\(at\)", "@", text, flags=re.IGNORECASE)
    text = re.sub(r"\[dot\]|\(dot\)", ".", text, flags=re.IGNORECASE)
    all_candidates = re.findall(r'[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}', text)
    
    result = []
    seen = set()
    for em in all_candidates:
        em = em.lower().strip().rstrip(".")
        if em in seen: continue
        seen.add(em)
        prefix = em.split("@")[0]
        if prefix in _GENERIC_PREFIXES: continue
        if not domain or domain in em:
            result.append(em)
    return result


def _split_name_simple(full_name: str) -> tuple[str, str]:
    parts = full_name.strip().lower().split()
    while parts and re.sub(r"[^a-z]", "", parts[0]) in _HONORIFICS: parts = parts[1:]
    if not parts: return "", ""
    return re.sub(r"[^a-z]", "", parts[0]), re.sub(r"[^a-z]", "", parts[-1])


def _semantic_extract(html: str, domain: str, company: str) -> list[FoundLead]:
    """
    Extract people from a team/about page using semantic HTML structure.
    company param is used only when it looks like a real company name —
    falls back to domain-derived name when co_hint is an article headline.
    """
    soup  = BeautifulSoup(html, "html.parser")
    leads = []
    seen  = set()
    all_page_emails = _extract_emails_from_html(html, domain)

    # Validate the company hint — reject it if it looks like an article headline.
    # Heuristics: real company names are short, don't contain verbs/articles,
    # and don't end with common headline words.
    def _is_valid_company(name: str) -> bool:
        if not name or len(name) > 60:
            return False
        # Looks like a sentence/headline (contains common article words)
        headline_signals = {
            "the", "and", "or", "for", "with", "among", "amid", "how",
            "why", "what", "when", "where", "led", "says", "grows",
            "raises", "launches", "joins", "reveals", "announces",
        }
        words = name.lower().split()
        # More than 6 words = almost certainly a headline
        if len(words) > 6:
            return False
        # Contains verb/article = headline
        if sum(1 for w in words if w in headline_signals) >= 2:
            return False
        return True

    # Domain is the only source of truth for company name.
    # The Serper result title (co_hint) is often an article headline, a
    # directory listing title, or a page title — not the company name.
    # swissnex.org → "Swissnex", not "Startups" or whatever Serper said.
    clean_company = _company_from_domain(domain) if domain else (
        company if _is_valid_company(company) else ""
    )

    # If any emails were found on this page, infer and cache the pattern.
    # This means every person extracted from the same domain will get
    # their email guessed using the pattern this company actually uses,
    # rather than always defaulting to first.last@
    if all_page_emails and domain and domain not in _domain_pattern_cache:
        for em in all_page_emails:
            prefix = em.split("@")[0]
            # Try to match against any reasonable first/last combo from
            # a quick scan of name tags — just need one confirmed match
            for tag_scan in soup.find_all(_NAME_TAGS):
                name_scan = tag_scan.get_text(strip=True)
                if not _is_person_name(name_scan):
                    continue
                f_scan, l_scan = _split_name_simple(name_scan)
                if not f_scan:
                    continue
                pattern = _infer_email_pattern(em, f_scan, l_scan)
                if pattern:
                    _domain_pattern_cache[domain] = pattern
                    log.debug("Domain %s pattern inferred: %s", domain, pattern)
                    break
            if domain in _domain_pattern_cache:
                break

    for tag in soup.find_all(_NAME_TAGS):
        name = tag.get_text(strip=True)
        if not _is_person_name(name) or name.lower() in seen:
            continue

        # Reject generic corporate names (e.g. "Cepedwebit Support")
        if clean_company and clean_company.lower() in name.lower():
            continue

        # Real-time deduplication — skip if already in vault
        with db_session() as db:
            co_filter = clean_company if clean_company else None
            exists = db.query(Lead).filter(
                Lead.full_name == name,
                Lead.company   == co_filter
            ).first()
            if exists:
                seen.add(name.lower())
                continue

        # Look for title in adjacent siblings (up to 3)
        title = ""
        sibling = tag.next_sibling
        for _ in range(3):
            if sibling and hasattr(sibling, "get_text"):
                txt = sibling.get_text(strip=True)
                if 3 < len(txt) < 120:
                    title = txt
                    break
            sibling = sibling.next_sibling if sibling else None

        # Also check parent container children for title
        if not title and tag.parent:
            for child in tag.parent.find_all(_TITLE_TAGS, recursive=False):
                txt = child.get_text(strip=True)
                if txt != name and 3 < len(txt) < 120:
                    title = txt
                    break

        # Reject if Name == Title (likely a section header)
        if title.lower() == name.lower():
            continue

        first, last = _split_name_simple(name)
        email = ""
        for em in all_page_emails:
            if first and last and first in em and last in em:
                email = em
                break

        # LinkedIn URL from same card
        linkedin_url = ""
        if tag.parent:
            for a in tag.parent.find_all("a", href=True):
                if "linkedin.com/in/" in a["href"].lower():
                    linkedin_url = a["href"].split("?")[0]
                    break

        seen.add(name.lower())
        leads.append(FoundLead(
            name=name, title=title, company=clean_company,
            domain=domain, email=email, linkedin_url=linkedin_url,
        ))
    return leads


def _impressum_extract(html: str, domain: str, company: str) -> list[FoundLead]:
    soup         = BeautifulSoup(html, "html.parser")
    text         = soup.get_text("\n", strip=True)
    leads        = []
    seen         = set()
    clean_co     = _company_from_domain(domain) if domain else company
    for line in text.split("\n"):
        if any(m in line.lower() for m in ["geschäftsführer", "vertreten durch", "founder", "ceo"]):
            cand = re.sub(r".*[:\s]", "", line).strip()
            if _is_person_name(cand) and cand.lower() not in seen:
                # Real-time deduplication — skip if already in vault
                with db_session() as db:
                    exists = db.query(Lead).filter(
                        Lead.full_name == cand,
                        Lead.company   == clean_co
                    ).first()
                    if exists:
                        seen.add(cand.lower())
                        continue

                seen.add(cand.lower())
                leads.append(FoundLead(name=cand, title="Founder / MD (Impressum)", company=clean_co, domain=domain))
    return leads


# ─────────────────────────────────────────────────────────────────────────────
# LLM lead filter (optional — requires Groq key or Ollama)
# ─────────────────────────────────────────────────────────────────────────────

# System prompt for the filter — intentionally tiny to minimise token usage.
# Uses llama-3.1-8b-instant (14,400 req/day free, 6K TPM) — the parse model,
# not the pitch model — so this never eats into pitch quota.
# Few-shot prompt for Llama models.
# Llama responds best to: explicit format + few-shot examples showing
# the exact transformation. No long explanations — just show it what to do.
_FILTER_SYSTEM = (
    "Task: Given a numbered list of entries, output ONLY the indices of real human names. "
    "STRICTLY REJECT: \n"
    "- Marketing slogans (e.g., 'Positively Engaging Marketing')\n"
    "- Team descriptors (e.g., 'Brilliant Humans', 'Our Team')\n"
    "- Product/Service names (e.g., 'Data Enrichment', 'Creator Discovery')\n"
    "- Navigation labels or page sections (e.g., 'About Us', 'Contact').\n\n"
    "Examples:\n"
    "Input:\n"
    "0: Sarah Jenkins, Founder\n"
    "1: Positively Engaging Marketing\n"
    "2: Marcus Thorne, CEO\n"
    "3: Brilliant Humans\n"
    "Output: [0,2]\n\n"
    "Now output ONLY the JSON array. No explanation. No other text."
)


def _get_best_ollama_model_for_filter() -> str:
    """
    Get best available Ollama model. Returns "" if Ollama not running/no models.
    Mirrors llm_adapter._best_ollama_model without importing it.
    """
    try:
        r = httpx.get("http://localhost:11434/api/tags", timeout=2)
        if r.status_code != 200:
            return ""
        models = [m["name"] for m in r.json().get("models", [])]
        if not models:
            return ""
        for pref in ["llama3.3", "llama3.1:70b", "llama3.1", "llama3.2",
                     "llama3", "mistral", "mixtral", "gemma2"]:
            match = next((m for m in models if pref in m), None)
            if match:
                return match
        return models[0]
    except Exception:
        return ""


def _parse_filter_indices(text: str) -> list[int] | None:
    """Parse a JSON array of integers from LLM response. Returns None on failure."""
    import json as _json
    try:
        match = re.search(r"\[[\d,\s]*\]", text)
        if match:
            indices = _json.loads(match.group(0))
            if isinstance(indices, list):
                return [i for i in indices if isinstance(i, int)]
    except Exception:
        pass
    return None


def _llm_filter_sync(leads: list[FoundLead], groq_key: str) -> list[int] | None:
    """
    Synchronous LLM filter — called via run_in_executor.
    Returns list of valid indices, or None if all providers failed.
    None = silent fallback, caller uses all leads unfiltered.

    Provider order: Groq → Ollama.
    Groq failure (any reason including 429) falls through to Ollama.
    Only returns None when both providers are unavailable or both fail.
    """
    if not leads:
        return None

    entries = []
    for i, l in enumerate(leads):
        title_part = f", {l.title[:40]}" if l.title else ""
        entries.append(f"{i}: {l.name}{title_part}")
    user_text = "\n".join(entries)

    # ── Groq ──────────────────────────────────────────────────────────────────
    if groq_key:
        try:
            resp = httpx.post(
                "https://api.groq.com/openai/v1/chat/completions",
                headers={"Authorization": f"Bearer {groq_key}",
                         "Content-Type": "application/json"},
                json={
                    "model":       "llama-3.1-8b-instant",  # parse model — preserves pitch quota
                    "messages":    [{"role": "system", "content": _FILTER_SYSTEM},
                                    {"role": "user",   "content": user_text}],
                    "max_tokens":  80,
                    "temperature": 0.0,
                },
                timeout=10,
            )
            if resp.status_code == 200:
                indices = _parse_filter_indices(
                    resp.json()["choices"][0]["message"]["content"].strip()
                )
                if indices is not None:
                    log.debug("LLM filter: Groq filtered %d → %d leads",
                              len(leads), len(indices))
                    return indices
                log.debug("LLM filter: Groq unparseable response — trying Ollama")
            elif resp.status_code == 429:
                log.debug("LLM filter: Groq quota exceeded — trying Ollama")
            else:
                log.debug("LLM filter: Groq HTTP %d — trying Ollama", resp.status_code)
        except Exception as e:
            log.debug("LLM filter: Groq error (%s) — trying Ollama", e)

    # ── Ollama ────────────────────────────────────────────────────────────────
    ollama_model = _get_best_ollama_model_for_filter()
    if ollama_model:
        try:
            resp = httpx.post(
                "http://localhost:11434/api/chat",
                json={
                    "model":    ollama_model,
                    "messages": [{"role": "system", "content": _FILTER_SYSTEM},
                                  {"role": "user",   "content": user_text}],
                    "options":  {"temperature": 0.0, "num_predict": 80},
                    "stream":   False,
                },
                timeout=30,
            )
            if resp.status_code == 200:
                indices = _parse_filter_indices(
                    resp.json()["message"]["content"].strip()
                )
                if indices is not None:
                    log.debug("LLM filter: Ollama (%s) filtered %d → %d leads",
                              ollama_model, len(leads), len(indices))
                    return indices
                log.debug("LLM filter: Ollama unparseable response")
            else:
                log.debug("LLM filter: Ollama HTTP %d", resp.status_code)
        except Exception as e:
            log.debug("LLM filter: Ollama error (%s)", e)
    else:
        log.debug("LLM filter: Ollama not available")

    # Both providers failed — silent fallback
    return None


async def _llm_filter(leads: list[FoundLead], groq_key: str) -> list[FoundLead]:
    """
    Async wrapper — runs LLM filter in executor.
    Returns a list of leads — either filtered or original (on system error only).
    """
    if not leads:
        return leads

    try:
        loop    = asyncio.get_event_loop()
        indices = await loop.run_in_executor(
            None, _llm_filter_sync, leads, groq_key
        )
        
        # indices is None means the API CALL failed (network/429/key error).
        # In this case we MUST return original leads (silent fallback).
        if indices is None:
            return leads
            
        # If indices is [] (empty list), it means the LLM successfully 
        # identified that NO leads in this batch are real people.
        # We trust the LLM and return the empty list.
        filtered = [leads[i] for i in indices if i < len(leads)]
        removed  = len(leads) - len(filtered)
        if removed > 0:
            log.info("LLM filter removed %d non-person leads", removed)
        return filtered

    except Exception as e:
        log.debug("LLM filter exception: %s — skipping", e)
        return leads


# ─────────────────────────────────────────────────────────────────────────────
# Scoring & Save logic
# ─────────────────────────────────────────────────────────────────────────────

def _score(lead: FoundLead, query: str = "") -> float:
    s = 0.4
    if _query_wants_seniority(query):
        tier = _title_tier(lead.title)
        s += {3: 0.4, 2: 0.2, 1: 0.1}.get(tier, -0.2)
    if lead.email: s += 0.1
    if lead.linkedin_url: s += 0.1
    return round(min(max(s, 0.0), 1.0), 2)


def _save(
    db:         Session,
    lead:       FoundLead,
    query:      str,
    email:      str   = "",
    confidence: float = 0.0,
    verified:   bool  = False,
    source:     str   = "serper",
) -> Optional[int]:
    """
    Save a FoundLead to DB.
    """
    if not lead.name or not lead.is_valid(): return None

    first, last = _split_name_simple(lead.name)
    final_email = email or lead.email or ""

    # Drop leads with no actionable contact path — useless to the user
    # Must have: email OR linkedin URL OR github URL.
    if not final_email and not lead.linkedin_url and not lead.github_url:
        return None

    exists = db.query(Lead).filter(
        Lead.full_name == lead.name,
        Lead.company   == lead.company,
    ).first()
    if exists: return None

    # Tiering: 1=Email, 3=LinkedIn, 4=GitHub (signal), 2=Partial (deprecated logic check)
    t = 1 if final_email else (3 if lead.linkedin_url else (4 if lead.github_url else 2))
    tier_caps = {1: 100, 2: 65, 3: 45, 4: 40, 0: 25}
    icp_score = min(int(_score(lead, query) * 100), tier_caps.get(t, 25))

    db_lead = Lead(
        full_name        = lead.name,
        first_name       = first,
        last_name        = last,
        role             = lead.title or None,
        company          = lead.company or None,
        location         = lead.location or None,
        website          = f"https://{lead.domain}" if lead.domain else None,
        email            = final_email or None,
        email_verified   = verified,
        email_confidence = confidence if final_email else 0.0,
        icp_score        = icp_score,
        status           = LeadStatus.NEW,
        source           = source,
        linkedin_url     = lead.linkedin_url or None,
        github_url       = lead.github_url or None,
    )
    db.add(db_lead)
    db.flush()
    return db_lead.id


# ─────────────────────────────────────────────────────────────────────────────
# Main Discovery Pipeline
# ─────────────────────────────────────────────────────────────────────────────

async def find_leads(query: str, limit: int = 40, progress_callback = None, fetch_pages: bool = True) -> FinderResult:
    result = FinderResult(query=query)
    cfg = load_cfg()
    api_key  = (cfg.get("scraper", {}).get("serper_api_key") or "").strip()
    groq_key = (cfg.get("llm", {}).get("groq_api_key") or "").strip()
    if not api_key: return result

    all_tried = set()
    # Clear per-session caches
    _domain_mx_cache.clear()
    _domain_pattern_cache.clear()
    # Calculate LinkedIn quota once for the entire session — not per pass.
    # 30% of requested limit reserved for LinkedIn leads across all passes.
    linkedin_quota = max(1, round(limit * 0.30))
    # Accumulate LinkedIn profiles across all passes — deduplicated by URL.
    # Without this, each pass discards LinkedIn entries from previous passes.
    all_linkedin_entries: list[tuple[str, dict]] = []

    for pass_num in range(1, 5):
        shortfall = limit - result.leads_saved
        if shortfall <= 0: break
        
        if progress_callback: await progress_callback("searching", result.leads_saved, limit, f"🔍 Pass {pass_num}...")
        domains = discover_company_domains(query, api_key, shortfall, pass_num, all_tried)
        
        domain_saves: dict[str, int] = {}
        per_domain_cap = max(2, limit // 10)

        # Cap how many company domains we actually fetch per pass.
        # LinkedIn _linkedin_ entries are free (no fetch) so always included.
        linkedin_entries = [(d, c) for d, c in domains if d == "_linkedin_"]
        company_entries  = [(d, c) for d, c in domains if d != "_linkedin_"]
        max_companies    = min(len(company_entries), max(shortfall * 3, 10))
        # Accumulate LinkedIn entries across passes — deduplicate by URL
        seen_li_urls = {lp.get("url", "") for _, lp in all_linkedin_entries}
        for d, lp in linkedin_entries:
            if isinstance(lp, dict) and lp.get("url", "") not in seen_li_urls:
                all_linkedin_entries.append((d, lp))
                seen_li_urls.add(lp.get("url", ""))

        # ── Step 1: Fetch company domains concurrently ───────────────────────
        # Company leads (with email) are processed batch-by-batch.
        # LinkedIn fills remaining slots only after company fetching completes.
        # ─────────────────────────────────────────────────────────────────────
        DOMAIN_CONCURRENCY = 4
        company_cap = limit - linkedin_quota

        async def _fetch_domain(domain: str, co_hint: str) -> list[FoundLead]:
            try:
                pages = await fetch_team_pages(domain)
            except Exception as e:
                log.debug("Domain fetch error %s: %s", domain, e)
                return []

            all_tried.add(domain)
            raw_leads = []
            for url, html in pages:
                extracted = (
                    _impressum_extract(html, domain, co_hint)
                    if "impressum" in url
                    else _semantic_extract(html, domain, co_hint)
                )
                raw_leads.extend(extracted)

            filtered_leads = await _llm_filter(raw_leads, groq_key)

            leads = []
            for lead in filtered_leads:
                lead.score = _score(lead, query)
                leads.append(lead)
            return leads

        async def _resolve_lead_email(lead: FoundLead) -> tuple[FoundLead, str, float, bool]:
            """Resolve email for one lead. Returns (lead, email, confidence, verified)."""
            if lead.email and "@" in lead.email:
                em_domain = lead.email.split("@")[1]
                has_mx    = await _mx_check(em_domain)
                if not has_mx:
                    return lead, "", 0.0, False
                return lead, lead.email, 0.95, True

            if lead.domain:
                first, last = _split_name_simple(lead.name)
                email, conf, verified = await _resolve_email(first, last, lead.domain)
                return lead, email, conf, verified

            return lead, "", 0.0, False

        # Process in batches of DOMAIN_CONCURRENCY so progress updates fire
        # between batches — UI stays live rather than freezing until all done
        targets = company_entries[:max_companies]
        for batch_start in range(0, len(targets), DOMAIN_CONCURRENCY):
            if (result.tier1 + result.tier2) >= company_cap:
                break

            batch = targets[batch_start:batch_start + DOMAIN_CONCURRENCY]

            if progress_callback:
                done  = batch_start
                total = len(targets)
                await progress_callback(
                    "fetching", result.leads_saved, limit,
                    f"🌐 Pass {pass_num} — Companies {done+1}-{min(done+DOMAIN_CONCURRENCY, total)}"
                    f" of {total}..."
                )

            batch_results = await asyncio.gather(
                *[_fetch_domain(d, c) for d, c in batch],
                return_exceptions=True,
            )

            batch_leads = []
            for item in batch_results:
                if isinstance(item, Exception):
                    continue
                batch_leads.extend(item)
            
            if not batch_leads:
                continue

            # ── Step 2: Resolve emails async for the batch ───────────────────
            # Email resolution (MX check + pattern inference) happens HERE —
            # before saving — so leads with undeliverable domains are dropped
            # immediately and their slots stay open for better leads.
            resolution_tasks = [_resolve_lead_email(l) for l in batch_leads]
            resolved = await asyncio.gather(*resolution_tasks, return_exceptions=True)

            # ── Step 3: Save the batch ───────────────────────────────────────
            with db_session() as db:
                for item in resolved:
                    if isinstance(item, Exception):
                        continue
                    l, email, confidence, verified = item
                    
                    company_leads_saved = result.tier1 + result.tier2
                    if company_leads_saved >= company_cap:
                        break
                    
                    lead_dom = l.domain or ""
                    if lead_dom and domain_saves.get(lead_dom, 0) >= per_domain_cap:
                        continue
                        
                    lid = _save(db, l, query, email=email,
                                confidence=confidence, verified=verified)
                    if lid:
                        result.lead_ids.append(lid)
                        result.leads_saved += 1
                        if lead_dom:
                            domain_saves[lead_dom] = domain_saves.get(lead_dom, 0) + 1
                        t = 1 if email else 2
                        if t == 1: result.tier1 += 1
                        else:      result.tier2 += 1
                    else:
                        result.leads_skipped += 1

        # ── Step 4: LinkedIn leads ────────────────────────────────────────────────
        # Normal case: company leads filled their 70% → LinkedIn fills its 30%.
        # Fallback case: company leads failed or were all bad → LinkedIn fills
        # 100% of remaining slots so the user always gets their requested limit.
        linkedin_already_saved    = result.tier3
        linkedin_remaining        = linkedin_quota - linkedin_already_saved
        shortfall_after_companies = limit - result.leads_saved
        company_leads_total       = result.tier1 + result.tier2

        if company_leads_total == 0:
            # Company fetching produced nothing — LinkedIn takes all slots
            linkedin_slots = shortfall_after_companies
        elif company_leads_total < company_cap:
            # Companies produced some but didn't fill their cap —
            # LinkedIn fills its quota PLUS the unused company slots
            unused_company_slots = company_cap - company_leads_total
            linkedin_slots = min(
                linkedin_remaining + unused_company_slots,
                shortfall_after_companies,
            )
        else:
            # Companies filled their full 70% — LinkedIn fills its 30% only
            linkedin_slots = min(linkedin_remaining, shortfall_after_companies)
        if linkedin_slots > 0 and all_linkedin_entries:
            if progress_callback:
                await progress_callback("saving", result.leads_saved, limit,
                                        "💾 Adding LinkedIn leads...")
            linkedin_leads = []
            seen_in_pass = set()
            for _, lp in all_linkedin_entries:
                raw_title   = lp.get("title", "")
                profile_url = lp.get("url", "")
                clean = re.sub(r"\s*\|\s*LinkedIn\s*$", "", raw_title,
                               flags=re.IGNORECASE).strip()
                name, title, company = "", "", ""
                if " - " in clean:
                    parts    = clean.split(" - ", 2)
                    name_raw = parts[0].strip()
                    rest     = parts[1].strip() if len(parts) > 1 else ""
                    if " at " in rest.lower():
                        at_idx       = rest.lower().index(" at ")
                        title        = rest[:at_idx].strip()
                        company_part = rest[at_idx + 4:].strip()
                        if company_part and 2 <= len(company_part) <= 60                                 and not company_part.endswith(("…", "...")):
                            company = company_part
                    else:
                        title = rest
                    name = name_raw
                else:
                    name = clean
                
                if not name or not _is_person_name(name):
                    continue

                # Reject generic corporate accounts (e.g. "Cepedwebit Support")
                if company and company.lower() in name.lower():
                    continue

                # Pass-level deduplication (e.g. multiple LinkedIn results for same person)
                name_key = name.lower()
                if name_key in seen_in_pass:
                    continue
                seen_in_pass.add(name_key)

                # Real-time deduplication — skip if already in vault
                with db_session() as db:
                    co_filter = company if company else None
                    exists = db.query(Lead).filter(
                        Lead.full_name == name,
                        Lead.company   == co_filter
                    ).first()
                    if exists:
                        continue

                lead = FoundLead(
                    name=name, title=title, company=company,
                    domain="", linkedin_url=profile_url, source_url=profile_url,
                )
                lead.score = _score(lead, query)
                linkedin_leads.append(lead)

            with db_session() as db:
                linkedin_saved_this_pass = 0
                for l in linkedin_leads:
                    if result.leads_saved >= limit:
                        break
                    if linkedin_saved_this_pass >= linkedin_slots:
                        break
                    lid = _save(db, l, query)
                    if lid:
                        result.lead_ids.append(lid)
                        result.leads_saved += 1
                        result.tier3 += 1
                        linkedin_saved_this_pass += 1
    
    if progress_callback: await progress_callback("done", result.leads_saved, limit, f"✓ {result.leads_saved} leads found")
    return result

def find_leads_sync(query: str, limit: int = 40, fetch_pages: bool = True) -> FinderResult:
    return asyncio.run(find_leads(query, limit, fetch_pages=fetch_pages))
