"""
scrylo/email_tools/engine.py
------------------------------
Three-tier email confidence model.

The fundamental rule: a domain is only usable for pattern-guessing if we
can CONFIRM it belongs to this person. Domains that merely appeared near
their name in a snippet are never trusted.

  TIER 1 — HIGH CONFIDENCE (0.95) — use directly, mark verified
    Source: email found on a page that contains the person's own name
    Source: GitHub API (email in profile)
    Source: email found via mailto: link on confirmed company page

  TIER 2 — MEDIUM CONFIDENCE (0.75) — pattern guess on confirmed domain
    Domain confirmed: Serper person-lookup returned it as top result AND
    the person's FULL NAME (first + last) appears on that domain's pages.
    Mark as unverified until SMTP confirms.

  TIER 3 — NEVER USE
    Domain appeared in snippet near person's name (not confirmed as employer)
    Domain is a media/conference/directory/job-board site
    Pattern guess on an unconfirmed domain

FIXES vs previous version:
  - _confirm_domain previously only required first_name on the page.
    Common names (John, Sarah, James) caused widespread false confirmations.
    Now requires BOTH first + last name present together in a ±50 char window.
  - _split_name in engine now strips honorifics (was inconsistent with
    finder.py's version) — prevents "dr.segun@..." style bad guesses.
  - _find_domain_for_company now sanitises the company name before building
    the Serper query, preventing 400 errors for companies with symbols.
  - verify_candidates now receives the known_pattern when we scraped a
    real email — so the pattern-guessed address uses the format we already
    know the company uses.
  - Catch-all domains from mx_validator are handled: confidence is capped
    at 0.5 so they don't bleed through as high-confidence leads.
  - _guess_domain now also tries .ai and .co.uk TLD variants.
"""

from __future__ import annotations

import logging
import re
from dataclasses import dataclass, field

from scrylo.db.models import Lead
from scrylo.db.session import db_session
from scrylo.email_tools.email_guesser import guess_emails, domain_from_url
from scrylo.email_tools.mx_validator import verify_candidates, check_mx, MxResult

log = logging.getLogger("scrylo.email_engine")

VERIFY_THRESHOLD = 0.7

# Per-batch caches — cleared at start of each run
_domain_mx_cache:  dict[str, MxResult | None] = {}
_website_cache:    dict[str, str]              = {}
_domain_confirmed: dict[str, bool]             = {}
_serper_cache:     dict[str, str]              = {}

_EMAIL_RE = re.compile(
    r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}", re.IGNORECASE
)
_VALID_TLD_RE = re.compile(
    r"\.[a-zA-Z]{2,6}(?:\.[a-zA-Z]{2})?$", re.IGNORECASE
)

_SKIP_PREFIXES = {
    "noreply", "no-reply", "donotreply", "support", "help", "hello",
    "info", "contact", "admin", "team", "careers", "jobs", "press",
    "media", "legal", "privacy", "sales", "billing", "accounts",
    "newsletter", "unsubscribe", "abuse", "security", "feedback",
    "enquiries", "enquiry", "office", "mail", "post", "webmaster",
    "reception", "general", "hi", "hey", "subs", "subscriptions",
    "subscribe", "notifications", "alerts", "do-not-reply", "bounce",
    "mailer", "daemon", "postmaster", "service", "services", "reply",
}

_SKIP_EMAIL_DOMAINS = {
    "rocketreach.co", "contactout.com", "hunter.io", "apollo.io",
    "zoominfo.com", "clearbit.com", "lusha.com", "seamless.ai",
    "snov.io", "findthatlead.com", "voilanorbert.com",
    "cnbc.com", "techcrunch.com", "forbes.com", "bloomberg.com",
    "businessinsider.com", "theguardian.com", "bbc.co.uk", "bbc.com",
    "ft.com", "wsj.com", "wired.com", "theregister.com",
    "saastr.com", "producthunt.com", "ycombinator.com", "fi.co",
    "techstars.com", "seedcamp.com", "antler.co",
    "wellfound.com", "angellist.com", "crunchbase.com", "glassdoor.com",
    "indeed.com", "linkedin.com", "clutch.co", "g2.com", "capterra.com",
    "twitter.com", "x.com", "facebook.com", "instagram.com",
    "youtube.com", "tiktok.com", "reddit.com",
    "econsultancy.com", "hubspot.com", "salesforce.com",
    "openai.com", "microsoft.com", "google.com", "amazon.com",
    "apple.com",
}

_HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/124.0.0.0 Safari/537.36"
    ),
    "Accept": "text/html,application/xhtml+xml",
}


# ── Honorifics (must match email_guesser.py) ──────────────────────────────

_HONORIFICS = {
    "dr", "mr", "mrs", "ms", "miss", "prof", "rev",
    "sir", "eng", "hon", "sr", "jr",
}


# ── Email extraction ───────────────────────────────────────────────────────

def _extract_emails(html: str, domain: str = "") -> list[str]:
    """
    Extract personal emails from HTML.
    Handles: plain text, mailto:, Cloudflare obfuscation, [at]/[dot] variants.
    Rejects: generic prefixes, broker domains, truncated emails, wrong domain.
    """
    import html as _html_mod
    text = _html_mod.unescape(html)
    text = re.sub(r"\[at\]",              "@", text, flags=re.IGNORECASE)
    text = re.sub(r"\(at\)",              "@", text, flags=re.IGNORECASE)
    text = re.sub(r"\s+at\s+(?=[a-z])",  "@", text)
    text = re.sub(r"\[dot\]|\(dot\)",     ".", text, flags=re.IGNORECASE)

    mailto    = re.findall(r'mailto:([^\s"\'<>?&]+)', text, re.IGNORECASE)
    cf_emails = []
    for encoded in re.findall(r'data-cfemail="([a-f0-9]+)"', text, re.IGNORECASE):
        try:
            data = bytes.fromhex(encoded)
            key  = data[0]
            cf_emails.append("".join(chr(b ^ key) for b in data[1:]))
        except Exception:
            pass
    regex_emails = _EMAIL_RE.findall(text)

    seen, result = set(), []
    for em in mailto + cf_emails + regex_emails:
        em = em.lower().strip().rstrip(".,;)'\"")
        if not em or "@" not in em or em in seen:
            continue
        seen.add(em)

        if any(em.endswith(x) for x in (".png", ".jpg", ".gif", ".svg", ".css", ".js")):
            continue

        prefix    = em.split("@")[0]
        em_domain = em.split("@")[1] if "@" in em else ""

        if prefix in _SKIP_PREFIXES:
            continue
        if not _VALID_TLD_RE.search(em_domain):
            continue
        if any(em_domain == s or em_domain.endswith("." + s)
               for s in _SKIP_EMAIL_DOMAINS):
            continue
        if re.search(r"\b(example|test|demo|sample|placeholder)\b", em):
            continue
        if domain:
            base = ".".join(domain.split(".")[-2:])
            if base not in em_domain:
                continue

        result.append(em)

    # Sort shortest local-part first — shorter = more likely personal
    result.sort(key=lambda e: len(e.split("@")[0]))
    return result


# ── Page fetching ──────────────────────────────────────────────────────────

def _fetch_page(url: str) -> str:
    """
    Fetch a single URL, return HTML or empty string.
    Runs async fetch in a new event loop since engine runs in executor thread.
    """
    try:
        import asyncio
        from scrylo.scrapers.page_fetcher import fetch_url

        try:
            loop = asyncio.get_event_loop()
            if loop.is_running():
                import concurrent.futures
                with concurrent.futures.ThreadPoolExecutor(max_workers=1) as ex:
                    future = ex.submit(asyncio.run, fetch_url(url))
                    return future.result(timeout=15) or ""
            else:
                return loop.run_until_complete(fetch_url(url)) or ""
        except RuntimeError:
            return asyncio.run(fetch_url(url)) or ""
    except Exception:
        pass

    try:
        import httpx
        resp = httpx.get(
            url, headers=_HEADERS,
            timeout=httpx.Timeout(connect=5.0, read=8.0, write=3.0, pool=3.0),
            follow_redirects=True,
        )
        if resp.status_code == 200:
            return resp.text
    except Exception:
        pass
    return ""


def _get_domain_pages(domain: str) -> str:
    """Fetch and concatenate HTML from most email-rich pages. Cached."""
    if domain in _website_cache:
        return _website_cache[domain]

    pages = [
        f"https://{domain}",
        f"https://{domain}/contact",
        f"https://{domain}/about",
        f"https://{domain}/team",
        f"https://{domain}/about-us",
        f"https://{domain}/people",
        f"https://{domain}/contact-us",
        f"https://{domain}/impressum",
        f"https://{domain}/imprint",
        f"https://{domain}/leadership",
        f"https://www.{domain}",
    ]
    combined = ""
    for url in pages:
        html = _fetch_page(url)
        if html:
            combined += html
            if len(combined) > 200_000:
                break

    _website_cache[domain] = combined
    return combined


# ── Domain confirmation ────────────────────────────────────────────────────

def _confirm_domain(domain: str, first_name: str, last_name: str = "") -> bool:
    """
    Confirm a domain belongs to this person by checking their name appears
    on the company's pages.

    IMPORTANT CHANGE vs previous version:
    The old logic only required first_name on the page. That caused widespread
    false positives — common names like "John", "Sarah", "James" appear on
    almost every company team page.

    New logic: both first AND last name must appear within 60 characters of
    each other in the page text. This is still fuzzy (doesn't require exact
    "First Last" sequence) but eliminates the false-positive flood from
    common first names coincidentally matching other employees.

    Falls back to first-name-only check only when last_name is not available.
    """
    if not domain or not first_name:
        return False

    cache_key = f"{domain}|{first_name.lower()}|{last_name.lower()}"
    if cache_key in _domain_confirmed:
        return _domain_confirmed[cache_key]

    html = _get_domain_pages(domain)
    if not html:
        _domain_confirmed[cache_key] = False
        return False

    text = html.lower()

    if last_name:
        # Require both names to appear within a 60-char window of each other.
        # This catches "Jane ... Smith" in team card HTML without requiring
        # exact full-name sequence (which CSS/tag splits would break).
        first_l = first_name.lower()
        last_l  = last_name.lower()

        # Find all positions of first name, check if last is nearby
        confirmed = False
        start = 0
        while True:
            pos = text.find(first_l, start)
            if pos == -1:
                break
            window = text[max(0, pos - 60): pos + 60 + len(first_l)]
            if last_l in window:
                confirmed = True
                break
            start = pos + 1
    else:
        # Only first name known — lower confidence check
        confirmed = first_name.lower() in text

    _domain_confirmed[cache_key] = confirmed
    log.debug(
        "Domain confirm %s for %s %s → %s",
        domain, first_name, last_name, confirmed
    )
    return confirmed


# ── Domain resolution ──────────────────────────────────────────────────────

def _serper_search(query: str, serper_key: str, num: int = 5) -> list[dict]:
    """Run a Serper search, return organic results. Cached."""
    if query in _serper_cache:
        import json
        return json.loads(_serper_cache[query])
    import httpx, json
    try:
        resp = httpx.post(
            "https://google.serper.dev/search",
            headers={"X-API-KEY": serper_key, "Content-Type": "application/json"},
            json={"q": query, "num": num},
            timeout=8,
        )
        if resp.status_code == 200:
            results = resp.json().get("organic", [])
            _serper_cache[query] = json.dumps(results)
            return results
    except Exception:
        pass
    return []


_DOMAIN_SKIP = {
    "linkedin.com", "wikipedia.org", "facebook.com", "twitter.com", "x.com",
    "bloomberg.com", "crunchbase.com", "glassdoor.com", "zoominfo.com",
    "trustpilot.com", "clutch.co", "g2.com", "rocketreach.co",
    "contactout.com", "hunter.io", "apollo.io", "wellfound.com",
    "angellist.com", "saastr.com", "producthunt.com", "techcrunch.com",
    "forbes.com", "businessinsider.com", "medium.com", "substack.com",
}


def _domain_from_url(url: str) -> str:
    d = re.sub(r"^https?://(www\.)?", "", url).split("/")[0].split(":")[0].lower()
    return d if "." in d else ""


def _sanitise_company_for_query(company: str) -> str:
    """
    Strip characters that cause Serper/Google 400 errors from company names.
    "A/B Corp" → "AB Corp",  "Data & AI Ltd" → "Data AI Ltd"
    Also strips legal suffixes that add noise to domain lookups.
    """
    # Remove characters Google treats as operators
    clean = re.sub(r"[/\\&|<>{}()\[\]^~*?!@#$%]", " ", company)
    # Collapse whitespace
    clean = re.sub(r"\s+", " ", clean).strip()
    # Strip trailing legal suffixes — they never appear in domains
    clean = re.sub(
        r"\b(ltd|limited|inc|llc|corp|corporation|plc|gmbh|bv|ag|srl|pty|co)\b\.?$",
        "", clean, flags=re.IGNORECASE,
    ).strip()
    return clean


def _find_domain_for_company(company: str, serper_key: str) -> str:
    """Find the official website domain for a known company name."""
    if not company or not serper_key:
        return ""
    # Sanitise before building query — prevents 400s for companies with symbols
    clean = _sanitise_company_for_query(company)
    if not clean:
        return ""
    results = _serper_search(f'"{clean}" official website', serper_key)
    for item in results:
        d = _domain_from_url(item.get("link", ""))
        if d and not any(s in d for s in _DOMAIN_SKIP):
            return d
    return ""


def _find_domain_for_person(
    name: str, role: str, serper_key: str
) -> tuple[str, str]:
    """
    Find company + domain for a person when we have no company info.
    Returns (company, domain) — confirms by checking FULL NAME on page.
    """
    if not name or not serper_key:
        return "", ""

    parts = name.split()
    first = parts[0]
    last  = parts[-1] if len(parts) > 1 else ""

    results = _serper_search(f'"{name}" {role}', serper_key)
    for item in results:
        d = _domain_from_url(item.get("link", ""))
        if not d or any(s in d for s in _DOMAIN_SKIP):
            continue
        # Require full name (first + last) on page — not just first name
        if _confirm_domain(d, first, last):
            snippet = item.get("snippet", "") + " " + item.get("title", "")
            co_match = re.search(
                rf"{re.escape(first)}.{{0,20}}(?:at|@)\s+([A-Z][^,\.\n]{{2,40}})",
                snippet, re.IGNORECASE,
            )
            company = co_match.group(1).strip().rstrip(".,") if co_match else ""
            return company, d

    return "", ""


def _guess_domain(company: str) -> list[str]:
    """
    Generate TLD candidates from company name slug.
    Tries .com, .io, .co, .ai and .co.uk — covers most startup domains.
    """
    if not company:
        return []
    noise = (r"\b(ltd|limited|inc|llc|plc|group|global|solutions|technologies|"
             r"tech|digital|ventures|consulting|and|of|the|b2b|ai|labs?)\b")
    name = re.sub(noise, "", company.lower(), flags=re.IGNORECASE)
    name = re.sub(r"[^a-z0-9]", "", name).strip()
    if len(name) < 3:
        return []
    return [
        f"{name}.com",
        f"{name}.io",
        f"{name}.co",
        f"{name}.ai",
        f"{name}.co.uk",
    ]


def _resolve_domain(candidates: list[str]) -> str:
    """Return first candidate that has MX records."""
    for d in candidates:
        if d in _domain_mx_cache:
            if _domain_mx_cache[d] and _domain_mx_cache[d].has_mx:
                return d
            continue
        probe = check_mx(f"probe@{d}")
        _domain_mx_cache[d] = probe
        if probe.has_mx:
            return d
    return ""


# ── GitHub ─────────────────────────────────────────────────────────────────

def _github_email(github_url: str) -> str | None:
    if not github_url or "github.com" not in github_url:
        return None
    try:
        username = github_url.rstrip("/").split("/")[-1]
        if not username:
            return None
        import httpx
        resp = httpx.get(
            f"https://api.github.com/users/{username}",
            headers={"Accept": "application/vnd.github.v3+json",
                     "User-Agent": "Scrylo/1.0"},
            timeout=6,
        )
        if resp.status_code == 200:
            email = resp.json().get("email") or ""
            if email and "@" in email and email.split("@")[0] not in _SKIP_PREFIXES:
                return email.lower()
    except Exception:
        pass
    return None


# ── Name splitting (honorific-aware) ──────────────────────────────────────

def _split_name(full_name: str) -> tuple[str, str]:
    """
    Split into (first, last), stripping honorific prefixes.
    Consistent with email_guesser._split_name to ensure the same
    first/last parts are used for both guessing and confirmation.

    "Dr. Segun Aina"  → ("segun", "aina")  — was ("dr", "aina") before
    "Mary Jane Smith" → ("mary", "smith")
    """
    parts = (full_name or "").strip().lower().split()
    # Strip leading honorifics
    while parts and re.sub(r"[^a-z]", "", parts[0]) in _HONORIFICS:
        parts = parts[1:]
    first = parts[0] if parts else ""
    last  = parts[-1] if len(parts) > 1 else ""
    return first, last


# ── Infer observed email pattern ──────────────────────────────────────────

def _infer_pattern(email: str, first: str, last: str) -> str:
    """
    Given a real email and the person's name, infer the pattern string.
    Used to pass to verify_candidates so the guessed address uses the
    format we already know this company uses.

    Returns a pattern key ("first.last", "flast", "first", etc.) or "".
    """
    if not email or "@" not in email:
        return ""
    local = email.split("@")[0].lower()
    if not first or not last:
        return ""
    fi = first[0]
    li = last[0]

    if local == f"{first}.{last}":  return "first.last"
    if local == f"{fi}{last}":      return "flast"
    if local == f"{fi}.{last}":     return "f.last"
    if local == first:              return "first"
    if local == f"{first}_{last}":  return "first_last"
    if local == f"{first}{li}":     return "firstl"
    if local == last:               return "last"
    if local == f"{last}.{first}":  return "last.first"
    return ""


# ── Main per-lead processor ────────────────────────────────────────────────

def _process_lead(lead: Lead, serper_key: str = "") -> dict:
    """
    Three-strategy email pipeline. Never raises.

    Strategy 1 — GitHub API (Tier 1, free)
    Strategy 2 — Scrape confirmed domain for real email (Tier 1)
    Strategy 3 — Pattern guess on confirmed domain only (Tier 2)
    Tier 3 (never): guessing on unconfirmed domains.
    """
    if lead.email_verified and lead.email:
        _set_cm(lead, "email")
        return _row(lead, "already verified", lead.email_confidence or 0.9, "")

    first, last = _split_name(lead.full_name or "")

    # ── Strategy 1: GitHub ─────────────────────────────────────────────────
    try:
        github_url = lead.github_url
    except Exception:
        github_url = None
    if github_url:
        email = _github_email(github_url)
        if email:
            lead.email            = email
            lead.email_confidence = 0.95
            lead.email_verified   = True
            _set_cm(lead, "email")
            return _row(lead, "✓ github", 0.95, "")

    # ── Resolve a candidate domain ─────────────────────────────────────────
    domain        = ""
    domain_source = ""

    if lead.website:
        domain        = domain_from_url(lead.website)
        domain_source = "website"

    if not domain and lead.company and serper_key:
        domain        = _find_domain_for_company(lead.company, serper_key)
        domain_source = "company_lookup"

    if not domain and serper_key and lead.full_name:
        found_co, found_domain = _find_domain_for_person(
            lead.full_name, lead.role or "", serper_key
        )
        if found_domain:
            domain        = found_domain
            domain_source = "person_lookup"
            if found_co and not lead.company:
                lead.company = found_co
            if not lead.website:
                lead.website = f"https://{found_domain}"

    if not domain and lead.company:
        candidates = _guess_domain(lead.company)
        domain     = _resolve_domain(candidates)
        domain_source = "slug_guess"

    if not domain:
        if lead.linkedin_url:
            _set_cm(lead, "linkedin_dm")
            return _no_email(lead, "→ LinkedIn DM (no domain found)")
        return _no_email(lead, "no contact")

    # ── Confirm domain belongs to this person ──────────────────────────────
    # Only skip confirmation when domain came from lead.website (explicit).
    if domain_source != "website":
        confirmed = _confirm_domain(domain, first, last)
    else:
        confirmed = True

    # ── Tier 1: Scrape confirmed domain for real email ─────────────────────
    scraped_pattern = ""
    if confirmed:
        html   = _get_domain_pages(domain)
        emails = _extract_emails(html, domain)
        if emails:
            # Record observed pattern to inform any fallback guess
            scraped_pattern = _infer_pattern(emails[0], first, last)
            lead.email            = emails[0]
            lead.email_confidence = 0.95
            lead.email_verified   = True
            _set_cm(lead, "email")
            return _row(lead, "✓ scraped", 0.95, domain)

    # ── Tier 2: Pattern guess on CONFIRMED domain only ─────────────────────
    if confirmed and lead.full_name:
        candidates = guess_emails(lead.full_name, domain)
        if candidates:
            if domain not in _domain_mx_cache:
                # Pass the observed pattern so the best format is prioritised
                _domain_mx_cache[domain] = verify_candidates(
                    candidates, known_pattern=scraped_pattern
                )
            mx = _domain_mx_cache[domain]
            if mx and mx.has_mx:
                # Catch-all domains: cap confidence — we can't verify mailboxes
                conf = min(mx.confidence, 0.5) if mx.is_catchall else mx.confidence
                lead.email            = mx.email
                lead.email_confidence = conf
                lead.email_verified   = conf >= VERIFY_THRESHOLD
                _set_cm(lead, "email")
                status = "✓ verified" if lead.email_verified else (
                    "guessed (catch-all)" if mx.is_catchall else "guessed"
                )
                return _row(lead, status, conf, domain, mx.mx_host)

    # ── Tier 3 (skipped) ───────────────────────────────────────────────────
    if lead.linkedin_url:
        _set_cm(lead, "linkedin_dm")
        return _no_email(lead, "→ LinkedIn DM (domain unconfirmed)")
    return _no_email(lead, "no contact")


def _set_cm(lead: Lead, method: str) -> None:
    try:
        lead.contact_method = method
    except Exception:
        pass


def _row(lead: Lead, status: str, conf: float,
         domain: str, mx_host: str = "") -> dict:
    return {
        "id": lead.id, "name": lead.full_name, "email": lead.email,
        "confidence": conf, "status": status,
        "mx_host": mx_host or domain, "contact_method": "email",
    }


def _no_email(lead: Lead, status: str) -> dict:
    return {
        "id": lead.id, "name": lead.full_name,
        "email": None, "confidence": 0.0,
        "status": status, "mx_host": "",
        "contact_method": getattr(lead, "contact_method", None),
    }


# ── Batch runner ───────────────────────────────────────────────────────────

@dataclass
class EmailEngineResult:
    total_processed:   int = 0
    emails_guessed:    int = 0
    emails_verified:   int = 0
    emails_skipped:    int = 0
    linkedin_dm_count: int = 0
    errors: list[str]       = field(default_factory=list)
    lead_emails: list[dict] = field(default_factory=list)


def run_email_engine(
    lead_ids:  list[int] | None = None,
    all_leads: bool = False,
    progress_callback=None,
) -> EmailEngineResult:
    result = EmailEngineResult()
    _domain_mx_cache.clear()
    _website_cache.clear()
    _domain_confirmed.clear()
    _serper_cache.clear()

    try:
        from scrylo.core.config import load as load_cfg
        _cfg = load_cfg()
        serper_key = (_cfg.get("scraper", {}).get("serper_api_key") or "").strip()
    except Exception:
        serper_key = ""

    with db_session() as db:
        if all_leads:
            leads = db.query(Lead).filter(
                Lead.email_verified == False,  # noqa: E712
            ).filter(
                (Lead.website != None) | (Lead.company != None)  # noqa: E711
                | (Lead.linkedin_url != None)  # noqa: E711
            ).all()
        elif lead_ids:
            leads = db.query(Lead).filter(Lead.id.in_(lead_ids)).all()
        else:
            leads = []

        result.total_processed = len(leads)

        for i, lead in enumerate(leads):
            if progress_callback:
                progress_callback(i, len(leads),
                                  f"Processing {lead.full_name or 'lead'}...")
            try:
                row = _process_lead(lead, serper_key)
                result.lead_emails.append(row)
                if row.get("email"):
                    result.emails_guessed += 1
                    if row.get("confidence", 0) >= VERIFY_THRESHOLD:
                        result.emails_verified += 1
                elif row.get("contact_method") == "linkedin_dm":
                    result.linkedin_dm_count += 1
                else:
                    result.emails_skipped += 1
            except Exception as e:
                err = f"Lead {lead.id} ({lead.full_name}): {e}"
                log.warning(err)
                result.errors.append(err)

        if progress_callback:
            progress_callback(len(leads), len(leads), "Done")

    return result
