# Familiar - Self-Hosted AI Companion Platform
# Copyright (c) 2026 George Scott Foley
# ORCID: 0009-0006-4957-0540
# Email: Georgescottfoley@proton.me
# Licensed under the MIT License - see LICENSE file for details

"""
Inbox Monitor — real-time background email watcher.

Polls Gmail or IMAP on a configurable interval, classifies new emails
through the Themis triage engine, and returns priority hits for delivery
to the user's configured notification channel.

State persisted at: ~/.familiar/data/email_monitor.json
  {
    "enabled": false,
    "interval_minutes": 10,
    "last_check_time": null,           # ISO8601 UTC
    "alert_on": ["urgent"],            # priority levels that trigger alerts
    "watch_senders": [],               # always alert on these email addresses or domains
    "watch_topics": [],                # always alert on these keywords in subject/snippet
    "quiet_hours_start": "22:00",
    "quiet_hours_end": "07:00",
    "task_id": null                    # scheduler task ID (set by watch_inbox tool)
  }
"""

import json
import logging
from datetime import datetime, timedelta, timezone
from pathlib import Path

logger = logging.getLogger(__name__)

_DEFAULT_CONFIG: dict = {
    "enabled": False,
    "interval_minutes": 10,
    "last_check_time": None,
    "alert_on": ["urgent"],
    "watch_senders": [],
    "watch_topics": [],
    "quiet_hours_start": "22:00",
    "quiet_hours_end": "07:00",
    "task_id": None,
    # Smart Mode (LLM tier for ambiguous emails)
    "smart_mode_enabled": False,
    "smart_batch_size": 3,
    "smart_daily_budget_tokens": 1000,
    "smart_tokens_used_today": 0,
    "smart_tokens_reset_date": None,
}


# ── Config helpers ─────────────────────────────────────────────────────────────


def _data_dir() -> Path:
    try:
        from familiar.core.paths import DATA_DIR

        return DATA_DIR
    except ImportError:
        return Path.home() / ".familiar" / "data"


def _monitor_file() -> Path:
    return _data_dir() / "email_monitor.json"


def get_monitor_config() -> dict:
    """Load monitor config from disk, filling in defaults for missing keys."""
    path = _monitor_file()
    if path.exists():
        try:
            data = json.loads(path.read_text())
            return {**_DEFAULT_CONFIG, **data}
        except (json.JSONDecodeError, OSError) as e:
            logger.warning("Failed to load monitor config: %s", e)
    return dict(_DEFAULT_CONFIG)


def save_monitor_config(config: dict):
    """Persist monitor config at chmod 0o600."""
    path = _monitor_file()
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(json.dumps(config, indent=2) + "\n")
    path.chmod(0o600)


# ── Quiet-hours check ──────────────────────────────────────────────────────────


def _in_quiet_hours(config: dict) -> bool:
    """Return True if the current local time falls inside the quiet window."""
    now = datetime.now()
    start_str = config.get("quiet_hours_start", "22:00")
    end_str = config.get("quiet_hours_end", "07:00")
    try:
        qs = datetime.strptime(start_str, "%H:%M").replace(
            year=now.year, month=now.month, day=now.day
        )
        qe = datetime.strptime(end_str, "%H:%M").replace(
            year=now.year, month=now.month, day=now.day
        )
        # Overnight window (e.g. 22:00 – 07:00)
        if qs > qe:
            return now >= qs or now <= qe
        return qs <= now <= qe
    except ValueError:
        return False


# ── Email classification ───────────────────────────────────────────────────────


def _classify_email(
    from_header: str,
    subject: str,
    snippet: str,
    has_list_unsubscribe: bool = False,
    header_signals: dict = None,
) -> tuple:
    """
    Classify an email using the unified six-signal scorer.
    Returns (category: str, priority: str).
    Falls back to ("Other", "fyi") if scorer is unavailable.
    """
    import re

    _email_match = re.search(r"<([^>]+)>", from_header)
    sender_email = (_email_match.group(1) if _email_match else from_header).lower().strip()
    sender_name = from_header.split("<")[0].strip().strip('"') if "<" in from_header else ""

    try:
        from familiar.skills.triage.scorer import score_email

        result = score_email(
            sender_email=sender_email,
            sender_name=sender_name,
            subject=subject,
            snippet=snippet,
            has_list_unsubscribe=has_list_unsubscribe,
            header_signals=header_signals,
        )
        return result["category"], result["priority"]
    except Exception as e:
        logger.debug("Unified scorer unavailable, falling back: %s", e)
        # Fallback to direct triage call
        try:
            from familiar.skills.triage.skill import _categorize_email, _get_priority

            category, _ = _categorize_email(sender_email, sender_name, subject, snippet, has_list_unsubscribe)
            priority = _get_priority(subject, category, sender_email)
            return category, priority
        except Exception:
            return "Other", "fyi"


def _should_alert(email_data: dict, config: dict) -> bool:
    """Return True if this email warrants a push alert."""
    priority = email_data.get("priority", "fyi")
    alert_on = set(config.get("alert_on", ["urgent"]))

    if priority in alert_on:
        return True

    # Watch-sender match (exact address or domain suffix)
    sender = email_data.get("from_email", "").lower()
    for ws in config.get("watch_senders", []):
        ws_lower = ws.lower().lstrip("@")
        if sender == ws_lower or sender.endswith("@" + ws_lower):
            return True

    # Watch-topic keyword in subject or snippet
    subject_lower = email_data.get("subject", "").lower()
    snippet_lower = email_data.get("snippet", "").lower()
    for wt in config.get("watch_topics", []):
        wt_lower = wt.lower()
        if wt_lower in subject_lower or wt_lower in snippet_lower:
            return True

    return False


# ── Gmail fetch ────────────────────────────────────────────────────────────────


def _fetch_gmail_since(service, since_dt: datetime) -> list:
    """Fetch unread Gmail messages received after since_dt (max 50)."""
    since_epoch = int(since_dt.timestamp())
    query = f"is:unread after:{since_epoch}"

    try:
        result = service.users().messages().list(userId="me", q=query, maxResults=50).execute()
    except Exception as e:
        logger.warning("Gmail list failed: %s", e)
        return []

    message_ids = [m["id"] for m in result.get("messages", [])]
    emails = []

    for msg_id in message_ids:
        try:
            msg = (
                service.users()
                .messages()
                .get(
                    userId="me",
                    id=msg_id,
                    format="metadata",
                    metadataHeaders=[
                        "From", "Subject", "Date", "List-Unsubscribe",
                        "Precedence", "X-Mailer", "X-Campaign-Id", "Return-Path",
                    ],
                )
                .execute()
            )
            headers = {
                h["name"].lower(): h["value"] for h in msg.get("payload", {}).get("headers", [])
            }
            emails.append(
                {
                    "id": msg_id,
                    "from": headers.get("from", ""),
                    "subject": headers.get("subject", "(no subject)"),
                    "snippet": msg.get("snippet", "")[:200],
                    "has_list_unsubscribe": "list-unsubscribe" in headers,
                    "precedence": headers.get("precedence", ""),
                    "x_mailer": headers.get("x-mailer", ""),
                    "x_campaign_id": headers.get("x-campaign-id", ""),
                    "return_path": headers.get("return-path", ""),
                    "source": "gmail",
                }
            )
        except Exception as e:
            logger.debug("Failed to fetch Gmail message %s: %s", msg_id, e)

    return emails


# ── IMAP fetch ─────────────────────────────────────────────────────────────────


def _fetch_imap_since(imap_cfg: dict, since_dt: datetime) -> list:
    """Fetch unseen IMAP messages received on or after since_dt (max 50)."""
    try:
        from familiar.skills.triage.skill import _decode_header, _imap_connect
    except ImportError as e:
        logger.debug("IMAP helpers not available: %s", e)
        return []

    since_str = since_dt.strftime("%d-%b-%Y")
    emails = []
    mail = None

    try:
        import email as _email_mod

        mail = _imap_connect(imap_cfg)
        mail.login(imap_cfg["address"], imap_cfg["password"])
        mail.select("INBOX", readonly=True)

        _, data = mail.search(None, f'(UNSEEN SINCE "{since_str}")')
        if not data or not data[0]:
            return []

        # Most recent 50 to avoid overwhelming the scheduler
        msg_ids = data[0].split()[-50:]

        for msg_id in msg_ids:
            try:
                _, msg_data = mail.fetch(
                    msg_id,
                    "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT LIST-UNSUBSCRIBE)])",
                )
                raw = msg_data[0][1] if msg_data and msg_data[0] else b""
                msg = _email_mod.message_from_bytes(raw)
                emails.append(
                    {
                        "id": msg_id.decode(),
                        "from": _decode_header(msg.get("From", "")),
                        "subject": _decode_header(msg.get("Subject", "(no subject)")),
                        "snippet": "",
                        "has_list_unsubscribe": bool(msg.get("List-Unsubscribe")),
                        "source": "imap",
                    }
                )
            except Exception as e:
                logger.debug("Failed to fetch IMAP message %s: %s", msg_id, e)

    except Exception as e:
        logger.warning("IMAP monitor fetch failed: %s", e)
    finally:
        if mail:
            try:
                mail.logout()
            except Exception:
                pass

    return emails


# ── Core check ─────────────────────────────────────────────────────────────────


def check_new_priority_emails() -> list:
    """
    Fetch emails since last check, classify them, and return those that
    match the configured alert criteria.

    Always updates last_check_time on return so the next poll starts fresh.
    Returns an empty list during quiet hours or when no priority hits exist.
    """
    config = get_monitor_config()

    if _in_quiet_hours(config):
        logger.debug("Inbox monitor: quiet hours — skipping")
        return []

    # Determine the look-back window
    last_check_str = config.get("last_check_time")
    if last_check_str:
        try:
            since_dt = datetime.fromisoformat(last_check_str.replace("Z", ""))
        except ValueError:
            since_dt = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(
                minutes=config.get("interval_minutes", 10) + 5
            )
    else:
        since_dt = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(
            minutes=config.get("interval_minutes", 10) + 5
        )

    # Fetch from ALL connected accounts (multi-account)
    raw_emails: list = []

    try:
        from familiar.skills.email.skill import get_all_email_services

        for account_email, provider, service_or_cfg in get_all_email_services():
            try:
                if provider == "gmail" and not isinstance(service_or_cfg, dict):
                    # Gmail API service object
                    acct_emails = _fetch_gmail_since(service_or_cfg, since_dt)
                else:
                    # IMAP config dict
                    acct_emails = _fetch_imap_since(service_or_cfg, since_dt)

                # Tag each email with account and provider
                for em in acct_emails:
                    em["account"] = account_email
                    em["provider"] = provider

                raw_emails.extend(acct_emails)
                logger.debug(
                    "Fetched %d emails from %s (%s)", len(acct_emails), account_email, provider
                )
            except Exception as e:
                logger.warning("Monitor fetch failed for %s: %s", account_email, e)
    except Exception as e:
        logger.debug("Multi-account fetch unavailable, trying legacy path: %s", e)
        # Legacy fallback — single account
        try:
            from familiar.skills.triage.skill import _get_gmail_service, _gmail_available

            if _gmail_available():
                svc = _get_gmail_service()
                raw_emails = _fetch_gmail_since(svc, since_dt)
        except Exception as e2:
            logger.debug("Gmail monitor path failed: %s", e2)

        if not raw_emails:
            try:
                from familiar.skills.email.accounts import get_account

                acct = get_account()
                if acct and acct.get("address") and acct.get("password"):
                    imap_cfg = {
                        "address": acct["address"],
                        "password": acct["password"],
                        "imap_server": acct.get("imap_server", "imap.gmail.com"),
                        "imap_port": acct.get("imap_port", 993),
                    }
                    raw_emails = _fetch_imap_since(imap_cfg, since_dt)
            except Exception as e2:
                logger.debug("IMAP monitor path failed: %s", e2)

    # Always advance the cursor regardless of fetch outcome
    config["last_check_time"] = datetime.now(timezone.utc).replace(tzinfo=None).isoformat()
    save_monitor_config(config)

    if not raw_emails:
        return []

    # Classify and filter for alert-worthy emails
    try:
        from familiar.skills.triage.skill import (
            _extract_email_address,
            _extract_sender_name,
        )
    except ImportError:

        def _extract_email_address(h):
            import re

            m = re.search(r"<([^>]+)>", h)
            return (m.group(1) if m else h).lower().strip()

        def _extract_sender_name(h):
            return h.split("<")[0].strip().strip('"') or h.split("@")[0]

    # Lazy imports for calendar detection and nonprofit signals
    try:
        from familiar.skills.triage.skill import (
            _detect_calendar_event,
            _detect_nonprofit_signals,
        )

        _has_triage_detection = True
    except ImportError:
        _has_triage_detection = False

    hits = []
    cal_emails = []
    np_action_emails = []

    for em in raw_emails:
        from_header = em.get("from", "")
        subject = em.get("subject", "")
        snippet = em.get("snippet", "")
        _hsigs = {
            "precedence": em.get("precedence", ""),
            "x_mailer": em.get("x_mailer", ""),
            "x_campaign_id": em.get("x_campaign_id", ""),
            "return_path": em.get("return_path", ""),
        }

        category, priority = _classify_email(
            from_header, subject, snippet, em.get("has_list_unsubscribe", False), _hsigs
        )

        email_data = {
            "id": em["id"],
            "from_email": _extract_email_address(from_header),
            "from_name": _extract_sender_name(from_header),
            "subject": subject,
            "snippet": snippet[:120],
            "category": category,
            "priority": priority,
            "source": em.get("source", "unknown"),
            "account": em.get("account", ""),
            "provider": em.get("provider", ""),
        }

        # Record engagement: received
        try:
            from familiar.skills.email.engagement import record_received
            record_received(email_data["from_email"])
        except Exception:
            pass

        # Calendar event detection on every email
        if _has_triage_detection:
            cal_info = _detect_calendar_event(subject, snippet)
            if cal_info:
                email_data["has_calendar_event"] = True
                email_data["calendar_hint"] = cal_info
                cal_emails.append(email_data)

            np_signals = _detect_nonprofit_signals(subject, snippet, category)
            if np_signals:
                email_data["nonprofit_signals"] = np_signals
                np_action_emails.append(email_data)

        if _should_alert(email_data, config):
            hits.append(email_data)

    # Cache ALL classified emails (not just hits) for dashboard access
    # Merge with existing cache so previous sorts aren't lost
    existing_cache = get_sorted_inbox()
    existing_ids = {e["id"] for e in existing_cache.get("emails", [])}
    all_classified = list(existing_cache.get("emails", []))

    for em in raw_emails:
        # Skip emails already in cache
        if em["id"] in existing_ids:
            continue
        from_header = em.get("from", "")
        subject = em.get("subject", "")
        snippet = em.get("snippet", "")
        _hsigs2 = {
            "precedence": em.get("precedence", ""),
            "x_mailer": em.get("x_mailer", ""),
            "x_campaign_id": em.get("x_campaign_id", ""),
            "return_path": em.get("return_path", ""),
        }
        category, priority = _classify_email(
            from_header, subject, snippet, em.get("has_list_unsubscribe", False), _hsigs2
        )
        # Get confidence + engagement from scorer
        _confidence = 0.0
        _engagement_score = 0.0
        _from_email = _extract_email_address(from_header)
        try:
            from familiar.skills.triage.scorer import score_email

            _sn2 = from_header.split("<")[0].strip().strip('"') if "<" in from_header else ""
            _full = score_email(_from_email, _sn2, subject, snippet, em.get("has_list_unsubscribe", False), _hsigs2)
            _confidence = _full.get("confidence", 0)
        except Exception:
            pass
        try:
            from familiar.skills.email.engagement import get_score

            _engagement_score = get_score(_from_email)
        except Exception:
            pass
        all_classified.append(
            {
                "id": em["id"],
                "from_email": _from_email,
                "from_name": _extract_sender_name(from_header),
                "subject": subject,
                "snippet": snippet[:120],
                "category": category,
                "priority": priority,
                "confidence": _confidence,
                "engagement_score": _engagement_score,
                "id_source": em.get("source", "gmail"),
                "account": em.get("account", ""),
                "provider": em.get("provider", ""),
            }
        )
    # Smart Mode: run LLM on ambiguous emails (category=Other)
    if config.get("smart_mode_enabled"):
        all_classified = _run_smart_mode(all_classified, config)

    _save_sorted_cache(all_classified)

    # Auto-harvest high-confidence classifications as training data
    try:
        from familiar.skills.email.training_data import harvest_high_confidence
        harvest_high_confidence(all_classified)
    except Exception as e:
        logger.debug("Training data harvest skipped: %s", e)

    # Queue detected calendar events as pending (non-blocking)
    _queue_pending_events(cal_emails)
    # Queue nonprofit action items (non-blocking)
    _queue_email_actions(np_action_emails)

    return hits


# ── Pending event & action queue pipelines ────────────────────────────────────


def _queue_pending_events(cal_emails: list):
    """Queue calendar events detected in emails for user review.

    Best-practice pattern: never auto-create calendar events silently.
    Batch them for confirmation in the next heartbeat/briefing.
    """
    if not cal_emails:
        return

    try:
        from familiar.skills.email.pending_events import get_pending_event_store
        from familiar.skills.triage.skill import _extract_event_details

        store = get_pending_event_store()
        for em in cal_emails:
            cal_hint = em.get("calendar_hint")
            details = _extract_event_details(
                em.get("subject", ""),
                em.get("snippet", ""),
                cal_hint,
            )
            if not details:
                continue

            store.add_event(
                email_id=em["id"],
                email_subject=em.get("subject", ""),
                email_from=f"{em.get('from_name', '')} <{em.get('from_email', '')}>",
                title=details["title"],
                start=details["start"],
                duration=details.get("duration", 60),
                location=details.get("location", ""),
                has_rsvp=details.get("has_rsvp", False),
                source_category=em.get("category", ""),
            )
        logger.info("Queued %d pending calendar events from inbox monitor", len(cal_emails))
    except Exception as e:
        logger.debug("Failed to queue pending events: %s", e)


def _queue_email_actions(np_emails: list):
    """Queue nonprofit-specific email actions for follow-up.

    Maps nonprofit signals to action types for the proactive system.
    """
    if not np_emails:
        return

    try:
        from familiar.skills.email.action_queue import get_action_store

        store = get_action_store()
        for em in np_emails:
            signals = em.get("nonprofit_signals", {})
            subject = em.get("subject", "")
            from_str = f"{em.get('from_name', '')} <{em.get('from_email', '')}>"

            if signals.get("donor_gift"):
                amount = signals.get("gift_amount", "")
                store.add_action(
                    email_id=em["id"],
                    email_subject=subject,
                    email_from=from_str,
                    action_type="thank_you",
                    suggested_response=f"Send thank-you for {amount} gift"
                    if amount
                    else "Send donor acknowledgment",
                    category=em.get("category", ""),
                    priority="action",
                )

            if signals.get("grant_deadline"):
                store.add_action(
                    email_id=em["id"],
                    email_subject=subject,
                    email_from=from_str,
                    action_type="grant_report",
                    suggested_response=f"Grant report deadline: {signals['grant_deadline']}",
                    deadline=signals["grant_deadline"],
                    category=em.get("category", ""),
                    priority="urgent",
                )

            if signals.get("board_reference"):
                store.add_action(
                    email_id=em["id"],
                    email_subject=subject,
                    email_from=from_str,
                    action_type="board_prep",
                    suggested_response="Board meeting referenced — review for prep",
                    category=em.get("category", ""),
                    priority="action",
                )

            if signals.get("legislative_action"):
                store.add_action(
                    email_id=em["id"],
                    email_subject=subject,
                    email_from=from_str,
                    action_type="legislative_action",
                    suggested_response="Advocacy action item — review and respond",
                    category=em.get("category", ""),
                    priority="urgent",
                )

            # RSVP detection (from calendar hint, not nonprofit signals)
            cal_hint = em.get("calendar_hint")
            if cal_hint and cal_hint.get("has_rsvp"):
                store.add_action(
                    email_id=em["id"],
                    email_subject=subject,
                    email_from=from_str,
                    action_type="rsvp",
                    suggested_response="RSVP requested — confirm attendance",
                    category=em.get("category", ""),
                    priority="action",
                )

        logger.info("Queued actions from %d nonprofit-signal emails", len(np_emails))
    except Exception as e:
        logger.debug("Failed to queue email actions: %s", e)


# ── Alert formatting ───────────────────────────────────────────────────────────


# ── Classified email cache ─────────────────────────────────────────────────────

# ── Batch Sort ("Sort My Inbox") ──────────────────────────────────────────────

_sort_inbox_state = {
    "running": False,
    "total": 0,
    "sorted": 0,
    "urgent": 0,
    "action": 0,
    "fyi": 0,
    "cancelled": False,
}


def get_sort_inbox_status() -> dict:
    return dict(_sort_inbox_state)


def cancel_sort_inbox():
    _sort_inbox_state["cancelled"] = True


def sort_inbox_batch():
    """Sort all unread emails across ALL connected accounts using Tier 1 (rule-based).

    Updates _sort_inbox_state with progress. Can be cancelled via cancel_sort_inbox().
    Skips emails already in the sorted cache (by message ID).
    Iterates Gmail API accounts and IMAP accounts in sequence.
    """
    global _sort_inbox_state
    _sort_inbox_state = {
        "running": True, "total": 0, "sorted": 0,
        "urgent": 0, "action": 0, "fyi": 0, "cancelled": False,
    }

    try:
        # Load existing cache to skip already-sorted
        existing_cache = get_sorted_inbox()
        existing_ids = {em["id"] for em in existing_cache.get("emails", [])}
        all_classified = list(existing_cache.get("emails", []))

        _MAX_SORT_BATCH = 500  # Hard cap to prevent runaway on huge inboxes

        try:
            from familiar.skills.triage.skill import (
                _extract_email_address,
                _extract_sender_name,
            )
        except ImportError:
            import re as _re

            def _extract_email_address(h):
                m = _re.search(r"<([^>]+)>", h)
                return (m.group(1) if m else h).lower().strip()

            def _extract_sender_name(h):
                return h.split("<")[0].strip().strip('"') or h.split("@")[0]

        # Get all connected email services
        try:
            from familiar.skills.email.skill import get_all_email_services
            services = get_all_email_services()
        except Exception:
            services = []

        if not services:
            _sort_inbox_state["running"] = False
            return

        for account_email, provider, service_or_cfg in services:
            if _sort_inbox_state["cancelled"]:
                break
            if _sort_inbox_state["sorted"] >= _MAX_SORT_BATCH:
                break

            if provider == "gmail" and not isinstance(service_or_cfg, dict):
                # Gmail API path
                _sort_gmail_account(
                    service_or_cfg, account_email, provider,
                    existing_ids, all_classified, _extract_email_address,
                    _extract_sender_name,
                )
            else:
                # IMAP path
                _sort_imap_account(
                    service_or_cfg, account_email, provider,
                    existing_ids, all_classified, _extract_email_address,
                    _extract_sender_name,
                )

        logger.info(
            "Sort inbox complete: %d sorted (%d urgent, %d action, %d fyi) across %d accounts",
            _sort_inbox_state["sorted"], _sort_inbox_state["urgent"],
            _sort_inbox_state["action"], _sort_inbox_state["fyi"], len(services),
        )

    except Exception as e:
        logger.error("Sort inbox failed: %s", e)
    finally:
        _sort_inbox_state["running"] = False


def _sort_gmail_account(
    service, account_email, provider, existing_ids, all_classified,
    _extract_email_address, _extract_sender_name,
):
    """Sort unread emails from a single Gmail API account."""
    import time

    _MAX_SORT_BATCH = 500

    # Fetch all recent emails (last 30 days), not just unread
    _SORT_QUERY = "newer_than:30d"

    # Count total for this account
    try:
        count_result = service.users().messages().list(
            userId="me", q=_SORT_QUERY, maxResults=1
        ).execute()
        _sort_inbox_state["total"] += count_result.get("resultSizeEstimate", 0)
    except Exception:
        pass

    page_token = None

    while not _sort_inbox_state["cancelled"] and _sort_inbox_state["sorted"] < _MAX_SORT_BATCH:
        try:
            result = service.users().messages().list(
                userId="me", q=_SORT_QUERY, maxResults=50, pageToken=page_token,
            ).execute()
        except Exception as e:
            logger.warning("Sort inbox Gmail list failed for %s: %s", account_email, e)
            break

        messages = result.get("messages", [])
        if not messages:
            break

        if result.get("nextPageToken"):
            _sort_inbox_state["total"] = max(
                _sort_inbox_state["total"],
                _sort_inbox_state["sorted"] + len(messages) + 50,
            )

        for msg_meta in messages:
            if _sort_inbox_state["cancelled"]:
                break

            msg_id = msg_meta["id"]
            if msg_id in existing_ids:
                _sort_inbox_state["sorted"] += 1
                continue

            try:
                msg = service.users().messages().get(
                    userId="me", id=msg_id, format="metadata",
                    metadataHeaders=[
                        "From", "Subject", "List-Unsubscribe",
                        "Precedence", "X-Mailer", "X-Campaign-Id", "Return-Path",
                    ],
                ).execute()
                headers = {
                    h["name"].lower(): h["value"]
                    for h in msg.get("payload", {}).get("headers", [])
                }
                from_header = headers.get("from", "")
                subject = headers.get("subject", "(no subject)")
                snippet = msg.get("snippet", "")[:200]
                has_unsub = "list-unsubscribe" in headers
                header_signals = {
                    "precedence": headers.get("precedence", ""),
                    "x_mailer": headers.get("x-mailer", ""),
                    "x_campaign_id": headers.get("x-campaign-id", ""),
                    "return_path": headers.get("return-path", ""),
                }

                category, priority = _classify_email(
                    from_header, subject, snippet, has_unsub, header_signals
                )

                classified = {
                    "id": msg_id,
                    "from_email": _extract_email_address(from_header),
                    "from_name": _extract_sender_name(from_header),
                    "subject": subject,
                    "snippet": snippet[:120],
                    "category": category,
                    "priority": priority,
                    "id_source": "gmail",
                    "account": account_email,
                    "provider": provider,
                }
                all_classified.append(classified)
                existing_ids.add(msg_id)

                try:
                    from familiar.skills.email.engagement import record_received
                    record_received(classified["from_email"])
                except Exception:
                    pass

                if priority == "urgent":
                    _sort_inbox_state["urgent"] += 1
                elif priority == "action":
                    _sort_inbox_state["action"] += 1
                else:
                    _sort_inbox_state["fyi"] += 1

            except Exception as e:
                logger.debug("Sort inbox: failed to classify %s: %s", msg_id, e)

            _sort_inbox_state["sorted"] += 1
            time.sleep(0.1)

        _save_sorted_cache(all_classified)

        page_token = result.get("nextPageToken")
        if not page_token:
            break


def _sort_imap_account(
    imap_cfg, account_email, provider, existing_ids, all_classified,
    _extract_email_address, _extract_sender_name,
):
    """Sort unread emails from a single IMAP account."""
    import time

    _MAX_SORT_BATCH = 500
    mail = None

    try:
        from familiar.skills.triage.skill import _decode_header
    except ImportError:
        def _decode_header(h):
            return h

    try:
        import email as _email_mod
        from familiar.skills.email.skill import _imap_connect

        mail = _imap_connect(imap_cfg)
        mail.login(imap_cfg["address"], imap_cfg["password"])
        mail.select("INBOX", readonly=True)

        # Fetch all recent emails (last 30 days), not just unseen
        from datetime import datetime as _dt, timedelta as _td
        _since_str = (_dt.now() - _td(days=30)).strftime("%d-%b-%Y")
        _, data = mail.search(None, f'(SINCE "{_since_str}")')
        if not data or not data[0]:
            return

        msg_ids = data[0].split()
        _sort_inbox_state["total"] += len(msg_ids)

        # Cap to prevent runaway
        msg_ids = msg_ids[-_MAX_SORT_BATCH:]

        for msg_id in msg_ids:
            if _sort_inbox_state["cancelled"]:
                break
            if _sort_inbox_state["sorted"] >= _MAX_SORT_BATCH:
                break

            # Use account-scoped ID to avoid collisions between accounts
            scoped_id = f"{account_email}:{msg_id.decode()}"
            if scoped_id in existing_ids:
                _sort_inbox_state["sorted"] += 1
                continue

            try:
                _, msg_data = mail.fetch(
                    msg_id,
                    "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT LIST-UNSUBSCRIBE "
                    "PRECEDENCE X-MAILER X-CAMPAIGN-ID RETURN-PATH)])",
                )
                raw = msg_data[0][1] if msg_data and msg_data[0] else b""
                msg = _email_mod.message_from_bytes(raw)

                from_header = _decode_header(msg.get("From", ""))
                subject = _decode_header(msg.get("Subject", "(no subject)"))
                has_unsub = bool(msg.get("List-Unsubscribe"))
                header_signals = {
                    "precedence": msg.get("Precedence", ""),
                    "x_mailer": msg.get("X-Mailer", ""),
                    "x_campaign_id": msg.get("X-Campaign-Id", ""),
                    "return_path": msg.get("Return-Path", ""),
                }

                category, priority = _classify_email(
                    from_header, subject, "", has_unsub, header_signals
                )

                classified = {
                    "id": scoped_id,
                    "from_email": _extract_email_address(from_header),
                    "from_name": _extract_sender_name(from_header),
                    "subject": subject,
                    "snippet": "",
                    "category": category,
                    "priority": priority,
                    "id_source": "imap",
                    "account": account_email,
                    "provider": provider,
                }
                all_classified.append(classified)
                existing_ids.add(scoped_id)

                try:
                    from familiar.skills.email.engagement import record_received
                    record_received(classified["from_email"])
                except Exception:
                    pass

                if priority == "urgent":
                    _sort_inbox_state["urgent"] += 1
                elif priority == "action":
                    _sort_inbox_state["action"] += 1
                else:
                    _sort_inbox_state["fyi"] += 1

            except Exception as e:
                logger.debug("Sort inbox IMAP: failed to classify %s: %s", msg_id, e)

            _sort_inbox_state["sorted"] += 1
            time.sleep(0.1)

        _save_sorted_cache(all_classified)

    except Exception as e:
        logger.warning("Sort inbox IMAP failed for %s: %s", account_email, e)
    finally:
        if mail:
            try:
                mail.logout()
            except Exception:
                pass


# ── Smart Mode (LLM Tier 2) ───────────────────────────────────────────────────

_SMART_SYSTEM = (
    "You classify emails for a personal AI assistant. For each email, determine the "
    "category and priority based on sender, subject, preview, and engagement history.\n\n"
    "Categories:\n"
    "- Work: professional correspondence, colleagues, clients, invoices\n"
    "- Personal: friends, family, personal matters\n"
    "- Notifications: automated alerts, CI/CD, payment confirmations, account notices\n"
    "- Promotional: marketing, sales, newsletters, deals, political fundraising\n"
    "- Campaign: political campaigns, advocacy action alerts\n"
    "- Other: doesn't fit above categories\n\n"
    "Priorities:\n"
    "- urgent: needs response today (deadlines, failures, time-sensitive)\n"
    "- action: needs response this week (follow-ups, requests)\n"
    "- fyi: informational only (receipts, newsletters, updates)\n\n"
    "If engagement history shows user never opens emails from a sender, lean toward Promotional.\n"
    "If user replies frequently, lean toward Work or Personal.\n\n"
    "Return ONLY valid JSON: [{\"id\":1,\"cat\":\"Work\",\"pri\":\"action\"},...]"
)


def _check_smart_budget(config: dict) -> bool:
    """Return True if smart mode has budget remaining today."""
    from datetime import date

    today = date.today().isoformat()
    reset_date = config.get("smart_tokens_reset_date")
    if reset_date != today:
        config["smart_tokens_used_today"] = 0
        config["smart_tokens_reset_date"] = today
        save_monitor_config(config)

    budget = config.get("smart_daily_budget_tokens", 1000)
    used = config.get("smart_tokens_used_today", 0)
    return used < budget


def _run_smart_mode(all_classified: list, config: dict) -> list:
    """Run LLM classification on ambiguous emails (category=Other).

    Batches ambiguous emails, sends a single compact prompt to the
    lightweight LLM, parses results, and learns them back into Tier 1
    sender_categories for future instant classification.

    Returns the updated all_classified list with improved categories.
    """
    if not _check_smart_budget(config):
        logger.debug("Smart Mode: daily token budget exhausted, skipping")
        return all_classified

    # Find ambiguous emails
    ambiguous = [(i, em) for i, em in enumerate(all_classified) if em.get("category") == "Other"]
    if not ambiguous:
        return all_classified

    batch_size = config.get("smart_batch_size", 3)
    batch = ambiguous[:batch_size]

    # Build contextual prompt with signal data
    lines = []
    for idx, (_, em) in enumerate(batch, 1):
        from_str = em.get("from_email", "") or em.get("from_name", "")
        subj = em.get("subject", "")[:80]
        snippet = em.get("snippet", "")[:60]

        # Add engagement context if available
        eng_ctx = ""
        try:
            from familiar.skills.email.engagement import get_engagement

            eng = get_engagement(em.get("from_email", ""))
            if eng and eng.get("total_received", 0) > 1:
                eng_ctx = (
                    f" | History: {eng.get('total_received',0)} received, "
                    f"{eng.get('opened',0)} opened, {eng.get('replied',0)} replied"
                )
        except Exception:
            pass

        line = f"{idx}. From: {from_str} | Subj: {subj}"
        if snippet:
            line += f" | Preview: {snippet}"
        if eng_ctx:
            line += eng_ctx
        lines.append(line)

    user_prompt = "Classify these emails:\n\n" + "\n".join(lines)

    try:
        import urllib.request

        # Use lightweight Ollama model — no agent overhead
        _ollama_url = "http://localhost:11434"
        _model = "qwen2.5:0.5b"
        try:
            from familiar.core.config import load_config

            cfg = load_config()
            _ollama_url = cfg.llm.ollama_base_url or _ollama_url
            # Prefer lightweight model for classification
            from familiar.core.providers import LIGHTWEIGHT_MODELS

            if hasattr(cfg.llm, "ollama_model"):
                _model = cfg.llm.ollama_model
        except Exception:
            pass

        payload = json.dumps({
            "model": _model,
            "messages": [
                {"role": "system", "content": _SMART_SYSTEM},
                {"role": "user", "content": user_prompt},
            ],
            "stream": False,
            "options": {"num_predict": 256},
        }).encode()

        req = urllib.request.Request(
            f"{_ollama_url}/api/chat",
            data=payload,
            headers={"Content-Type": "application/json"},
            method="POST",
        )
        with urllib.request.urlopen(req, timeout=30) as resp:
            body = json.loads(resp.read())

        response_text = body.get("message", {}).get("content", "")
        tokens_used = body.get("eval_count", len(response_text) // 4)

        # Track token usage
        config["smart_tokens_used_today"] = config.get("smart_tokens_used_today", 0) + tokens_used
        save_monitor_config(config)

        # Parse LLM response — extract JSON array
        import re

        json_match = re.search(r"\[.*\]", response_text, re.DOTALL)
        if not json_match:
            logger.debug("Smart Mode: no JSON array in response")
            return all_classified

        results = json.loads(json_match.group())

        # Apply classifications
        learned_count = 0
        for r in results:
            idx = r.get("id", 0) - 1  # 1-indexed in prompt
            if idx < 0 or idx >= len(batch):
                continue
            orig_idx, em = batch[idx]
            new_cat = r.get("cat", "")
            new_pri = r.get("pri", "")

            if new_cat and new_cat != "Other":
                all_classified[orig_idx]["category"] = new_cat
                all_classified[orig_idx]["smart_classified"] = True
            if new_pri:
                all_classified[orig_idx]["priority"] = new_pri

            # Learn back into Tier 1
            if new_cat and new_cat != "Other" and em.get("from_email"):
                try:
                    from familiar.skills.triage.skill import (
                        _load_triage_data,
                        _save_triage_data,
                    )

                    triage = _load_triage_data()
                    sender = em["from_email"].lower()
                    triage.setdefault("sender_categories", {})[sender] = new_cat
                    if new_pri:
                        triage.setdefault("sender_priorities", {})[sender] = new_pri

                    # Domain learning (skip freemail)
                    domain = sender.split("@")[-1] if "@" in sender else ""
                    freemail = {
                        "gmail.com", "outlook.com", "yahoo.com", "hotmail.com",
                        "aol.com", "icloud.com", "protonmail.com", "proton.me",
                    }
                    if domain and domain not in freemail:
                        triage.setdefault("domain_categories", {})[domain] = new_cat

                    _save_triage_data(triage)
                    learned_count += 1
                except Exception as learn_err:
                    logger.debug("Smart Mode learn-back failed: %s", learn_err)

            # Memory Tree + training corpus
            if new_cat and new_cat != "Other" and em.get("from_email"):
                try:
                    from familiar.core.memory_tree import get_memory_tree

                    tree = get_memory_tree()
                    title = f"Email preference: {em['from_email']}"
                    body_text = (
                        f"AI classified emails from {em['from_email']} as "
                        f"category={new_cat}, priority={new_pri or 'fyi'}. "
                        f"Learned from Smart Mode classification."
                    )
                    existing = tree.search(f"email preference {em['from_email']}", limit=1)
                    if existing and existing[0].title == title:
                        tree.update(existing[0].memory_id, body=body_text)
                    else:
                        tree.add(
                            title=title, body=body_text,
                            tier="working", source="smart_mode",
                            importance=0.6,
                            meta={"training_eligible": "true",
                                  "domain": "email_preference",
                                  "pair_type": "smart_classification"},
                        )
                except Exception:
                    pass

        logger.info(
            "Smart Mode: classified %d/%d ambiguous emails, learned %d senders, "
            "%d tokens used (%d/%d budget)",
            len(results), len(batch), learned_count, tokens_used,
            config.get("smart_tokens_used_today", 0),
            config.get("smart_daily_budget_tokens", 1000),
        )

    except Exception as e:
        logger.warning("Smart Mode LLM call failed: %s", e)

    return all_classified


# ── Classified email cache ─────────────────────────────────────────────────────

_SORTED_CACHE_FILE = "email_sorted_cache.json"


def _sorted_cache_path() -> Path:
    return _data_dir() / _SORTED_CACHE_FILE


def get_sorted_inbox() -> dict:
    """Return the cached pre-sorted inbox, or empty if no cache exists.

    Returns:
        {
            "emails": [{id, from_name, from_email, subject, snippet, category,
                        priority, source, has_calendar_event?, nonprofit_signals?}],
            "summary": {"total": N, "urgent": N, "action": N, "fyi": N},
            "updated_at": "ISO8601",
        }
    """
    path = _sorted_cache_path()
    if path.exists():
        try:
            return json.loads(path.read_text())
        except (json.JSONDecodeError, OSError):
            pass
    return {"emails": [], "summary": {}, "updated_at": None}


def _save_sorted_cache(classified_emails: list):
    """Persist classified emails to disk for instant dashboard access.

    Priority-aware cap: important categories (Family, Work, Personal, Financial)
    are kept first, then remaining slots filled with other emails. This prevents
    bulk promotional emails from pushing out emails the user cares about.
    """
    # Priority categories that should never be evicted by bulk mail
    _PRIORITY_CATS = {"Family", "Work", "Personal", "Financial", "Events", "Dev/CI"}

    priority_emails = [e for e in classified_emails if e.get("category") in _PRIORITY_CATS]
    other_emails = [e for e in classified_emails if e.get("category") not in _PRIORITY_CATS]

    # Keep all priority emails (up to 100), fill rest with others
    cap = 500
    kept = priority_emails[:100] + other_emails[: cap - min(len(priority_emails), 100)]

    urgent = sum(1 for e in classified_emails if e.get("priority") == "urgent")
    action = sum(1 for e in classified_emails if e.get("priority") == "action")
    fyi = len(classified_emails) - urgent - action

    cache = {
        "emails": kept,
        "summary": {
            "total": len(classified_emails),
            "urgent": urgent,
            "action": action,
            "fyi": fyi,
        },
        "updated_at": datetime.now(timezone.utc).isoformat(),
    }
    path = _sorted_cache_path()
    try:
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_text(json.dumps(cache, indent=2, default=str))
    except OSError as e:
        logger.warning("Failed to save sorted inbox cache: %s", e)


def format_alert_message(hits: list) -> str:
    """Format a concise push notification for a list of priority email hits."""
    if not hits:
        return ""

    urgent = [h for h in hits if h["priority"] == "urgent"]
    action = [h for h in hits if h["priority"] == "action"]
    watched = [h for h in hits if h["priority"] == "fyi"]

    count = len(hits)
    noun = "email" if count == 1 else "emails"
    lines = [f"📬 **{count} priority {noun}:**\n"]

    for em in urgent[:4]:
        subj = em["subject"][:52] + ("…" if len(em["subject"]) > 52 else "")
        lines.append(f"  🔴 {em['from_name']} — {subj}")

    for em in action[:3]:
        subj = em["subject"][:52] + ("…" if len(em["subject"]) > 52 else "")
        lines.append(f"  🟡 {em['from_name']} — {subj}")

    for em in watched[:2]:
        subj = em["subject"][:52] + ("…" if len(em["subject"]) > 52 else "")
        lines.append(f"  👁️  {em['from_name']} — {subj}")

    remaining = count - len(urgent[:4]) - len(action[:3]) - len(watched[:2])
    if remaining > 0:
        lines.append(f"  … and {remaining} more")

    lines.append("\nRun `triage_inbox` to see the full list.")
    return "\n".join(lines)
