# Familiar - Self-Hosted AI Companion Platform
# Copyright (c) 2026 George Scott Foley
# ORCID: 0009-0006-4957-0540
# Email: Georgescottfoley@proton.me
# Licensed under the MIT License - see LICENSE file for details

"""
Email Triage Skill - Themis, Titan of Order

Themis was the Titan of divine law, order, and custom. She weighs each
matter with her scales and assigns it to its proper place.

Sort incoming emails into organizational categories.
Learns sender categorizations over time.
Custom keywords can be added/removed per category.

Categories:
- Campaign
- Events / Trans Pride
- Operations (GJL)
- Staff
- Board
- Coalitions
- Donors
- Grants
- Media
- Volunteers
- Notifications (CI/CD, system alerts, automated)
- Promotional (marketing, newsletters, ads)
- Other
"""

import email
import imaplib
import json
import logging
import os
import re
from datetime import datetime, timedelta
from email.header import decode_header
from pathlib import Path

logger = logging.getLogger(__name__)


IMAP_TIMEOUT = 20  # seconds — prevent hung IMAP connections


def _imap_connect(cfg: dict, timeout: int = IMAP_TIMEOUT):
    """Create an IMAP connection using the right transport for the port.

    Port 993 → IMAP4_SSL (implicit TLS, standard providers).
    Port 1143 / 143 / other → IMAP4 + STARTTLS (Proton Bridge, plain IMAP).
    """
    server = cfg["imap_server"]
    port = cfg["imap_port"]
    if port == 993:
        return imaplib.IMAP4_SSL(server, port, timeout=timeout)
    mail = imaplib.IMAP4(server, port, timeout=timeout)
    mail.starttls()
    return mail


# ── Category configuration ────────────────────────────────────────────────────
CATEGORIES = [
    "Campaign",
    "Trans Pride",
    "GJL Operations",
    "Staff",
    "Board",
    "Coalitions",
    "Donors",
    "Grants",
    "Media",
    "Volunteers",
    "Notifications",
    "Promotional",
    "Other",
]

# Default keyword hints for auto-categorization (lowercase).
# Users can add/remove keywords via manage_keywords tool.
DEFAULT_KEYWORDS = {
    "Campaign": [
        "campaign",
        "action alert",
        "advocacy",
        "petition",
        "lobby",
        "legislative",
        "bill",
        "vote",
        "call to action",
        "mobilize",
        "rally",
        "protest",
        "testimony",
        "policy",
        "ordinance",
        "rights",
        "discrimination",
        "equality",
        "justice",
    ],
    "Trans Pride": [
        "trans pride",
        "pride event",
        "pride parade",
        "pride festival",
        "pride volunteer",
        "pride planning",
        "march",
        "celebration",
        "pride seattle",
        "pride committee",
    ],
    "GJL Operations": [
        "operations",
        "admin",
        "office",
        "facilities",
        "supplies",
        "invoice",
        "vendor",
        "contract",
        "lease",
        "utilities",
        "meeting room",
    ],
    "Staff": [
        "staff meeting",
        "team standup",
        "pto",
        "time off",
        "1:1",
        "one on one",
        "payroll",
        "benefits",
        "all staff",
        "staff retreat",
        "performance review",
    ],
    "Board": [
        "board meeting",
        "board member",
        "governance",
        "bylaws",
        "board agenda",
        "board minutes",
        "fiduciary",
        "treasurer",
        "board vote",
        "executive committee",
        "board packet",
    ],
    "Coalitions": [
        "coalition",
        "partner",
        "alliance",
        "collaborative",
        "solidarity",
        "collective",
        "lgbtq",
        "community partner",
        "co-sign",
        "sign-on",
    ],
    "Donors": [
        "donation",
        "gift",
        "contribute",
        "donor",
        "pledge",
        "matching gift",
        "thank you for your support",
        "tax receipt",
        "giving tuesday",
    ],
    "Grants": [
        "grant",
        "foundation",
        "funder",
        "proposal",
        "application",
        "report due",
        "grant report",
        "funding",
        "regranting",
    ],
    "Media": [
        "press",
        "reporter",
        "journalist",
        "interview request",
        "statement",
        "press release",
        "media inquiry",
        "on the record",
    ],
    "Volunteers": [
        "volunteer",
        "volunteering",
        "sign up",
        "shift",
        "help needed",
        "opportunity",
    ],
    "Notifications": [
        "run failed",
        "build failed",
        "ci failed",
        "test failed",
        "deployment failed",
        "action failed",
        "pipeline failed",
        "workflow run",
        "github action",
        "pull request",
        "merge conflict",
        "dependabot",
        "security alert",
        "security advisory",
        "server alert",
        "disk space",
        "cpu usage",
        "memory usage",
        "uptime",
        "monitoring",
        "cron job",
        "backup completed",
        "backup failed",
        "ssl certificate",
        "domain expir",
    ],
    "Promotional": [
        "unsubscribe",
        "opt out",
        "email preferences",
        "manage subscriptions",
        "view in browser",
        "limited time",
        "special offer",
        "discount",
        "% off",
        "sale ends",
        "shop now",
        "buy now",
        "free shipping",
        "order now",
        "deal of the day",
        "exclusive offer",
        "coupon",
        "promo code",
        "black friday",
        "cyber monday",
        "flash sale",
        "clearance",
        "earn rewards",
        "travel rewards",
        "credit score",
        "pre-approved",
        "apply now",
        "credit card",
        "no annual fee",
    ],
}

# Domains that are almost always promotional / marketing
PROMOTIONAL_DOMAINS = frozenset({
    # Email service providers (bulk senders)
    "marketing.com", "mailchimp.com", "sendgrid.net", "constantcontact.com",
    "hubspot.com", "mailgun.org", "amazonses.com", "sparkpost.com",
    "campaign-archive.com", "list-manage.com", "createsend.com",
    # Financial / credit
    "experian.com", "creditkarma.com", "nerdwallet.com", "transunion.com",
    # Deals / shopping
    "groupon.com", "retailmenot.com", "slickdeals.net",
    # Gaming / entertainment
    "humblebundle.com", "steampowered.com", "epicgames.com",
    "disneyplus.com", "disney.com", "hulu.com", "netflix.com",
    "marvel.com", "marvelentertainment.com",
    # Job boards
    "ziprecruiter.com", "indeed.com", "glassdoor.com",
    "ihirechefs.com", "culinaryagents.com",
    # Sports / lifestyle
    "mlssoccer.com", "nfl.com", "nba.com",
    # Food delivery / meal kits
    "shef.com", "doordash.com", "ubereats.com", "grubhub.com",
    "hellofresh.com", "blueapron.com",
    # Political fundraising platforms
    "actblue.com", "winred.com", "everyaction.com", "ngpvan.com",
    "actionnetwork.org", "mobilize.us", "nationbuilder.com",
    "politicalemails.com", "endlessurgency.com",
})

# Political fundraising heuristic — sender name patterns
_POLITICAL_FUNDRAISING_PATTERNS = [
    "for congress", "for senate", "for governor", "for texas",
    "for america", "for president", "for office",
    "campaign committee", "victory fund", "pac",
    "democrat", "republican", "progressive",
    "donate now", "chip in", "rush",
]

# Domains that are CI/CD or system notifications
NOTIFICATION_DOMAINS = frozenset({
    "github.com", "gitlab.com", "bitbucket.org",
    "circleci.com", "travis-ci.com", "app.travis-ci.com",
    "vercel.com", "netlify.com", "heroku.com",
    "sentry.io", "pagerduty.com", "opsgenie.com",
    "datadog.com", "newrelic.com",
    "aws.amazon.com", "cloud.google.com", "azure.com",
    "pypi.org",
})


# ── Urgency keywords ─────────────────────────────────────────────────────────
URGENT_KEYWORDS = [
    "urgent",
    "asap",
    "immediate",
    "deadline today",
    "action required",
    "time sensitive",
    "critical",
    "emergency",
    "outage",
    "incident",
    "p0",
    "p1",
    "sev1",
    "sev2",
]

ACTION_KEYWORDS = [
    "request",
    "please",
    "need",
    "can you",
    "review needed",
    "approval needed",
    "sign off",
    "rsvp",
    "confirm",
    "deadline",
    "due date",
    "expiring",
    "expires",
    "renewal",
    "report due",
    "grant report",
    "run failed",
    "build failed",
    "test failed",
    "deployment failed",
    "pipeline failed",
]


# ── Storage ───────────────────────────────────────────────────────────────────
TRIAGE_DATA_FILE = Path.home() / ".familiar" / "data" / "email_triage.json"


def _load_triage_data() -> dict:
    """Load learned sender mappings and custom keywords."""
    if TRIAGE_DATA_FILE.exists():
        try:
            with open(TRIAGE_DATA_FILE, encoding="utf-8") as f:
                return json.load(f)
        except (json.JSONDecodeError, IOError, OSError) as e:
            logger.warning(f"Failed to load triage data: {e}")
    return {
        "sender_categories": {},  # email -> category
        "domain_categories": {},  # domain -> category
        "custom_keywords": {},    # category -> [keywords] (user additions)
        "removed_keywords": {},   # category -> [keywords] (user removals from defaults)
        "custom_urgent_keywords": [],   # user-added urgent keywords
        "removed_urgent_keywords": [],  # user-removed default urgent keywords
        "custom_action_keywords": [],   # user-added action keywords
        "removed_action_keywords": [],  # user-removed default action keywords
        "sender_priorities": {},        # email/domain -> "urgent"|"action"|"fyi"
    }


def _save_triage_data(data: dict):
    """Save learned mappings and custom keywords."""
    TRIAGE_DATA_FILE.parent.mkdir(parents=True, exist_ok=True)
    with open(TRIAGE_DATA_FILE, "w") as f:
        json.dump(data, f, indent=2)


def _get_effective_keywords() -> dict[str, list[str]]:
    """Merge default keywords with user customizations.

    Returns the final keyword dict: defaults + custom additions - removals.
    """
    triage_data = _load_triage_data()
    custom = triage_data.get("custom_keywords", {})
    removed = triage_data.get("removed_keywords", {})

    result = {}
    for cat in CATEGORIES:
        base = list(DEFAULT_KEYWORDS.get(cat, []))
        # Add user custom keywords
        for kw in custom.get(cat, []):
            if kw not in base:
                base.append(kw)
        # Remove user-removed keywords
        removals = set(removed.get(cat, []))
        result[cat] = [kw for kw in base if kw not in removals]
    return result


def _get_effective_priority_keywords() -> tuple[list[str], list[str]]:
    """Return (urgent_keywords, action_keywords) after user customizations."""
    triage_data = _load_triage_data()

    # Urgent keywords
    custom_urgent = triage_data.get("custom_urgent_keywords", [])
    removed_urgent = set(triage_data.get("removed_urgent_keywords", []))
    urgent = list(URGENT_KEYWORDS)
    for kw in custom_urgent:
        if kw not in urgent:
            urgent.append(kw)
    urgent = [kw for kw in urgent if kw not in removed_urgent]

    # Action keywords
    custom_action = triage_data.get("custom_action_keywords", [])
    removed_action = set(triage_data.get("removed_action_keywords", []))
    action = list(ACTION_KEYWORDS)
    for kw in custom_action:
        if kw not in action:
            action.append(kw)
    action = [kw for kw in action if kw not in removed_action]

    return urgent, action


# ── Email helpers ─────────────────────────────────────────────────────────────


def _get_email_config(account: str | None = None):
    """Get email configuration, preferring the account registry."""
    try:
        from familiar.skills.email.accounts import get_account

        acct = get_account(account)
        if acct:
            return {
                "address": acct["address"],
                "password": acct["password"],
                "imap_server": acct["imap_server"],
                "imap_port": acct["imap_port"],
            }
    except ImportError as e:
        logger.debug("Optional dependency not available: %s", e)
    return {
        "address": os.environ.get("EMAIL_ADDRESS", ""),
        "password": os.environ.get("EMAIL_PASSWORD", ""),
        "imap_server": os.environ.get("EMAIL_IMAP_SERVER", "imap.gmail.com"),
        "imap_port": int(os.environ.get("EMAIL_IMAP_PORT", "993")),
    }


# ── Gmail API helpers (mirrors email/skill.py) ─────────────────────────────


def _gmail_token_file() -> Path:
    try:
        from familiar.core.paths import DATA_DIR

        return DATA_DIR / "google_token_gmail.json"
    except ImportError:
        return Path.home() / ".familiar" / "data" / "google_token_gmail.json"


def _gmail_available() -> bool:
    token = _gmail_token_file()
    if not token.exists():
        return False
    try:
        from googleapiclient.discovery import build  # noqa
        from google.oauth2.credentials import Credentials  # noqa

        return True
    except ImportError:
        return False


def _get_gmail_service():
    from google.auth.transport.requests import Request
    from google.oauth2.credentials import Credentials
    from googleapiclient.discovery import build

    token = _gmail_token_file()
    scopes = [
        "https://www.googleapis.com/auth/gmail.readonly",
        "https://www.googleapis.com/auth/gmail.modify",
    ]
    creds = Credentials.from_authorized_user_file(str(token), scopes)
    if not creds.valid:
        if creds.expired and creds.refresh_token:
            try:
                creds.refresh(Request())
            except Exception as e:
                logger.warning("Gmail token refresh failed: %s", e)
                token.unlink(missing_ok=True)
                raise RuntimeError(
                    "Gmail token expired or revoked. "
                    "Run /connect google auth gmail to re-authorize."
                ) from e
            token.write_text(creds.to_json())
        else:
            raise RuntimeError("Gmail token expired. Run /connect google auth gmail.")
    return build("gmail", "v1", credentials=creds)


def _gmail_header(payload: dict, name: str) -> str:
    for h in payload.get("headers", []):
        if h["name"].lower() == name.lower():
            return h["value"]
    return ""


def _decode_header(header):
    """Decode email header."""
    if header is None:
        return ""
    decoded = decode_header(header)
    result = []
    for part, encoding in decoded:
        if isinstance(part, bytes):
            result.append(part.decode(encoding or "utf-8", errors="replace"))
        else:
            result.append(part)
    return "".join(result)


def _extract_email_address(from_header: str) -> str:
    """Extract just the email address from From header."""
    match = re.search(r"<([^>]+)>", from_header)
    if match:
        return match.group(1).lower()
    if "@" in from_header:
        return from_header.strip().lower()
    return from_header.lower()


def _extract_sender_name(from_header: str) -> str:
    """Extract display name from From header."""
    if "<" in from_header:
        name = from_header.split("<")[0].strip().strip('"').strip("'")
        if name:
            return name
    return from_header.split("@")[0]


def _get_domain(email_addr: str) -> str:
    """Get domain from email address."""
    if "@" in email_addr:
        return email_addr.split("@")[1].lower()
    return ""


# ── Classification reason tracking ────────────────────────────────────────────
# Store last triage's classification reasons (keyed by email index)
_last_triage_reasons: dict[int, list[str]] = {}
_last_category_reasons: list[str] = []
_last_priority_reasons: list[str] = []


# ── Classification ────────────────────────────────────────────────────────────


def _is_promotional(
    domain: str, sender_name: str, content: str, has_list_unsubscribe: bool = False
) -> bool:
    """Heuristic: detect promotional/marketing emails."""
    # Known promotional domains (exact match or subdomain)
    if domain in PROMOTIONAL_DOMAINS:
        return True
    # Subdomain check: e.g. "email.marvel.com" → check "marvel.com"
    parts = domain.split(".")
    if len(parts) > 2:
        parent = ".".join(parts[-2:])
        if parent in PROMOTIONAL_DOMAINS:
            return True

    # Political fundraising: sender name contains campaign patterns
    sender_lower = sender_name.lower()
    if any(pat in sender_lower for pat in _POLITICAL_FUNDRAISING_PATTERNS):
        return True

    # Political fundraising: content patterns (catches cases where sender name is
    # just a person's name but subject/body reveals fundraising intent)
    _POLITICAL_CONTENT_PATTERNS = [
        "chip in", "rush a donation", "donate now", "donate today",
        "contribute now", "fundraising deadline", "fundraising goal",
        "match your donation", "double your impact", "triple match",
        "we need you to donate", "can you chip in",
        "running for", "re-election", "reelection",
        "stand with", "fight back", "fight for",
    ]
    if any(pat in content for pat in _POLITICAL_CONTENT_PATTERNS):
        return True

    # List-Unsubscribe header (RFC 2369) — strongest signal from email headers.
    # Required by Gmail / Yahoo for bulk senders since Feb 2024.
    if has_list_unsubscribe:
        # Check if this is an internal org email that happens to have the header
        org_keywords = _get_effective_keywords()
        org_hits = 0
        for cat in ["GJL Operations", "Staff", "Board"]:
            for kw in org_keywords.get(cat, []):
                if kw in content:
                    org_hits += 1
        if org_hits == 0:
            return True

    # "Unsubscribe" in body — required by CAN-SPAM
    if "unsubscribe" in content:
        # But exclude internal org emails that happen to have footers
        promo_signals = sum(1 for kw in [
            "shop now", "buy now", "limited time", "special offer",
            "% off", "free shipping", "deal", "sale", "coupon",
            "earn rewards", "credit", "pre-approved", "apply now",
            "view in browser", "email preferences", "opt out",
            "donate", "chip in", "rush", "contribute",
        ] if kw in content)
        if promo_signals >= 1:
            return True
        # If unsubscribe is present but no other promo signals,
        # still lean promotional if no org keywords match
        org_keywords = _get_effective_keywords()
        org_hits = 0
        for cat in ["Campaign", "GJL Operations", "Staff", "Board",
                     "Coalitions", "Donors", "Grants", "Media"]:
            for kw in org_keywords.get(cat, []):
                if kw in content:
                    org_hits += 1
        if org_hits == 0:
            return True
    return False


def _is_notification(domain: str, sender_email: str, content: str) -> bool:
    """Heuristic: detect CI/CD and system notification emails."""
    if domain in NOTIFICATION_DOMAINS:
        return True
    # noreply/no-reply senders with technical content
    if "noreply" in sender_email or "no-reply" in sender_email:
        tech_signals = sum(1 for kw in [
            "run failed", "build", "deploy", "ci", "test",
            "pipeline", "workflow", "action", "alert", "monitor",
        ] if kw in content)
        if tech_signals >= 1:
            return True
    return False


def _categorize_email(
    sender_email: str,
    sender_name: str,
    subject: str,
    body_preview: str = "",
    has_list_unsubscribe: bool = False,
) -> tuple[str, float]:
    """
    Categorize an email based on sender and content.
    Returns (category, confidence) where confidence is 0-1.

    Priority order:
    1. Learned sender mappings (highest confidence)
    2. Promotional/notification heuristics
    3. Keyword matching across all categories
    4. Fallback to "Other"

    Side effect: populates module-level _last_category_reasons for explain_triage.
    """
    global _last_category_reasons
    _last_category_reasons = []

    triage_data = _load_triage_data()

    sender_email = sender_email.lower()
    domain = _get_domain(sender_email)
    subject_lower = subject.lower()
    content = f"{subject_lower} {body_preview.lower()}"

    # 1. Check if we've seen this exact sender before (highest confidence)
    if sender_email in triage_data.get("sender_categories", {}):
        cat = triage_data["sender_categories"][sender_email]
        _last_category_reasons.append(f"Learned sender: {sender_email} → {cat} (95%)")
        return cat, 0.95

    # 2. Check domain mapping
    if domain in triage_data.get("domain_categories", {}):
        cat = triage_data["domain_categories"][domain]
        _last_category_reasons.append(f"Learned domain: *@{domain} → {cat} (85%)")
        return cat, 0.85

    # 3. Promotional heuristic (before keyword matching)
    if _is_promotional(domain, sender_name, content, has_list_unsubscribe):
        reasons = [f"Promotional heuristic: domain={domain}"]
        if has_list_unsubscribe:
            reasons.append("List-Unsubscribe header present")
        if "unsubscribe" in content:
            reasons.append("'unsubscribe' in body")
        _last_category_reasons.extend(reasons)
        return "Promotional", 0.80

    # 4. Notification heuristic
    if _is_notification(domain, sender_email, content):
        _last_category_reasons.append(
            f"Notification heuristic: domain={domain}, sender={sender_email}"
        )
        return "Notifications", 0.80

    # 5. Keyword matching
    keywords = _get_effective_keywords()
    best_category = "Other"
    best_score = 0
    best_matches: list[str] = []

    for category, kws in keywords.items():
        if category in ("Promotional", "Notifications"):
            continue  # Already handled by heuristics above
        score = 0
        matches: list[str] = []
        for keyword in kws:
            if keyword in content:
                score += 1
                matches.append(keyword)
            if keyword in sender_name.lower():
                score += 0.5
                matches.append(f"{keyword} (sender name)")

        if score > best_score:
            best_score = score
            best_category = category
            best_matches = matches

    confidence = min(0.7, best_score * 0.15) if best_score > 0 else 0.1

    if best_matches:
        kw_display = ", ".join(repr(m) for m in best_matches[:3])
        _last_category_reasons.append(f"Keyword match: {kw_display} → {best_category}")
    else:
        _last_category_reasons.append("No keywords matched → Other (fallback)")

    return best_category, confidence


def _get_priority(subject: str, sender_category: str, sender_email: str = "") -> str:
    """Determine email priority based on subject content, category, and sender rules.

    Side effect: populates module-level _last_priority_reasons for explain_triage.
    """
    global _last_priority_reasons
    _last_priority_reasons = []

    subject_lower = subject.lower()
    triage_data = _load_triage_data()

    # 1. Sender priority rules (highest precedence)
    sender_priorities = triage_data.get("sender_priorities", {})
    if sender_email:
        sender_email_lower = sender_email.lower()
        if sender_email_lower in sender_priorities:
            p = sender_priorities[sender_email_lower]
            _last_priority_reasons.append(f"Priority: sender rule {sender_email_lower} → {p}")
            return p
        domain = _get_domain(sender_email_lower)
        if domain and domain in sender_priorities:
            p = sender_priorities[domain]
            _last_priority_reasons.append(f"Priority: domain rule *@{domain} → {p}")
            return p

    # 2. Effective urgent keywords
    urgent_kws, action_kws = _get_effective_priority_keywords()

    matched_urgent = [kw for kw in urgent_kws if kw in subject_lower]
    if matched_urgent:
        _last_priority_reasons.append(f"Priority: keyword '{matched_urgent[0]}' → urgent")
        return "urgent"

    # 3. Category-based logic
    # Board emails often need attention
    if sender_category == "Board":
        _last_priority_reasons.append("Priority: Board category → action")
        return "action"

    # CI/CD failures need attention
    if sender_category == "Notifications":
        fail_signals = ["failed", "error", "broken", "down", "alert", "critical"]
        matched_fails = [s for s in fail_signals if s in subject_lower]
        if matched_fails:
            _last_priority_reasons.append(
                f"Priority: Notification + '{matched_fails[0]}' → action"
            )
            return "action"

    # Grants with deadlines
    if sender_category == "Grants":
        grant_signals = ["due", "deadline", "expir"]
        matched_grant = [w for w in grant_signals if w in subject_lower]
        if matched_grant:
            _last_priority_reasons.append(
                f"Priority: Grant + '{matched_grant[0]}' → action"
            )
            return "action"

    # 4. Action keywords (questions/requests)
    if "?" in subject:
        _last_priority_reasons.append("Priority: question mark in subject → action")
        return "action"

    matched_action = [w for w in action_kws if w in subject_lower]
    if matched_action:
        _last_priority_reasons.append(f"Priority: keyword '{matched_action[0]}' → action")
        return "action"

    # 5. Promotional is always low priority
    if sender_category == "Promotional":
        _last_priority_reasons.append("Priority: Promotional category → fyi")
        return "fyi"

    _last_priority_reasons.append("Priority: default → fyi")
    return "fyi"


# ── Gmail triage ──────────────────────────────────────────────────────────────


def _gmail_triage_inbox(data: dict) -> str:
    """Gmail API path for triage — uses same categorization logic as IMAP path."""
    import base64
    from datetime import datetime
    from datetime import timedelta as td

    limit = data.get("limit", 25)
    unread_only = data.get("unread_only", True)
    days_back = data.get("days_back", 3)

    svc = _get_gmail_service()
    q_parts = []
    if unread_only:
        q_parts.append("is:unread")
    if days_back:
        since = (datetime.now() - td(days=days_back)).strftime("%Y/%m/%d")
        q_parts.append(f"after:{since}")

    res = svc.users().messages().list(userId="me", q=" ".join(q_parts), maxResults=limit).execute()
    msgs = res.get("messages", [])
    if not msgs:
        return "📭 No emails to triage."

    categorized = {cat: [] for cat in CATEGORIES}
    global _last_triage_reasons
    _last_triage_reasons = {}
    _reasons_by_id: dict[str, list[str]] = {}

    for m in msgs:
        try:
            full = svc.users().messages().get(userId="me", id=m["id"], format="full").execute()
            payload = full.get("payload", {})
            subject = _gmail_header(payload, "Subject") or "(no subject)"
            from_header = _gmail_header(payload, "From")
            sender_email = _extract_email_address(from_header)
            sender_name = _extract_sender_name(from_header)

            def _extract_text(p, prefer_plain=True):
                """Extract text preview, with HTML fallback."""
                if prefer_plain and p.get("mimeType") == "text/plain":
                    raw = p.get("body", {}).get("data", "")
                    if raw:
                        return base64.urlsafe_b64decode(raw + "==").decode(
                            "utf-8", errors="replace"
                        )[:500]
                for part in p.get("parts", []):
                    r = _extract_text(part, prefer_plain=prefer_plain)
                    if r:
                        return r
                # HTML fallback: strip tags to get readable text for classification
                if p.get("mimeType") == "text/html":
                    raw = p.get("body", {}).get("data", "")
                    if raw:
                        html = base64.urlsafe_b64decode(raw + "==").decode(
                            "utf-8", errors="replace"
                        )
                        html = re.sub(r"<style[^>]*>.*?</style>", "", html,
                                      flags=re.DOTALL | re.IGNORECASE)
                        html = re.sub(r"<script[^>]*>.*?</script>", "", html,
                                      flags=re.DOTALL | re.IGNORECASE)
                        html = re.sub(r"<!--.*?-->", "", html, flags=re.DOTALL)
                        text = re.sub(r"<[^<]+?>", "", html)
                        text = re.sub(r"\s+", " ", text).strip()
                        return text[:500]
                return ""

            body_preview = _extract_text(payload)
            # Check List-Unsubscribe header (RFC 2369) — strong promo signal
            has_list_unsub = bool(_gmail_header(payload, "List-Unsubscribe"))
            category, confidence = _categorize_email(
                sender_email, sender_name, subject, body_preview,
                has_list_unsubscribe=has_list_unsub,
            )
            priority = _get_priority(subject, category, sender_email)
            _reasons_by_id[m["id"]] = (
                _last_category_reasons.copy() + _last_priority_reasons.copy()
            )

            categorized[category].append(
                {
                    "id": m["id"],
                    "thread_id": full.get("threadId", m["id"]),
                    "from": sender_name,
                    "from_email": sender_email,
                    "subject": subject,
                    "priority": priority,
                    "confidence": confidence,
                }
            )
        except Exception as e:
            logger.error(f"Gmail triage msg error: {e}")
            continue

    priority_emoji = {"urgent": "🔴", "action": "🟡", "fyi": "🟢"}
    total = sum(len(v) for v in categorized.values())
    lines = [f"📬 Inbox Triage — Gmail API ({total} emails)\n"]

    # Build a flat ordered list for session_context (enables "reply to #3")
    ordered_emails = []

    # Display order: org categories first, then Notifications, Promotional last
    display_order = [c for c in CATEGORIES if c not in ("Notifications", "Promotional", "Other")]
    display_order.extend(["Notifications", "Promotional", "Other"])

    for category in display_order:
        emails_sorted = sorted(
            categorized.get(category, []),
            key=lambda x: {"urgent": 0, "action": 1, "fyi": 2}.get(x["priority"], 2),
        )
        for em in emails_sorted:
            idx = len(ordered_emails) + 1
            ordered_emails.append(
                {
                    "index": idx,
                    "id": em["id"],
                    "thread_id": em.get("thread_id", em["id"]),
                    "id_source": "gmail",
                    "from_name": em["from"],
                    "from_email": em["from_email"],
                    "subject": em["subject"],
                    "category": category,
                    "priority": em["priority"],
                }
            )
            _last_triage_reasons[idx] = _reasons_by_id.get(em["id"], [])

    # Build confidence lookup by id for display
    _conf_by_id = {e["id"]: e.get("confidence", 1.0) for cat in categorized.values() for e in cat}

    for category in display_order:
        cat_emails = [e for e in ordered_emails if e["category"] == category]
        if not cat_emails:
            continue
        lines.append(f"\n━━━ {category.upper()} ({len(cat_emails)}) ━━━")
        for em in cat_emails:
            p_emoji = priority_emoji.get(em["priority"], "⚪")
            conf = "?" if _conf_by_id.get(em["id"], 1.0) < 0.5 else ""
            subj = em["subject"][:50] + "..." if len(em["subject"]) > 50 else em["subject"]
            lines.append(f"  #{em['index']} {p_emoji}{conf} {em['from_name']}")
            lines.append(f"      {subj}")

    urgent_count = sum(1 for e in ordered_emails if e["priority"] == "urgent")
    action_count = sum(1 for e in ordered_emails if e["priority"] == "action")
    promo_count = sum(1 for e in ordered_emails if e["category"] == "Promotional")

    summary_parts = []
    if urgent_count:
        summary_parts.append(f"{urgent_count} urgent")
    if action_count:
        summary_parts.append(f"{action_count} need action")
    if promo_count:
        summary_parts.append(f"{promo_count} promotional (hidden below)")
    if summary_parts:
        lines.append(f"\n⚡ {', '.join(summary_parts)}")

    result_text = "\n".join(lines)

    # Phase 4: Write ordered_emails to session_context directly when available,
    # and also to thread-local as fallback for agent.py pickup.
    import datetime as _dt4t
    import threading as _threading

    _triage_payload = {"emails": ordered_emails, "total": total}
    _threading.current_thread()._familiar_triage_ctx = _triage_payload

    # Direct write path: context dict injected by tools.execute
    _ctx = data.get("_context") or {}
    _session = _ctx.get("session")
    _sessions_mgr = _ctx.get("session_manager")
    if _session is not None:
        _session.session_context["triage_emails"] = ordered_emails
        _session.session_context["triage_at"] = _dt4t.datetime.now(_dt4t.timezone.utc).isoformat()
        if _sessions_mgr is not None:
            _sessions_mgr.save_session(_session)

    return result_text


# ── IMAP triage ───────────────────────────────────────────────────────────────


def triage_inbox(data: dict) -> str:
    """
    Fetch and categorize emails, sorted by category.
    Uses Gmail API if connected, IMAP/SMTP otherwise.
    """
    account = data.get("account")
    if _gmail_available() and (account is None or account.lower() == "gmail"):
        try:
            return _gmail_triage_inbox(data)
        except Exception as e:
            logger.warning(f"Gmail triage failed, falling back to IMAP: {e}")

    global _last_triage_reasons
    _last_triage_reasons = {}

    config = _get_email_config(account)

    if not config["address"] or not config["password"]:
        return (
            "❌ Email not configured. Set EMAIL_ADDRESS and EMAIL_PASSWORD environment variables."
        )

    limit = data.get("limit", 25)
    unread_only = data.get("unread_only", True)
    days_back = data.get("days_back", 3)

    try:
        mail = _imap_connect(config, timeout=IMAP_TIMEOUT)
        mail.login(config["address"], config["password"])
        mail.select("INBOX")

        # Search criteria
        criteria = []
        if unread_only:
            criteria.append("UNSEEN")
        if days_back:
            since = (datetime.now() - timedelta(days=days_back)).strftime("%d-%b-%Y")
            criteria.append(f'SINCE "{since}"')

        search_str = " ".join(criteria) if criteria else "ALL"
        status, messages = mail.search(None, search_str)

        if status != "OK" or not messages[0]:
            return "📭 No emails to triage."

        email_ids = messages[0].split()[-limit:]

        # Fetch and categorize
        categorized = {cat: [] for cat in CATEGORIES}
        _reasons_by_eid: dict[str, list[str]] = {}

        for eid in email_ids:
            try:
                status, msg_data = mail.fetch(eid, "(RFC822)")
                if status != "OK":
                    continue

                raw = msg_data[0][1]
                msg = email.message_from_bytes(raw)

                subject = _decode_header(msg["Subject"]) or "(no subject)"
                from_header = _decode_header(msg["From"]) or ""
                sender_email = _extract_email_address(from_header)
                sender_name = _extract_sender_name(from_header)

                # Get body preview for better categorization
                body_preview = ""
                html_fallback = ""
                if msg.is_multipart():
                    for part in msg.walk():
                        ctype = part.get_content_type()
                        if ctype == "text/plain" and not body_preview:
                            try:
                                body_preview = part.get_payload(decode=True).decode(
                                    "utf-8", errors="replace"
                                )[:500]
                            except (UnicodeDecodeError, AttributeError, TypeError) as e:
                                logger.debug("Unicode decode error suppressed: %s", e)
                        elif ctype == "text/html" and not html_fallback:
                            try:
                                raw_html = part.get_payload(decode=True).decode(
                                    "utf-8", errors="replace"
                                )
                                raw_html = re.sub(
                                    r"<style[^>]*>.*?</style>", "", raw_html,
                                    flags=re.DOTALL | re.IGNORECASE
                                )
                                raw_html = re.sub(
                                    r"<script[^>]*>.*?</script>", "", raw_html,
                                    flags=re.DOTALL | re.IGNORECASE
                                )
                                html_fallback = re.sub(r"<[^<]+?>", "", raw_html)
                                html_fallback = re.sub(r"\s+", " ", html_fallback).strip()[:500]
                            except (UnicodeDecodeError, AttributeError, TypeError) as e:
                                logger.debug("HTML decode error suppressed: %s", e)
                else:
                    try:
                        body_preview = msg.get_payload(decode=True).decode(
                            "utf-8", errors="replace"
                        )[:500]
                    except (UnicodeDecodeError, AttributeError, TypeError) as e:
                        logger.debug("Unicode decode error suppressed: %s", e)
                if not body_preview:
                    body_preview = html_fallback
                # Check List-Unsubscribe header (RFC 2369)
                has_list_unsub = bool(msg.get("List-Unsubscribe"))
                category, confidence = _categorize_email(
                    sender_email, sender_name, subject, body_preview,
                    has_list_unsubscribe=has_list_unsub,
                )
                priority = _get_priority(subject, category, sender_email)
                eid_str = eid.decode() if isinstance(eid, bytes) else str(eid)
                _reasons_by_eid[eid_str] = (
                    _last_category_reasons.copy() + _last_priority_reasons.copy()
                )

                categorized[category].append(
                    {
                        "id": eid_str,
                        "from": sender_name,
                        "from_email": sender_email,
                        "subject": subject,
                        "priority": priority,
                        "confidence": confidence,
                    }
                )
            except Exception as e:
                logger.error(f"Error processing email: {e}")
                continue

        mail.logout()

        # Format output
        priority_emoji = {"urgent": "🔴", "action": "🟡", "fyi": "🟢"}

        lines = [f"📬 Inbox Triage ({sum(len(v) for v in categorized.values())} emails)\n"]

        display_order = [c for c in CATEGORIES if c not in ("Notifications", "Promotional", "Other")]
        display_order.extend(["Notifications", "Promotional", "Other"])

        ordered_emails_imap = []
        for category in display_order:
            for em in sorted(
                categorized.get(category, []),
                key=lambda x: {"urgent": 0, "action": 1, "fyi": 2}.get(x["priority"], 2),
            ):
                idx = len(ordered_emails_imap) + 1
                ordered_emails_imap.append(
                    {
                        "index": idx,
                        "id": em["id"],
                        "id_source": "imap",
                        "from_name": em["from"],
                        "from_email": em["from_email"],
                        "subject": em["subject"],
                        "category": category,
                        "priority": em["priority"],
                    }
                )
                _last_triage_reasons[idx] = _reasons_by_eid.get(em["id"], [])

        _conf_imap = {}
        for cat_name, cat_emails_list in categorized.items():
            for ce in cat_emails_list:
                _conf_imap[ce["id"]] = ce.get("confidence", 1.0)

        for category in display_order:
            cat_emails = [e for e in ordered_emails_imap if e["category"] == category]
            if not cat_emails:
                continue

            lines.append(f"\n━━━ {category.upper()} ({len(cat_emails)}) ━━━")

            for em in cat_emails:
                p_emoji = priority_emoji.get(em["priority"], "⚪")
                conf = "?" if _conf_imap.get(em["id"], 1.0) < 0.5 else ""
                subj = em["subject"][:50] + "..." if len(em["subject"]) > 50 else em["subject"]
                lines.append(f"  #{em['index']} {p_emoji}{conf} {em['from_name']}")
                lines.append(f"      {subj}")

        # Summary
        urgent_count = sum(1 for e in ordered_emails_imap if e["priority"] == "urgent")
        action_count = sum(1 for e in ordered_emails_imap if e["priority"] == "action")
        promo_count = sum(1 for e in ordered_emails_imap if e["category"] == "Promotional")

        summary_parts = []
        if urgent_count:
            summary_parts.append(f"{urgent_count} urgent")
        if action_count:
            summary_parts.append(f"{action_count} need action")
        if promo_count:
            summary_parts.append(f"{promo_count} promotional")
        if summary_parts:
            lines.append(f"\n⚡ {', '.join(summary_parts)}")

        # Phase 4: dual write — direct + thread-local fallback
        import datetime as _dt4ti
        import threading as _threading

        _triage_imap_payload = {
            "emails": ordered_emails_imap,
            "total": sum(len(v) for v in categorized.values()),
        }
        _threading.current_thread()._familiar_triage_ctx = _triage_imap_payload

        _ictx = data.get("_context") or {}
        _isess = _ictx.get("session")
        _isess_mgr = _ictx.get("session_manager")
        if _isess is not None:
            _isess.session_context["triage_emails"] = ordered_emails_imap
            _isess.session_context["triage_at"] = _dt4ti.datetime.now(
                _dt4ti.timezone.utc
            ).isoformat()
            if _isess_mgr is not None:
                _isess_mgr.save_session(_isess)

        return "\n".join(lines)

    except Exception as e:
        logger.error(f"Triage failed: {e}")
        return "❌ Email operation failed. Check server connection."


# ── Sender teaching ───────────────────────────────────────────────────────────


def teach_sender(data: dict) -> str:
    """
    Teach the agent which category a sender belongs to.
    This helps improve future categorization.
    """
    sender = data.get("sender", "").lower().strip()
    category = data.get("category", "").strip()

    if not sender:
        return "Please provide the sender email address."

    # Normalize category
    category_match = None
    for cat in CATEGORIES:
        if cat.lower() == category.lower():
            category_match = cat
            break

    if not category_match:
        return f"Unknown category: {category}\nValid categories: {', '.join(CATEGORIES)}"

    triage_data = _load_triage_data()

    if "@" in sender:
        triage_data.setdefault("sender_categories", {})[sender] = category_match
        _save_triage_data(triage_data)
        return f"✅ Learned: {sender} → {category_match}"
    else:
        domain = sender.replace("@", "").strip()
        triage_data.setdefault("domain_categories", {})[domain] = category_match
        _save_triage_data(triage_data)
        return f"✅ Learned: *@{domain} → {category_match}"


def list_learned_senders(data: dict) -> str:
    """Show all learned sender categorizations."""
    triage_data = _load_triage_data()

    senders = triage_data.get("sender_categories", {})
    domains = triage_data.get("domain_categories", {})

    if not senders and not domains:
        return "No learned categorizations yet. Use teach_sender to train me."

    lines = ["📚 Learned Categorizations:\n"]

    by_category = {cat: [] for cat in CATEGORIES}

    for email_addr, cat in senders.items():
        by_category.setdefault(cat, []).append(email_addr)

    for domain, cat in domains.items():
        by_category.setdefault(cat, []).append(f"*@{domain}")

    for cat in CATEGORIES:
        items = by_category.get(cat, [])
        if items:
            lines.append(f"\n{cat}:")
            for item in sorted(items):
                lines.append(f"  • {item}")

    return "\n".join(lines)


def forget_sender(data: dict) -> str:
    """Remove a learned sender categorization."""
    sender = data.get("sender", "").lower().strip()

    if not sender:
        return "Please provide the sender to forget."

    triage_data = _load_triage_data()
    removed = False

    if sender in triage_data.get("sender_categories", {}):
        del triage_data["sender_categories"][sender]
        removed = True

    domain = sender.replace("@", "").strip()
    if domain in triage_data.get("domain_categories", {}):
        del triage_data["domain_categories"][domain]
        removed = True

    if removed:
        _save_triage_data(triage_data)
        return f"✅ Forgot categorization for {sender}"
    else:
        return f"No learned categorization found for {sender}"


# ── Keyword management ────────────────────────────────────────────────────────


def manage_keywords(data: dict) -> str:
    """Add, remove, or list custom keywords for triage categories.

    Actions:
      - add: Add a keyword to a category
      - remove: Remove a keyword from a category (hides default keywords too)
      - list: Show all keywords (default + custom) for a category or all categories
      - reset: Reset a category to default keywords (removes all customizations)
    """
    action = data.get("action", "list").lower().strip()
    category = data.get("category", "").strip()
    keyword = data.get("keyword", "").lower().strip()

    triage_data = _load_triage_data()

    if action == "list":
        keywords = _get_effective_keywords()
        if category:
            # Match category loosely
            matched = None
            for cat in CATEGORIES:
                if cat.lower() == category.lower() or category.lower() in cat.lower():
                    matched = cat
                    break
            if not matched:
                return f"Unknown category: {category}\nValid: {', '.join(CATEGORIES)}"

            kws = keywords.get(matched, [])
            custom = triage_data.get("custom_keywords", {}).get(matched, [])
            removed = triage_data.get("removed_keywords", {}).get(matched, [])

            lines = [f"🏷️ Keywords for {matched} ({len(kws)} active)\n"]
            for kw in sorted(kws):
                marker = " ⭐" if kw in custom else ""
                lines.append(f"  • {kw}{marker}")
            if removed:
                lines.append(f"\n  Removed ({len(removed)}):")
                for kw in sorted(removed):
                    lines.append(f"  ✗ {kw}")
            lines.append("\n⭐ = custom keyword")
            return "\n".join(lines)
        else:
            # Show all categories with keyword counts
            lines = ["🏷️ Triage Keywords\n"]
            for cat in CATEGORIES:
                kws = keywords.get(cat, [])
                custom_count = len(triage_data.get("custom_keywords", {}).get(cat, []))
                custom_note = f" (+{custom_count} custom)" if custom_count else ""
                lines.append(f"  {cat}: {len(kws)} keywords{custom_note}")
            lines.append("\nUse manage_keywords with a category name for details.")
            return "\n".join(lines)

    if action == "add":
        if not category or not keyword:
            return "Please provide both category and keyword."
        matched = None
        for cat in CATEGORIES:
            if cat.lower() == category.lower() or category.lower() in cat.lower():
                matched = cat
                break
        if not matched:
            return f"Unknown category: {category}\nValid: {', '.join(CATEGORIES)}"

        custom = triage_data.setdefault("custom_keywords", {}).setdefault(matched, [])
        if keyword in custom:
            return f"Keyword '{keyword}' already exists in {matched}."
        custom.append(keyword)

        # Also remove from removed_keywords if it was previously removed
        removed = triage_data.get("removed_keywords", {}).get(matched, [])
        if keyword in removed:
            removed.remove(keyword)

        _save_triage_data(triage_data)
        return f"✅ Added keyword '{keyword}' to {matched}"

    if action == "remove":
        if not category or not keyword:
            return "Please provide both category and keyword."
        matched = None
        for cat in CATEGORIES:
            if cat.lower() == category.lower() or category.lower() in cat.lower():
                matched = cat
                break
        if not matched:
            return f"Unknown category: {category}\nValid: {', '.join(CATEGORIES)}"

        # Check if it's a custom keyword — remove directly
        custom = triage_data.get("custom_keywords", {}).get(matched, [])
        if keyword in custom:
            custom.remove(keyword)
            _save_triage_data(triage_data)
            return f"✅ Removed custom keyword '{keyword}' from {matched}"

        # Check if it's a default keyword — add to removed list
        defaults = DEFAULT_KEYWORDS.get(matched, [])
        if keyword in defaults:
            removed = triage_data.setdefault("removed_keywords", {}).setdefault(matched, [])
            if keyword not in removed:
                removed.append(keyword)
            _save_triage_data(triage_data)
            return f"✅ Hidden default keyword '{keyword}' from {matched}"

        return f"Keyword '{keyword}' not found in {matched}."

    if action == "reset":
        if not category:
            return "Please provide a category to reset."
        matched = None
        for cat in CATEGORIES:
            if cat.lower() == category.lower() or category.lower() in cat.lower():
                matched = cat
                break
        if not matched:
            return f"Unknown category: {category}\nValid: {', '.join(CATEGORIES)}"

        if matched in triage_data.get("custom_keywords", {}):
            del triage_data["custom_keywords"][matched]
        if matched in triage_data.get("removed_keywords", {}):
            del triage_data["removed_keywords"][matched]
        _save_triage_data(triage_data)
        return f"✅ Reset {matched} to default keywords"

    return f"Unknown action: {action}. Use add, remove, list, or reset."


def get_category_summary(data: dict) -> str:
    """Get counts by category without full triage."""
    config = _get_email_config()

    if not config["address"] or not config["password"]:
        return "❌ Email not configured."

    days_back = data.get("days_back", 7)

    try:
        mail = _imap_connect(config, timeout=IMAP_TIMEOUT)
        mail.login(config["address"], config["password"])
        mail.select("INBOX")

        since = (datetime.now() - timedelta(days=days_back)).strftime("%d-%b-%Y")
        status, messages = mail.search(None, f'UNSEEN SINCE "{since}"')

        if status != "OK" or not messages[0]:
            return "📭 No unread emails."

        email_ids = messages[0].split()

        counts = {cat: 0 for cat in CATEGORIES}
        urgent_count = 0

        for eid in email_ids:
            try:
                status, msg_data = mail.fetch(eid, "(BODY[HEADER.FIELDS (FROM SUBJECT)])")
                if status != "OK":
                    continue

                header = msg_data[0][1].decode("utf-8", errors="replace")
                msg = email.message_from_string(header)

                subject = _decode_header(msg["Subject"]) or ""
                from_header = _decode_header(msg["From"]) or ""
                sender_email = _extract_email_address(from_header)
                sender_name = _extract_sender_name(from_header)

                category, _ = _categorize_email(sender_email, sender_name, subject)
                counts[category] += 1

                if _get_priority(subject, category, sender_email) == "urgent":
                    urgent_count += 1

            except (KeyError, AttributeError, TypeError) as e:
                logger.debug(f"Skipping malformed email: {e}")
                continue

        mail.logout()

        total = sum(counts.values())

        lines = [f"📊 Inbox Summary ({total} unread)\n"]

        if urgent_count:
            lines.append(f"🔴 {urgent_count} URGENT\n")

        for cat in CATEGORIES:
            if counts[cat] > 0:
                bar = "█" * min(counts[cat], 20)
                lines.append(f"  {cat}: {counts[cat]} {bar}")

        return "\n".join(lines)

    except Exception as e:
        logger.error(f"Triage operation failed: {e}")
        return "❌ Email operation failed. Check server connection."


def filter_by_category(data: dict) -> str:
    """Get emails from a specific category only."""
    config = _get_email_config()

    if not config["address"] or not config["password"]:
        return "❌ Email not configured."

    target_category = data.get("category", "").strip()

    matched_cat = None
    for cat in CATEGORIES:
        if cat.lower() == target_category.lower() or target_category.lower() in cat.lower():
            matched_cat = cat
            break

    if not matched_cat:
        return f"Unknown category: {target_category}\nAvailable: {', '.join(CATEGORIES)}"

    limit = data.get("limit", 20)
    days_back = data.get("days_back", 7)
    unread_only = data.get("unread_only", False)

    try:
        mail = _imap_connect(config, timeout=IMAP_TIMEOUT)
        mail.login(config["address"], config["password"])
        mail.select("INBOX")

        criteria = []
        if unread_only:
            criteria.append("UNSEEN")
        if days_back:
            since = (datetime.now() - timedelta(days=days_back)).strftime("%d-%b-%Y")
            criteria.append(f'SINCE "{since}"')

        search_str = " ".join(criteria) if criteria else "ALL"
        status, messages = mail.search(None, search_str)

        if status != "OK" or not messages[0]:
            return f"📭 No {matched_cat} emails found."

        email_ids = messages[0].split()

        matching = []
        priority_emoji = {"urgent": "🔴", "action": "🟡", "fyi": "🟢"}

        for eid in reversed(email_ids):  # Most recent first
            if len(matching) >= limit:
                break

            try:
                status, msg_data = mail.fetch(eid, "(RFC822)")
                if status != "OK":
                    continue

                raw = msg_data[0][1]
                msg = email.message_from_bytes(raw)

                subject = _decode_header(msg["Subject"]) or "(no subject)"
                from_header = _decode_header(msg["From"]) or ""
                date_str = msg["Date"] or ""
                sender_email = _extract_email_address(from_header)
                sender_name = _extract_sender_name(from_header)

                # Get body preview
                body_preview = ""
                if msg.is_multipart():
                    for part in msg.walk():
                        if part.get_content_type() == "text/plain":
                            try:
                                body_preview = part.get_payload(decode=True).decode(
                                    "utf-8", errors="replace"
                                )[:300]
                                break
                            except (UnicodeDecodeError, AttributeError, TypeError) as e:
                                logger.debug("Unicode decode error suppressed: %s", e)
                category, confidence = _categorize_email(
                    sender_email, sender_name, subject, body_preview
                )

                if category == matched_cat:
                    priority = _get_priority(subject, category, sender_email)
                    matching.append(
                        {
                            "from": sender_name,
                            "from_email": sender_email,
                            "subject": subject,
                            "date": date_str[:16] if date_str else "",
                            "priority": priority,
                            "confidence": confidence,
                        }
                    )
            except (KeyError, AttributeError, TypeError) as e:
                logger.debug(f"Skipping malformed email: {e}")
                continue

        mail.logout()

        if not matching:
            return f"📭 No {matched_cat} emails found in the last {days_back} days."

        lines = [f"📧 {matched_cat} ({len(matching)} emails)\n"]

        for em in matching:
            p = priority_emoji.get(em["priority"], "⚪")
            conf = " (?)" if em["confidence"] < 0.5 else ""
            subj = em["subject"][:55] + "..." if len(em["subject"]) > 55 else em["subject"]
            lines.append(f"{p} {em['from']}{conf}")
            lines.append(f"   {subj}")
            if em["date"]:
                lines.append(f"   {em['date']}")
            lines.append("")

        return "\n".join(lines)

    except Exception as e:
        logger.error(f"Triage operation failed: {e}")
        return "❌ Email operation failed. Check server connection."


# ── Priority keyword management ──────────────────────────────────────────────


def manage_priority_keywords(data: dict) -> str:
    """Add, remove, or list custom priority keywords (urgent/action levels).

    Actions:
      - add: Add a keyword to urgent or action level
      - remove: Remove a keyword from urgent or action level
      - list: Show all priority keywords (default + custom)
      - reset: Reset urgent or action keywords to defaults
    """
    action = data.get("action", "list").lower().strip()
    level = data.get("level", "").lower().strip()
    keyword = data.get("keyword", "").lower().strip()

    valid_levels = ("urgent", "action")
    triage_data = _load_triage_data()

    if action == "list":
        urgent_kws, action_kws = _get_effective_priority_keywords()
        custom_urgent = triage_data.get("custom_urgent_keywords", [])
        removed_urgent = triage_data.get("removed_urgent_keywords", [])
        custom_action = triage_data.get("custom_action_keywords", [])
        removed_action = triage_data.get("removed_action_keywords", [])

        if level == "urgent":
            lines = [f"🔴 Urgent Keywords ({len(urgent_kws)} active)\n"]
            for kw in sorted(urgent_kws):
                marker = " ⭐" if kw in custom_urgent else ""
                lines.append(f"  • {kw}{marker}")
            if removed_urgent:
                lines.append(f"\n  Removed ({len(removed_urgent)}):")
                for kw in sorted(removed_urgent):
                    lines.append(f"  ✗ {kw}")
            lines.append("\n⭐ = custom keyword")
            return "\n".join(lines)
        elif level == "action":
            lines = [f"🟡 Action Keywords ({len(action_kws)} active)\n"]
            for kw in sorted(action_kws):
                marker = " ⭐" if kw in custom_action else ""
                lines.append(f"  • {kw}{marker}")
            if removed_action:
                lines.append(f"\n  Removed ({len(removed_action)}):")
                for kw in sorted(removed_action):
                    lines.append(f"  ✗ {kw}")
            lines.append("\n⭐ = custom keyword")
            return "\n".join(lines)
        else:
            lines = ["🏷️ Priority Keywords\n"]
            lines.append(
                f"  🔴 Urgent: {len(urgent_kws)} keywords"
                + (f" (+{len(custom_urgent)} custom)" if custom_urgent else "")
            )
            lines.append(
                f"  🟡 Action: {len(action_kws)} keywords"
                + (f" (+{len(custom_action)} custom)" if custom_action else "")
            )
            lines.append("\nUse manage_priority_keywords with level='urgent' or 'action' for details.")
            return "\n".join(lines)

    if action == "add":
        if not level or level not in valid_levels:
            return f"Please provide level: {', '.join(valid_levels)}"
        if not keyword:
            return "Please provide a keyword to add."

        if level == "urgent":
            custom = triage_data.setdefault("custom_urgent_keywords", [])
            removed = triage_data.get("removed_urgent_keywords", [])
            defaults = URGENT_KEYWORDS
        else:
            custom = triage_data.setdefault("custom_action_keywords", [])
            removed = triage_data.get("removed_action_keywords", [])
            defaults = ACTION_KEYWORDS

        if keyword in custom or (keyword in defaults and keyword not in removed):
            return f"Keyword '{keyword}' already active in {level}."
        if keyword in removed:
            removed.remove(keyword)
        elif keyword not in defaults:
            custom.append(keyword)
        _save_triage_data(triage_data)
        return f"✅ Added '{keyword}' to {level} priority keywords"

    if action == "remove":
        if not level or level not in valid_levels:
            return f"Please provide level: {', '.join(valid_levels)}"
        if not keyword:
            return "Please provide a keyword to remove."

        if level == "urgent":
            custom = triage_data.get("custom_urgent_keywords", [])
            defaults = URGENT_KEYWORDS
            removed_key = "removed_urgent_keywords"
        else:
            custom = triage_data.get("custom_action_keywords", [])
            defaults = ACTION_KEYWORDS
            removed_key = "removed_action_keywords"

        if keyword in custom:
            custom.remove(keyword)
            _save_triage_data(triage_data)
            return f"✅ Removed custom keyword '{keyword}' from {level}"

        if keyword in defaults:
            removed = triage_data.setdefault(removed_key, [])
            if keyword not in removed:
                removed.append(keyword)
            _save_triage_data(triage_data)
            return f"✅ Hidden default keyword '{keyword}' from {level}"

        return f"Keyword '{keyword}' not found in {level}."

    if action == "reset":
        if not level or level not in valid_levels:
            return f"Please provide level: {', '.join(valid_levels)}"

        if level == "urgent":
            triage_data.pop("custom_urgent_keywords", None)
            triage_data.pop("removed_urgent_keywords", None)
        else:
            triage_data.pop("custom_action_keywords", None)
            triage_data.pop("removed_action_keywords", None)
        _save_triage_data(triage_data)
        return f"✅ Reset {level} priority keywords to defaults"

    return f"Unknown action: {action}. Use add, remove, list, or reset."


def explain_triage(data: dict) -> str:
    """Explain why an email was categorized and prioritized the way it was."""
    email_number = data.get("email_number")
    if email_number is None:
        return "Please provide an email_number (e.g., 1 for #1 from triage output)."

    try:
        email_number = int(email_number)
    except (ValueError, TypeError):
        return f"Invalid email number: {email_number}"

    # Try session context first
    ctx = data.get("_context") or {}
    session = ctx.get("session")
    triage_emails = None
    if session is not None:
        triage_emails = session.session_context.get("triage_emails")

    # Fallback to thread-local
    if not triage_emails:
        import threading
        triage_emails = getattr(
            threading.current_thread(), "_familiar_triage_ctx", {}
        ).get("emails")

    if not triage_emails:
        return "No triage data available. Run triage_inbox first."

    email_data = None
    for em in triage_emails:
        if em.get("index") == email_number:
            email_data = em
            break

    if not email_data:
        return f"Email #{email_number} not found. Valid range: 1-{len(triage_emails)}."

    priority_emoji = {"urgent": "🔴", "action": "🟡", "fyi": "🟢"}
    p_emoji = priority_emoji.get(email_data.get("priority", "fyi"), "⚪")

    lines = [
        f"📋 Triage Explanation — #{email_number}\n",
        f"  From: {email_data.get('from_name', '?')} <{email_data.get('from_email', '?')}>",
        f"  Subject: {email_data.get('subject', '?')}",
        f"  Category: {email_data.get('category', '?')}",
        f"  Priority: {p_emoji} {email_data.get('priority', '?')}",
    ]

    reasons = _last_triage_reasons.get(email_number, [])
    if reasons:
        lines.append("\n  Classification chain:")
        for reason in reasons:
            lines.append(f"    → {reason}")
    else:
        lines.append("\n  (Detailed reasons not available — re-run triage for full chain)")

    return "\n".join(lines)


def reclassify_email(data: dict) -> str:
    """Reclassify a triaged email to a different category and/or priority."""
    email_number = data.get("email_number")
    new_category = data.get("category", "").strip()
    new_priority = data.get("priority", "").strip().lower()

    if email_number is None:
        return "Please provide an email_number."

    try:
        email_number = int(email_number)
    except (ValueError, TypeError):
        return f"Invalid email number: {email_number}"

    if not new_category and not new_priority:
        return "Please provide a category and/or priority to reclassify."

    # Validate category
    category_match = None
    if new_category:
        for cat in CATEGORIES:
            if cat.lower() == new_category.lower() or new_category.lower() in cat.lower():
                category_match = cat
                break
        if not category_match:
            return f"Unknown category: {new_category}\nValid: {', '.join(CATEGORIES)}"

    # Validate priority
    if new_priority and new_priority not in ("urgent", "action", "fyi"):
        return f"Invalid priority: {new_priority}. Use urgent, action, or fyi."

    # Find email in session context
    ctx = data.get("_context") or {}
    session = ctx.get("session")
    sessions_mgr = ctx.get("session_manager")
    triage_emails = None
    if session is not None:
        triage_emails = session.session_context.get("triage_emails")

    if not triage_emails:
        import threading
        triage_ctx = getattr(threading.current_thread(), "_familiar_triage_ctx", {})
        triage_emails = triage_ctx.get("emails")

    if not triage_emails:
        return "No triage data available. Run triage_inbox first."

    email_data = None
    for em in triage_emails:
        if em.get("index") == email_number:
            email_data = em
            break

    if not email_data:
        return f"Email #{email_number} not found. Valid range: 1-{len(triage_emails)}."

    changes = []

    # Update category
    if category_match and category_match != email_data.get("category"):
        old_cat = email_data.get("category", "?")
        email_data["category"] = category_match
        changes.append(f"Category: {old_cat} → {category_match}")

        # Auto-teach sender
        sender = email_data.get("from_email", "")
        if sender:
            triage_data = _load_triage_data()
            triage_data.setdefault("sender_categories", {})[sender] = category_match
            _save_triage_data(triage_data)
            changes.append(f"Learned: {sender} → {category_match}")

    # Update priority
    if new_priority and new_priority != email_data.get("priority"):
        old_pri = email_data.get("priority", "?")
        email_data["priority"] = new_priority
        changes.append(f"Priority: {old_pri} → {new_priority}")

        # Auto-set sender priority rule
        sender = email_data.get("from_email", "")
        if sender:
            triage_data = _load_triage_data()
            triage_data.setdefault("sender_priorities", {})[sender] = new_priority
            _save_triage_data(triage_data)
            changes.append(f"Sender priority rule: {sender} → {new_priority}")

    if not changes:
        return f"Email #{email_number} already has that classification."

    # Save updated session
    if session is not None:
        session.session_context["triage_emails"] = triage_emails
        if sessions_mgr is not None:
            sessions_mgr.save_session(session)

    return "✅ Reclassified #{}:\n  {}".format(email_number, "\n  ".join(changes))


def set_sender_priority(data: dict) -> str:
    """Set or manage sender/domain priority rules."""
    action = data.get("action", "set").lower().strip()
    sender = data.get("sender", "").lower().strip()
    priority = data.get("priority", "").lower().strip()

    triage_data = _load_triage_data()
    sender_priorities = triage_data.setdefault("sender_priorities", {})

    if action == "list":
        if not sender_priorities:
            return "No sender priority rules set."
        lines = ["📋 Sender Priority Rules\n"]
        priority_emoji = {"urgent": "🔴", "action": "🟡", "fyi": "🟢"}
        by_priority: dict[str, list[str]] = {"urgent": [], "action": [], "fyi": []}
        for s, p in sender_priorities.items():
            by_priority.setdefault(p, []).append(s)
        for p in ("urgent", "action", "fyi"):
            senders = by_priority.get(p, [])
            if senders:
                lines.append(f"  {priority_emoji.get(p, '⚪')} {p.upper()}:")
                for s in sorted(senders):
                    lines.append(f"    • {s}")
        return "\n".join(lines)

    if action == "remove":
        if not sender:
            return "Please provide a sender email or domain to remove."
        if sender in sender_priorities:
            old = sender_priorities.pop(sender)
            _save_triage_data(triage_data)
            return f"✅ Removed priority rule for {sender} (was {old})"
        return f"No priority rule found for {sender}."

    # Default action: set
    if not sender:
        return "Please provide a sender email or domain."
    if not priority or priority not in ("urgent", "action", "fyi"):
        return "Please provide priority: urgent, action, or fyi."

    sender_priorities[sender] = priority
    _save_triage_data(triage_data)
    return f"✅ Sender priority rule: {sender} → {priority}"


# ── Tool definitions ──────────────────────────────────────────────────────────

TOOLS = [
    {
        "name": "triage_inbox",
        "description": (
            "Fetch and sort emails into organizational categories with priority flags. "
            "Categories: Campaign, Trans Pride, GJL Operations, Staff, Board, Coalitions, "
            "Donors, Grants, Media, Volunteers, Notifications (CI/system), Promotional. "
            "Uses Gmail API when connected, falling back to IMAP. "
            "Categorization combines learned sender mappings, promotional/notification "
            "heuristics, domain rules, and keyword analysis. "
            "Results are numbered for easy reference in follow-up commands."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "limit": {"type": "integer", "description": "Max emails to process", "default": 25},
                "unread_only": {
                    "type": "boolean",
                    "default": True,
                    "description": "Only include unread emails",
                },
                "days_back": {
                    "type": "integer",
                    "description": "How many days to look back",
                    "default": 3,
                },
                "account": {
                    "type": "string",
                    "description": "Account to triage: 'gmail', 'yahoo', etc. Omit for primary.",
                },
            },
        },
        "handler": triage_inbox,
        "category": "triage",
    },
    {
        "name": "filter_emails",
        "description": (
            "Get emails from one specific category only, such as 'show me Board emails' or "
            "'what Notifications came in'. Fetches emails via IMAP, categorizes each one, and "
            "returns only those matching the target category. "
            "Supports filtering by date range and read/unread status."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "category": {
                    "type": "string",
                    "description": (
                        "Category to filter: Campaign, Trans Pride, GJL Operations, Staff, "
                        "Board, Coalitions, Donors, Grants, Media, Volunteers, Notifications, "
                        "Promotional"
                    ),
                },
                "limit": {
                    "type": "integer",
                    "default": 20,
                    "description": "Maximum number of emails to return",
                },
                "days_back": {
                    "type": "integer",
                    "default": 7,
                    "description": "How many days to look back",
                },
                "unread_only": {
                    "type": "boolean",
                    "default": False,
                    "description": "Only include unread emails",
                },
            },
            "required": ["category"],
        },
        "handler": filter_by_category,
        "category": "triage",
    },
    {
        "name": "inbox_summary",
        "description": (
            "Quick count of unread emails grouped by category without fetching full message "
            "contents. Faster than triage_inbox because it only reads headers. "
            "Shows urgent email count and a bar chart of category distribution."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "days_back": {
                    "type": "integer",
                    "default": 7,
                    "description": "How many days to look back",
                }
            },
        },
        "handler": get_category_summary,
        "category": "triage",
    },
    {
        "name": "teach_sender",
        "description": (
            "Teach the agent which category a sender or domain belongs to for future email "
            "categorization. Provide a full email address for sender-level mapping, or just a "
            "domain for domain-level mapping. "
            "Learned mappings take highest priority in categorization."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "sender": {
                    "type": "string",
                    "description": "Email address or domain (e.g., 'jane@example.com' or 'example.com')",
                },
                "category": {
                    "type": "string",
                    "description": (
                        "Category: Campaign, Trans Pride, GJL Operations, Staff, Board, "
                        "Coalitions, Donors, Grants, Media, Volunteers, Notifications, Promotional"
                    ),
                },
            },
            "required": ["sender", "category"],
        },
        "handler": teach_sender,
        "category": "triage",
    },
    {
        "name": "manage_keywords",
        "description": (
            "Add, remove, or list custom keywords for email triage categories. "
            "Keywords are used to automatically categorize emails. "
            "Actions: 'add' a keyword to a category, 'remove' a keyword, "
            "'list' keywords for a category or all categories, "
            "'reset' a category to defaults. "
            "Custom keywords persist across restarts. "
            "Examples: add 'james talarico' to Promotional, "
            "remove 'schedule' from GJL Operations, "
            "list keywords for Staff."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "action": {
                    "type": "string",
                    "enum": ["add", "remove", "list", "reset"],
                    "description": "Action: add, remove, list, or reset",
                    "default": "list",
                },
                "category": {
                    "type": "string",
                    "description": (
                        "Category name (e.g., Staff, Promotional, Notifications). "
                        "Required for add/remove/reset. Optional for list (shows all if omitted)."
                    ),
                },
                "keyword": {
                    "type": "string",
                    "description": "Keyword to add or remove (lowercase). Required for add/remove.",
                },
            },
        },
        "handler": manage_keywords,
        "category": "triage",
    },
    {
        "name": "list_learned_senders",
        "description": (
            "Show all learned sender and domain categorizations grouped by category. "
            "Displays both individual sender mappings and domain-level wildcard mappings."
        ),
        "input_schema": {"type": "object", "properties": {}},
        "handler": list_learned_senders,
        "category": "triage",
    },
    {
        "name": "forget_sender",
        "description": (
            "Remove a previously learned sender or domain categorization. "
            "After forgetting, the sender will be categorized using keyword/heuristic matching."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "sender": {"type": "string", "description": "Email address or domain to forget"}
            },
            "required": ["sender"],
        },
        "handler": forget_sender,
        "category": "triage",
    },
    {
        "name": "manage_priority_keywords",
        "description": (
            "Add, remove, or list custom priority keywords that control email urgency flags. "
            "Two levels: 'urgent' (🔴) and 'action' (🟡). "
            "Actions: 'add' a keyword to a level, 'remove' a keyword, "
            "'list' keywords for a level or both, 'reset' a level to defaults. "
            "Examples: add 'lawsuit' as urgent keyword, "
            "remove 'p1' from urgent, list urgent keywords."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "action": {
                    "type": "string",
                    "enum": ["add", "remove", "list", "reset"],
                    "description": "Action: add, remove, list, or reset",
                    "default": "list",
                },
                "level": {
                    "type": "string",
                    "enum": ["urgent", "action"],
                    "description": "Priority level: 'urgent' or 'action'",
                },
                "keyword": {
                    "type": "string",
                    "description": "Keyword to add or remove (lowercase). Required for add/remove.",
                },
            },
        },
        "handler": manage_priority_keywords,
        "category": "triage",
    },
    {
        "name": "explain_triage",
        "description": (
            "Explain why a specific email was categorized and prioritized the way it was. "
            "Takes an email number from the triage output (e.g., #3) and shows the sender, "
            "subject, assigned category, priority, and the classification reasoning chain. "
            "Must run triage_inbox first."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "email_number": {
                    "type": "integer",
                    "description": "Email number from triage output (e.g., 3 for #3)",
                },
            },
            "required": ["email_number"],
        },
        "handler": explain_triage,
        "category": "triage",
    },
    {
        "name": "reclassify_email",
        "description": (
            "Reclassify a triaged email to a different category and/or priority. "
            "Updates the session context so subsequent references reflect the change. "
            "Automatically teaches the sender mapping and sets sender priority rules. "
            "Use after triage_inbox when a classification is wrong."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "email_number": {
                    "type": "integer",
                    "description": "Email number from triage output (e.g., 5 for #5)",
                },
                "category": {
                    "type": "string",
                    "description": (
                        "New category: Campaign, Trans Pride, GJL Operations, Staff, Board, "
                        "Coalitions, Donors, Grants, Media, Volunteers, Notifications, Promotional"
                    ),
                },
                "priority": {
                    "type": "string",
                    "enum": ["urgent", "action", "fyi"],
                    "description": "New priority level",
                },
            },
            "required": ["email_number"],
        },
        "handler": reclassify_email,
        "category": "triage",
    },
    {
        "name": "set_sender_priority",
        "description": (
            "Set a priority rule for a sender email or domain. "
            "Emails from this sender/domain will always get the specified priority. "
            "Sender priority rules take highest precedence over keyword matching. "
            "Actions: 'set' (default), 'remove', 'list'."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "action": {
                    "type": "string",
                    "enum": ["set", "remove", "list"],
                    "description": "Action: set (default), remove, or list",
                    "default": "set",
                },
                "sender": {
                    "type": "string",
                    "description": "Email address or domain (e.g., 'boss@example.com' or 'example.com')",
                },
                "priority": {
                    "type": "string",
                    "enum": ["urgent", "action", "fyi"],
                    "description": "Priority level to assign",
                },
            },
        },
        "handler": set_sender_priority,
        "category": "triage",
    },
]
