"""
Unified Memory Consolidation Bridge — HIPAA Compliant.

Bridges operational data (analytics, credentials, email, workspace) into the
Memory Tree. All data passes through PHI scrubbing, deduplication, and audit
logging before storage.

Architecture based on 2025-2026 research:
- Zep: temporal knowledge graph (valid_from/valid_to on every fact)
- Mem0: ADD/UPDATE/DELETE/NOOP classification
- MAGMA: semantic + temporal + causal + entity edges
- HIPAA: Safe Harbor 18 scrubbing, encryption enforcement, audit trail

See: familiar/core/memory_tree.py for the storage layer.
"""

from __future__ import annotations

import hashlib
import logging
import sqlite3
import threading
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Optional

logger = logging.getLogger(__name__)

# ── Consolidation action types (Mem0 pattern) ──

ACTION_ADD = "add"
ACTION_UPDATE = "update"
ACTION_INVALIDATE = "invalidate"
ACTION_NOOP = "noop"
ACTION_SKIP_PHI = "skip_phi"       # blocked by PHI gate
ACTION_SKIP_DEDUP = "skip_dedup"   # duplicate content
ACTION_SKIP_ENCRYPTION = "skip_encryption"  # encryption not available for PHI source

# ── Source identifiers for consolidated memories ──

SOURCE_ANALYTICS = "consolidation:analytics"
SOURCE_CREDENTIALS = "consolidation:credentials"
SOURCE_EMAIL = "consolidation:email"
SOURCE_WORKSPACE = "consolidation:workspace"

# ── Schema extension for consolidation audit trail ──

_CONSOLIDATION_SCHEMA = """
CREATE TABLE IF NOT EXISTS consolidation_log (
    id          INTEGER PRIMARY KEY AUTOINCREMENT,
    timestamp   TEXT NOT NULL,
    source      TEXT NOT NULL,
    action      TEXT NOT NULL,
    memory_id   TEXT,
    source_id   TEXT,
    details     TEXT,
    phi_detected BOOLEAN DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_consol_source ON consolidation_log(source);
CREATE INDEX IF NOT EXISTS idx_consol_timestamp ON consolidation_log(timestamp);
CREATE INDEX IF NOT EXISTS idx_consol_action ON consolidation_log(action);
"""


@dataclass
class ConsolidationResult:
    """Summary of a consolidation run."""

    source: str
    added: int = 0
    updated: int = 0
    invalidated: int = 0
    noops: int = 0
    skipped_phi: int = 0
    skipped_dedup: int = 0
    skipped_encryption: int = 0
    errors: int = 0
    duration_ms: float = 0.0


@dataclass
class ConsolidationStatus:
    """Overall bridge status."""

    last_run: Optional[str] = None
    results: dict[str, ConsolidationResult] = field(default_factory=dict)
    encryption_available: bool = False
    phi_scrubbing_available: bool = False


class ConsolidationBridge:
    """Orchestrates consolidation of operational data into the Memory Tree.

    Security constraints:
    - PHI scrubbing runs on ALL text before Memory Tree storage
    - Encryption enforced for PHI-sensitive sources (email, workspace)
    - Audit trail for every consolidation operation
    - No secrets (keys/tokens/passwords) ever stored in memories
    - Soft delete only (invalidate, never hard delete)
    """

    def __init__(self, tree=None, data_dir: Optional[Path] = None):
        """Initialize with a MemoryTree instance.

        Args:
            tree: MemoryTree instance. If None, creates a default one.
            data_dir: Override data directory (default: ~/.familiar/data).
                      Used for testing with mock data.
        """
        if tree is None:
            from familiar.core.memory_tree import MemoryTree

            tree = MemoryTree()
        self._tree = tree
        self._data_dir = data_dir or (Path.home() / ".familiar" / "data")
        self._lock = threading.Lock()
        self._last_run: Optional[str] = None
        self._results: dict[str, ConsolidationResult] = {}

        # Ensure consolidation_log table exists
        self._ensure_schema()

        # Check capabilities
        self._encryption_available = self._check_encryption()
        self._phi_available = self._check_phi_scrubbing()

        logger.info(
            "ConsolidationBridge initialized: encryption=%s, phi_scrub=%s",
            self._encryption_available,
            self._phi_available,
        )

    # ── Schema ──

    def _ensure_schema(self) -> None:
        """Create the consolidation_log table if it doesn't exist."""
        try:
            conn = self._tree._connect()
            try:
                conn.executescript(_CONSOLIDATION_SCHEMA)
                conn.commit()
            finally:
                conn.close()
        except Exception as e:
            logger.warning("Failed to create consolidation schema: %s", e)

    # ── Capability checks ──

    def _check_encryption(self) -> bool:
        """Check if encryption is available for the Memory Tree."""
        return getattr(self._tree, "_encrypt", False)

    def _check_phi_scrubbing(self) -> bool:
        """Check if PHI scrubbing infrastructure is available."""
        try:
            from familiar.skills.training.corpus import phi_scrub  # noqa: F401

            return True
        except ImportError:
            # Try fallback
            try:
                from familiar.skills.training.corpus import _phi_scrub_fallback  # noqa: F401

                return True
            except ImportError:
                return False

    # ── PHI Gate (HIPAA Safe Harbor 18) ──

    def _is_pii_enabled(self) -> bool:
        """Check if the user has enabled PII/PHI detection via the dashboard toggle.

        Checks multiple sources in order:
        1. Running agent config (most authoritative, live toggle)
        2. Guardrails config
        3. Default: False (user must opt in)
        """
        # Check the running agent's config first (set by /api/settings/pii toggle)
        try:
            from familiar.core.agent import _global_agent

            if _global_agent and hasattr(_global_agent, "config"):
                return getattr(_global_agent.config.agent, "enable_pii_detection", False)
        except (ImportError, Exception):
            pass

        # Fallback: check guardrails config
        try:
            from familiar.core.guardrails import get_guardrails_config

            cfg = get_guardrails_config()
            return getattr(cfg, "enable_pii_detection", False)
        except (ImportError, Exception):
            pass

        return False

    def phi_gate(self, text: str) -> tuple[str, bool]:
        """Run PHI scrubbing on text if PII detection is enabled.

        Returns (scrubbed_text, phi_was_found).

        Respects the user's PII detection toggle:
        - OFF: text passes through unchanged (user's choice)
        - ON: HIPAA Safe Harbor 18 scrubbing applied

        The toggle is checked at runtime so changes take effect immediately.
        """
        if not text:
            return text, False

        # Respect the user's PII toggle setting
        if not self._is_pii_enabled():
            return text, False

        try:
            from familiar.skills.training.corpus import phi_scrub

            return phi_scrub(text)
        except ImportError:
            # Fallback to regex-based scrubbing
            return self._phi_fallback(text)

    @staticmethod
    def _phi_fallback(text: str) -> tuple[str, bool]:
        """Lightweight regex PHI scrubbing (fallback if skill unavailable)."""
        import re

        patterns = [
            (re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"), "[EMAIL]"),
            (re.compile(r"\b\d{3}[-.\s]?\d{3}[-.\s]?\d{4}\b"), "[PHONE]"),
            (re.compile(r"\b\d{3}[-\s]?\d{2}[-\s]?\d{4}\b"), "[SSN]"),
            (re.compile(r"\b(?:\d{4}[-\s]?){3}\d{4}\b"), "[CARD]"),
            (re.compile(r"\b\d{1,2}/\d{1,2}/\d{2,4}\b"), "[DATE]"),
        ]
        scrubbed = text
        found = False
        for pat, placeholder in patterns:
            if pat.search(scrubbed):
                scrubbed = pat.sub(placeholder, scrubbed)
                found = True
        return scrubbed, found

    # ── Deduplication ──

    def dedup_check(self, title: str, body: str) -> Optional[str]:
        """Check if content already exists in Memory Tree.

        Returns the memory_id if a duplicate exists, None otherwise.
        Uses SHA-256 content hash (same as MemoryTree.store).
        """
        content = f"{title}\n{body}".strip()
        chash = hashlib.sha256(content.encode("utf-8")).hexdigest()
        existing = self._tree.get_by_hash(chash)
        if existing:
            return existing.memory_id
        return None

    # ── Audit Trail ──

    def log_consolidation(
        self,
        source: str,
        action: str,
        memory_id: Optional[str] = None,
        source_id: Optional[str] = None,
        details: str = "",
        phi_detected: bool = False,
    ) -> None:
        """Write an entry to the consolidation audit log."""
        try:
            conn = self._tree._connect()
            try:
                conn.execute(
                    "INSERT INTO consolidation_log (timestamp, source, action, memory_id, source_id, details, phi_detected) "
                    "VALUES (?, ?, ?, ?, ?, ?, ?)",
                    (
                        datetime.now(timezone.utc).isoformat(),
                        source,
                        action,
                        memory_id,
                        source_id,
                        details[:500] if details else "",  # truncate long details
                        1 if phi_detected else 0,
                    ),
                )
                conn.commit()
            finally:
                conn.close()
        except Exception as e:
            logger.warning("Failed to write consolidation log: %s", e)

    # ── Core Store with Audit ──

    def store_with_audit(
        self,
        source: str,
        title: str,
        body: str,
        tier: str = "long",
        tags: Optional[list[str]] = None,
        importance: float = 0.5,
        source_id: str = "",
        require_encryption: bool = False,
    ) -> Optional[str]:
        """Store a memory with full PHI scrubbing, dedup, and audit.

        Args:
            source: Consolidation source identifier (SOURCE_ANALYTICS, etc.)
            title: Memory title
            body: Memory body text
            tier: Memory tier (working/short/long/archival)
            tags: Tag list
            importance: 0.0-1.0 importance score
            source_id: Original record ID in source database
            require_encryption: If True, refuse to store without encryption

        Returns:
            memory_id if stored, None if skipped.
        """
        # Encryption gate for PHI-sensitive sources
        if require_encryption and not self._encryption_available:
            self.log_consolidation(
                source, ACTION_SKIP_ENCRYPTION,
                source_id=source_id,
                details=f"Skipped: encryption required but not available. Title: {title[:80]}",
            )
            return None

        # PHI gate — scrub ALL text
        scrubbed_title, title_phi = self.phi_gate(title)
        scrubbed_body, body_phi = self.phi_gate(body)
        phi_found = title_phi or body_phi

        if phi_found:
            logger.info("PHI detected in consolidation from %s: %s", source, title[:50])

        # Deduplication gate
        existing_id = self.dedup_check(scrubbed_title, scrubbed_body)
        if existing_id:
            self.log_consolidation(
                source, ACTION_SKIP_DEDUP,
                memory_id=existing_id,
                source_id=source_id,
                details=f"Duplicate of {existing_id}. Title: {scrubbed_title[:80]}",
                phi_detected=phi_found,
            )
            return None

        # Store in Memory Tree
        all_tags = list(tags or [])
        all_tags.append(source)  # Tag with consolidation source

        entry = self._tree.store(
            title=scrubbed_title,
            body=scrubbed_body,
            tier=tier,
            tags=all_tags,
            source=source,
            importance=importance,
        )

        # Audit trail
        self.log_consolidation(
            source, ACTION_ADD,
            memory_id=entry.memory_id,
            source_id=source_id,
            details=f"Stored: {scrubbed_title[:80]}",
            phi_detected=phi_found,
        )

        return entry.memory_id

    def update_with_audit(
        self,
        source: str,
        memory_id: str,
        title: str,
        body: str,
        source_id: str = "",
    ) -> bool:
        """Update an existing memory with PHI scrubbing and audit."""
        scrubbed_title, title_phi = self.phi_gate(title)
        scrubbed_body, body_phi = self.phi_gate(body)
        phi_found = title_phi or body_phi

        try:
            new_entry = self._tree.update_fact(
                old_id=memory_id,
                new_title=scrubbed_title,
                new_body=scrubbed_body,
            )
            new_id = new_entry.memory_id if new_entry else memory_id
            self.log_consolidation(
                source, ACTION_UPDATE,
                memory_id=new_id,
                source_id=source_id,
                details=f"Updated: {scrubbed_title[:80]} (supersedes {memory_id})",
                phi_detected=phi_found,
            )
            return True
        except Exception as e:
            logger.warning("Failed to update memory %s: %s", memory_id, e)
            return False

    def add_edge(
        self,
        from_id: str,
        to_id: str,
        relation_type: str = "related",
        weight: float = 1.0,
    ) -> None:
        """Create a typed relationship edge between two memories."""
        try:
            conn = self._tree._connect()
            try:
                conn.execute(
                    "INSERT OR REPLACE INTO memory_edges (from_id, to_id, relation_type, weight, created_at) "
                    "VALUES (?, ?, ?, ?, ?)",
                    (from_id, to_id, relation_type, weight, datetime.now(timezone.utc).isoformat()),
                )
                conn.commit()
            finally:
                conn.close()
        except Exception as e:
            logger.warning("Failed to add edge %s→%s: %s", from_id, to_id, e)

    # ── Find existing consolidated memory by tag + title pattern ──

    def find_existing(self, source: str, title_prefix: str) -> Optional[str]:
        """Find an existing consolidated memory by source tag and title prefix.

        Used for UPDATE operations — find the previous version to supersede.
        Returns memory_id or None.
        """
        try:
            results = self._tree.list_by_tag(source, tier=None, limit=50)
            for entry in results:
                if entry.title.startswith(title_prefix):
                    return entry.memory_id
        except Exception:
            pass
        return None

    # ── Consolidation orchestrator ──

    def consolidate_all(self) -> dict[str, ConsolidationResult]:
        """Run all consolidation adapters.

        Returns dict of source → ConsolidationResult.
        Thread-safe — only one consolidation runs at a time.
        """
        if not self._lock.acquire(blocking=False):
            logger.info("Consolidation already in progress, skipping")
            return self._results

        try:
            import time

            results: dict[str, ConsolidationResult] = {}

            # Sprint 1: Analytics
            t0 = time.monotonic()
            results[SOURCE_ANALYTICS] = self._consolidate_analytics()
            results[SOURCE_ANALYTICS].duration_ms = (time.monotonic() - t0) * 1000

            # Sprint 2: Credentials
            t0 = time.monotonic()
            results[SOURCE_CREDENTIALS] = self._consolidate_credentials()
            results[SOURCE_CREDENTIALS].duration_ms = (time.monotonic() - t0) * 1000

            # Sprint 3: Email
            t0 = time.monotonic()
            results[SOURCE_EMAIL] = self._consolidate_email()
            results[SOURCE_EMAIL].duration_ms = (time.monotonic() - t0) * 1000

            # Sprint 4: Workspace
            t0 = time.monotonic()
            results[SOURCE_WORKSPACE] = self._consolidate_workspace()
            results[SOURCE_WORKSPACE].duration_ms = (time.monotonic() - t0) * 1000

            self._results = results
            self._last_run = datetime.now(timezone.utc).isoformat()
            return results
        finally:
            self._lock.release()

    # ── Adapter stubs (implemented in Sprints 1-4) ──

    def _consolidate_analytics(self) -> ConsolidationResult:
        """Sprint 1: Consolidate LLM usage, costs, and errors from analytics.db."""
        result = ConsolidationResult(source=SOURCE_ANALYTICS)
        analytics_db = self._data_dir / "analytics.db"
        if not analytics_db.exists():
            return result

        try:
            import sqlite3 as _sql

            conn = _sql.connect(str(analytics_db))
            conn.row_factory = _sql.Row

            # 1. Monthly spend summary (WORKING tier — always in agent context)
            self._consolidate_monthly_spend(conn, result)

            # 2. Daily usage summaries (LONG tier — persistent knowledge)
            self._consolidate_daily_usage(conn, result)

            # 3. Error events (SHORT tier — session-relevant)
            self._consolidate_error_events(conn, result)

            conn.close()
        except Exception as e:
            logger.warning("Analytics consolidation failed: %s", e)
            result.errors += 1
        return result

    def _consolidate_monthly_spend(self, conn, result: ConsolidationResult) -> None:
        """Consolidate monthly LLM spend into a WORKING tier memory."""
        try:
            row = conn.execute("""
                SELECT substr(date, 1, 7) as month,
                       SUM(event_count) as events,
                       SUM(tokens_in) as tok_in,
                       SUM(tokens_out) as tok_out,
                       SUM(cost_usd) as cost,
                       SUM(error_count) as errors
                FROM daily_stats
                WHERE date >= date('now', 'start of month')
                GROUP BY month
            """).fetchone()
            if not row or not row["month"]:
                return

            # Get active model info
            model_row = conn.execute("""
                SELECT model, COUNT(*) as calls FROM events
                WHERE event_type = 'message' AND model != '' AND model NOT LIKE '%Mock%'
                  AND timestamp >= date('now', 'start of month')
                GROUP BY model ORDER BY calls DESC LIMIT 1
            """).fetchone()
            top_model = model_row["model"] if model_row else "unknown"

            month = row["month"]
            cost = row["cost"] or 0
            events = row["events"] or 0
            tok_in = row["tok_in"] or 0
            tok_out = row["tok_out"] or 0

            title = f"Monthly LLM Spend: ${cost:.2f} ({month})"
            body = (
                f"Total cost: ${cost:.4f}. Events: {events}. "
                f"Tokens: {tok_in:,} in / {tok_out:,} out. "
                f"Top model: {top_model}. Errors: {row['errors'] or 0}."
            )

            # Check if we already have a monthly entry to UPDATE
            existing = self.find_existing(SOURCE_ANALYTICS, f"Monthly LLM Spend")
            if existing:
                self.update_with_audit(
                    source=SOURCE_ANALYTICS,
                    memory_id=existing,
                    title=title,
                    body=body,
                    source_id=f"monthly_{month}",
                )
                result.updated += 1
            else:
                mid = self.store_with_audit(
                    source=SOURCE_ANALYTICS,
                    title=title,
                    body=body,
                    tier="working",
                    tags=["analytics", "spend", "monthly", month],
                    importance=0.7,
                    source_id=f"monthly_{month}",
                )
                if mid:
                    result.added += 1
                else:
                    result.noops += 1
        except Exception as e:
            logger.warning("Monthly spend consolidation failed: %s", e)
            result.errors += 1

    def _consolidate_daily_usage(self, conn, result: ConsolidationResult) -> None:
        """Consolidate daily LLM usage summaries into LONG tier memories."""
        try:
            # Get recent days not yet consolidated (check by tag)
            rows = conn.execute("""
                SELECT date,
                       GROUP_CONCAT(DISTINCT model) as models,
                       SUM(event_count) as events,
                       SUM(tokens_in) as tok_in,
                       SUM(tokens_out) as tok_out,
                       SUM(cost_usd) as cost,
                       SUM(error_count) as errors
                FROM daily_stats
                WHERE date >= date('now', '-7 days')
                GROUP BY date
                ORDER BY date DESC
            """).fetchall()

            prev_mid = None
            for row in rows:
                date = row["date"]
                models = row["models"] or "unknown"
                events = row["events"] or 0
                cost = row["cost"] or 0

                title = f"LLM Usage: {date}"
                body = (
                    f"Models: {models}. Events: {events}. "
                    f"Tokens: {row['tok_in'] or 0:,} in / {row['tok_out'] or 0:,} out. "
                    f"Cost: ${cost:.4f}. Errors: {row['errors'] or 0}."
                )

                mid = self.store_with_audit(
                    source=SOURCE_ANALYTICS,
                    title=title,
                    body=body,
                    tier="long",
                    tags=["analytics", "llm_usage", "daily", date],
                    importance=0.3,
                    source_id=f"daily_{date}",
                )
                if mid:
                    result.added += 1
                    # Edge: this day derived_from previous day
                    if prev_mid:
                        self.add_edge(mid, prev_mid, "derived_from")
                    prev_mid = mid
                else:
                    result.skipped_dedup += 1
                    result.noops += 1
        except Exception as e:
            logger.warning("Daily usage consolidation failed: %s", e)
            result.errors += 1

    def _consolidate_error_events(self, conn, result: ConsolidationResult) -> None:
        """Consolidate recent error events into SHORT tier memories."""
        try:
            rows = conn.execute("""
                SELECT timestamp, skill, channel, model, error_message, latency_ms
                FROM events
                WHERE (error_message IS NOT NULL AND error_message != '')
                   OR success = 0
                ORDER BY timestamp DESC
                LIMIT 20
            """).fetchall()

            for row in rows:
                ts = row["timestamp"] or ""
                skill = row["skill"] or "unknown"
                error_msg = row["error_message"] or "Unknown error"

                title = f"Error: {skill} — {error_msg[:50]}"
                body = (
                    f"Timestamp: {ts}. Skill: {skill}. "
                    f"Channel: {row['channel'] or '—'}. Model: {row['model'] or '—'}. "
                    f"Latency: {row['latency_ms'] or 0}ms. "
                    f"Error: {error_msg}"
                )

                mid = self.store_with_audit(
                    source=SOURCE_ANALYTICS,
                    title=title,
                    body=body,
                    tier="short",
                    tags=["analytics", "error", skill],
                    importance=0.5,
                    source_id=f"error_{ts}",
                )
                if mid:
                    result.added += 1
                else:
                    result.skipped_dedup += 1
                    result.noops += 1
        except Exception as e:
            logger.warning("Error events consolidation failed: %s", e)
            result.errors += 1

    def _consolidate_credentials(self) -> ConsolidationResult:
        """Sprint 2: Consolidate credential state changes from credential_audit.db.

        Only consolidates significant events: failed, updated, invalidated, preflight_blocked.
        Deduplicates by service_key — only the LATEST event per service is stored.
        NEVER stores credential values (keys/tokens) — only status and sanitized details.
        """
        result = ConsolidationResult(source=SOURCE_CREDENTIALS)
        audit_db = self._data_dir / "credential_audit.db"
        if not audit_db.exists():
            return result

        try:
            import sqlite3 as _sql

            conn = _sql.connect(str(audit_db))
            conn.row_factory = _sql.Row

            # 1. Per-service latest significant events
            self._consolidate_credential_events(conn, result)

            # 2. Service availability summary (WORKING tier)
            self._consolidate_credential_summary(conn, result)

            conn.close()
        except Exception as e:
            logger.warning("Credential consolidation failed: %s", e)
            result.errors += 1
        return result

    def _consolidate_credential_events(self, conn, result: ConsolidationResult) -> None:
        """Consolidate the latest significant event per service into SHORT tier."""
        try:
            # Get the most recent significant event per service_key
            # This avoids storing 495K repeated failures
            rows = conn.execute("""
                SELECT ce.service_key, ce.event_type, ce.timestamp, ce.details, ce.source
                FROM credential_events ce
                INNER JOIN (
                    SELECT service_key, MAX(timestamp) as max_ts
                    FROM credential_events
                    WHERE event_type IN ('failed', 'updated', 'invalidated', 'preflight_blocked')
                    GROUP BY service_key
                ) latest ON ce.service_key = latest.service_key AND ce.timestamp = latest.max_ts
                WHERE ce.event_type IN ('failed', 'updated', 'invalidated', 'preflight_blocked')
                ORDER BY ce.timestamp DESC
            """).fetchall()

            for row in rows:
                svc = row["service_key"]
                event_type = row["event_type"]
                ts = row["timestamp"] or ""
                details = row["details"] or ""
                source = row["source"] or ""

                # Get display name from registry if available
                display_name = svc.replace("_", " ").title()
                try:
                    from familiar.core.credential_registry import CREDENTIAL_REGISTRY
                    spec = CREDENTIAL_REGISTRY.get(svc)
                    if spec:
                        display_name = spec.display_name
                except ImportError:
                    pass

                importance = 0.6 if event_type == "failed" else 0.4

                title = f"Credential: {display_name} — {event_type}"
                # Sanitize details — strip any potential secret fragments
                safe_details = details[:200] if details else ""
                body = (
                    f"Service: {svc}. Event: {event_type}. "
                    f"Timestamp: {ts}. Source: {source}. "
                    f"Details: {safe_details}"
                )

                mid = self.store_with_audit(
                    source=SOURCE_CREDENTIALS,
                    title=title,
                    body=body,
                    tier="short",
                    tags=["credentials", svc, event_type],
                    importance=importance,
                    source_id=f"cred_{svc}_{ts}",
                )
                if mid:
                    result.added += 1
                else:
                    result.noops += 1
        except Exception as e:
            logger.warning("Credential events consolidation failed: %s", e)
            result.errors += 1

    def _consolidate_credential_summary(self, conn, result: ConsolidationResult) -> None:
        """Consolidate overall credential status summary into WORKING tier."""
        try:
            from familiar.core.credential_registry import CREDENTIAL_REGISTRY, is_configured

            configured = sum(1 for k in CREDENTIAL_REGISTRY if is_configured(k))
            total = len(CREDENTIAL_REGISTRY)

            # Count valid + attention from the status cache
            valid = 0
            attention = 0
            try:
                from familiar.core.credential_health import get_attention_needed, get_all_statuses
                attention_list = get_attention_needed()
                attention = len(attention_list)
                all_statuses = get_all_statuses()
                valid = sum(1 for s in all_statuses if s.valid)
            except Exception:
                pass

            title = f"Service Status: {configured} of {total} configured, {valid} valid, {attention} need attention"
            body = (
                f"Configured: {configured}/{total}. Valid: {valid}. "
                f"Attention needed: {attention}. "
                f"Services needing attention: {', '.join(a.service_key for a in (attention_list if attention else []))}"
            )

            existing = self.find_existing(SOURCE_CREDENTIALS, "Service Status:")
            if existing:
                self.update_with_audit(
                    source=SOURCE_CREDENTIALS,
                    memory_id=existing,
                    title=title,
                    body=body,
                    source_id="cred_summary",
                )
                result.updated += 1
            else:
                mid = self.store_with_audit(
                    source=SOURCE_CREDENTIALS,
                    title=title,
                    body=body,
                    tier="working",
                    tags=["credentials", "status_summary"],
                    importance=0.6,
                    source_id="cred_summary",
                )
                if mid:
                    result.added += 1
                else:
                    result.noops += 1
        except Exception as e:
            logger.warning("Credential summary consolidation failed: %s", e)
            result.errors += 1

    def _consolidate_email(self) -> ConsolidationResult:
        """Sprint 3: Consolidate email engagement and classification data.

        Sources:
        - email_engagement.json → sender profiles (LONG tier)
        - email_sorted_cache.json → sort stats summary (LONG tier)
        - email_training_pairs.json → classification corrections (already in Memory Tree via API)

        PHI note: Email addresses are PII. The PHI gate handles email patterns.
        Sender profiles store the address (necessary for feature) but scrub other PII.
        """
        result = ConsolidationResult(source=SOURCE_EMAIL)
        data_dir = self._data_dir

        # 1. Sender engagement profiles
        self._consolidate_sender_profiles(data_dir, result)

        # 2. Sort category stats
        self._consolidate_sort_stats(data_dir, result)

        # 3. Training pair stats (summary only, not individual pairs)
        self._consolidate_training_stats(data_dir, result)

        return result

    def _consolidate_sender_profiles(self, data_dir: Path, result: ConsolidationResult) -> None:
        """Consolidate email sender engagement profiles into LONG tier."""
        import json

        eng_path = data_dir / "email_engagement.json"
        if not eng_path.exists():
            return

        try:
            data = json.loads(eng_path.read_text(encoding="utf-8"))
            senders = data.get("senders", {})

            # Only consolidate senders with meaningful engagement (score > 1.0)
            for sender_email, stats in senders.items():
                opened = stats.get("opened", 0)
                replied = stats.get("replied", 0)
                total = stats.get("total_received", 0)
                trashed = stats.get("trashed", 0)

                # Compute engagement score (same as email intelligence)
                if total == 0:
                    continue
                score = (opened * 1.0 + replied * 2.0) / max(total, 1)
                if score < 1.0:
                    continue  # Skip low-engagement senders

                # Sender name (use email prefix as display)
                sender_display = sender_email.split("@")[0] if "@" in sender_email else sender_email

                title = f"Email Sender Profile: {sender_display}"
                body = (
                    f"Sender: {sender_email}. "
                    f"Received: {total}. Opened: {opened}. Replied: {replied}. "
                    f"Trashed: {trashed}. Engagement score: {score:.1f}."
                )

                # PHI note: sender email IS PII but is required for the feature.
                # The PHI gate will redact it — we store the redacted version.
                mid = self.store_with_audit(
                    source=SOURCE_EMAIL,
                    title=title,
                    body=body,
                    tier="long",
                    tags=["email", "engagement", "sender_profile"],
                    importance=0.4,
                    source_id=f"sender_{sender_email}",
                    require_encryption=True,  # PHI-sensitive
                )
                if mid:
                    result.added += 1
                else:
                    result.noops += 1

        except Exception as e:
            logger.warning("Sender profile consolidation failed: %s", e)
            result.errors += 1

    def _consolidate_sort_stats(self, data_dir: Path, result: ConsolidationResult) -> None:
        """Consolidate email sort category stats into LONG tier."""
        import json

        cache_path = data_dir / "email_sorted_cache.json"
        if not cache_path.exists():
            return

        try:
            data = json.loads(cache_path.read_text(encoding="utf-8"))
            summary = data.get("summary", {})
            updated_at = data.get("updated_at", "")
            total = summary.get("total", 0)

            if total == 0:
                return

            title = f"Email Sort Stats: {total} sorted"
            body = (
                f"Total sorted: {total}. "
                f"Urgent: {summary.get('urgent', 0)}. "
                f"Action: {summary.get('action', 0)}. "
                f"FYI: {summary.get('fyi', 0)}. "
                f"Updated: {updated_at}."
            )

            existing = self.find_existing(SOURCE_EMAIL, "Email Sort Stats:")
            if existing:
                self.update_with_audit(
                    source=SOURCE_EMAIL,
                    memory_id=existing,
                    title=title,
                    body=body,
                    source_id="sort_stats",
                )
                result.updated += 1
            else:
                mid = self.store_with_audit(
                    source=SOURCE_EMAIL,
                    title=title,
                    body=body,
                    tier="long",
                    tags=["email", "sort_stats"],
                    importance=0.3,
                    source_id="sort_stats",
                )
                if mid:
                    result.added += 1
                else:
                    result.noops += 1

        except Exception as e:
            logger.warning("Sort stats consolidation failed: %s", e)
            result.errors += 1

    def _consolidate_training_stats(self, data_dir: Path, result: ConsolidationResult) -> None:
        """Consolidate email training pair stats (summary only) into LONG tier."""
        import json

        tp_path = data_dir / "email_training_pairs.json"
        if not tp_path.exists():
            return

        try:
            data = json.loads(tp_path.read_text(encoding="utf-8"))
            stats = data.get("stats", {})
            pairs = data.get("pairs", [])
            pair_count = len(pairs)

            if pair_count == 0 and not stats:
                return

            confirmed = stats.get("confirmed", 0)
            corrected = stats.get("corrected", 0)
            high_conf = stats.get("high_confidence", 0)

            title = f"Email Training Data: {pair_count} pairs"
            body = (
                f"Training pairs: {pair_count}. "
                f"Confirmed: {confirmed}. Corrected: {corrected}. "
                f"High confidence: {high_conf}."
            )

            existing = self.find_existing(SOURCE_EMAIL, "Email Training Data:")
            if existing:
                self.update_with_audit(
                    source=SOURCE_EMAIL,
                    memory_id=existing,
                    title=title,
                    body=body,
                    source_id="training_stats",
                )
                result.updated += 1
            else:
                mid = self.store_with_audit(
                    source=SOURCE_EMAIL,
                    title=title,
                    body=body,
                    tier="long",
                    tags=["email", "training", "stats"],
                    importance=0.3,
                    source_id="training_stats",
                )
                if mid:
                    result.added += 1
                else:
                    result.noops += 1

        except Exception as e:
            logger.warning("Training stats consolidation failed: %s", e)
            result.errors += 1

    def _consolidate_workspace(self) -> ConsolidationResult:
        """Sprint 4: Consolidate workspace milestones and activity.

        Sources:
        - Creature config (job class info)
        - nonprofit.json (donors, grants, notes, legislation)
        - Workspace API (sections, cards, activity)
        - Scheduled tasks summary

        PHI gate: CRITICAL for Social Worker job class — case notes,
        client names, assessment scores are PHI. All workspace data
        passes through the PHI scrubber. Social Worker data requires
        encryption.
        """
        result = ConsolidationResult(source=SOURCE_WORKSPACE)
        data_dir = self._data_dir

        # 1. Job class + workspace overview
        self._consolidate_workspace_overview(data_dir, result)

        # 2. Workspace data milestones (nonprofit, etc.)
        self._consolidate_workspace_data(data_dir, result)

        # 3. Scheduled tasks summary
        self._consolidate_scheduled_tasks(data_dir, result)

        return result

    def _consolidate_workspace_overview(self, data_dir: Path, result: ConsolidationResult) -> None:
        """Consolidate workspace overview (job class, section counts) into WORKING tier."""
        import json

        try:
            # Get job class from creature config
            creature_path = data_dir / "creature.json"
            job_class = "helper"
            if creature_path.exists():
                creature = json.loads(creature_path.read_text(encoding="utf-8"))
                job_class = creature.get("job_class", "helper")

            # Count workspace data items
            item_counts: dict[str, int] = {}

            np_path = data_dir / "nonprofit.json"
            if np_path.exists():
                np_data = json.loads(np_path.read_text(encoding="utf-8"))
                for key in ["donors", "grants", "notes", "compliance"]:
                    items = np_data.get(key, [])
                    if items:
                        item_counts[key] = len(items)

            total_items = sum(item_counts.values())
            sections = len(item_counts)

            title = f"Workspace: {job_class.replace('_', ' ').title()} — {sections} sections, {total_items} items"
            body = (
                f"Job class: {job_class}. "
                f"Data sections: {', '.join(f'{k}: {v}' for k, v in item_counts.items()) or 'none'}. "
                f"Total items: {total_items}."
            )

            existing = self.find_existing(SOURCE_WORKSPACE, "Workspace:")
            if existing:
                self.update_with_audit(
                    source=SOURCE_WORKSPACE,
                    memory_id=existing,
                    title=title,
                    body=body,
                    source_id="workspace_overview",
                )
                result.updated += 1
            else:
                mid = self.store_with_audit(
                    source=SOURCE_WORKSPACE,
                    title=title,
                    body=body,
                    tier="working",
                    tags=["workspace", job_class, "overview"],
                    importance=0.5,
                    source_id="workspace_overview",
                )
                if mid:
                    result.added += 1
                else:
                    result.noops += 1

        except Exception as e:
            logger.warning("Workspace overview consolidation failed: %s", e)
            result.errors += 1

    def _consolidate_workspace_data(self, data_dir: Path, result: ConsolidationResult) -> None:
        """Consolidate significant workspace milestones into SHORT/LONG tier.

        Only consolidates status changes and milestones, not routine CRUD.
        PHI scrubbing is CRITICAL for Social Worker data.
        """
        import json

        # Determine if PHI-sensitive job class
        creature_path = data_dir / "creature.json"
        job_class = "helper"
        if creature_path.exists():
            try:
                creature = json.loads(creature_path.read_text(encoding="utf-8"))
                job_class = creature.get("job_class", "helper")
            except Exception:
                pass

        phi_sensitive = job_class in ("social_worker", "nonprofit_director")
        require_enc = phi_sensitive

        np_path = data_dir / "nonprofit.json"
        if not np_path.exists():
            return

        try:
            np_data = json.loads(np_path.read_text(encoding="utf-8"))

            # Grants with status (milestones)
            grants = np_data.get("grants", [])
            for g in grants:
                status = g.get("status", "")
                if not status or status == "prospect":
                    continue  # Only consolidate grants with meaningful status

                title_text = g.get("title", g.get("name", "Unnamed Grant"))
                amount = g.get("amount", g.get("award_amount", ""))
                title = f"Milestone: Grant — {title_text} ({status})"
                body = (
                    f"Grant: {title_text}. Status: {status}. "
                    f"Amount: {amount}. "
                    f"Agency: {g.get('agency', '')}. "
                    f"Close date: {g.get('close_date', '')}."
                )

                mid = self.store_with_audit(
                    source=SOURCE_WORKSPACE,
                    title=title,
                    body=body,
                    tier="short",
                    tags=["workspace", "milestone", "grant", status],
                    importance=0.7,
                    source_id=f"grant_{g.get('id', '')}",
                    require_encryption=require_enc,
                )
                if mid:
                    result.added += 1
                else:
                    result.noops += 1

            # Donors with significant contributions
            donors = np_data.get("donors", [])
            for d in donors:
                total_given = d.get("total_given", d.get("amount", 0))
                if not total_given or float(total_given) < 100:
                    continue  # Only consolidate significant donors

                donor_name = d.get("name", "Anonymous")
                title = f"Milestone: Donor — {donor_name}"
                body = (
                    f"Donor: {donor_name}. "
                    f"Total given: ${total_given}. "
                    f"Category: {d.get('category', '')}. "
                    f"Since: {d.get('first_gift_date', d.get('created_at', ''))}."
                )

                mid = self.store_with_audit(
                    source=SOURCE_WORKSPACE,
                    title=title,
                    body=body,
                    tier="long",
                    tags=["workspace", "milestone", "donor"],
                    importance=0.5,
                    source_id=f"donor_{d.get('id', '')}",
                    require_encryption=require_enc,
                )
                if mid:
                    result.added += 1
                else:
                    result.noops += 1

        except Exception as e:
            logger.warning("Workspace data consolidation failed: %s", e)
            result.errors += 1

    def _consolidate_scheduled_tasks(self, data_dir: Path, result: ConsolidationResult) -> None:
        """Consolidate scheduled tasks summary into LONG tier."""
        import json

        tasks_path = data_dir / "scheduled_tasks.json"
        if not tasks_path.exists():
            return

        try:
            data = json.loads(tasks_path.read_text(encoding="utf-8"))
            tasks = data if isinstance(data, list) else data.get("tasks", [])

            if not tasks:
                return

            # Count by type/status
            total = len(tasks)
            active = sum(1 for t in tasks if t.get("status") in ("active", "pending", None))
            completed = sum(1 for t in tasks if t.get("status") == "completed")

            title = f"Scheduled Tasks: {total} total, {active} active, {completed} completed"
            body = (
                f"Total scheduled tasks: {total}. "
                f"Active: {active}. Completed: {completed}. "
                f"Other: {total - active - completed}."
            )

            existing = self.find_existing(SOURCE_WORKSPACE, "Scheduled Tasks:")
            if existing:
                self.update_with_audit(
                    source=SOURCE_WORKSPACE,
                    memory_id=existing,
                    title=title,
                    body=body,
                    source_id="scheduled_tasks",
                )
                result.updated += 1
            else:
                mid = self.store_with_audit(
                    source=SOURCE_WORKSPACE,
                    title=title,
                    body=body,
                    tier="long",
                    tags=["workspace", "scheduled_tasks"],
                    importance=0.3,
                    source_id="scheduled_tasks",
                )
                if mid:
                    result.added += 1
                else:
                    result.noops += 1

        except Exception as e:
            logger.warning("Scheduled tasks consolidation failed: %s", e)
            result.errors += 1

    # ── Status ──

    def get_status(self) -> ConsolidationStatus:
        """Get current bridge status."""
        return ConsolidationStatus(
            last_run=self._last_run,
            results=self._results,
            encryption_available=self._encryption_available,
            phi_scrubbing_available=self._phi_available,
        )

    def get_log(self, limit: int = 50, source: Optional[str] = None) -> list[dict]:
        """Get recent consolidation log entries."""
        try:
            conn = self._tree._connect()
            try:
                if source:
                    rows = conn.execute(
                        "SELECT * FROM consolidation_log WHERE source = ? ORDER BY timestamp DESC LIMIT ?",
                        (source, limit),
                    ).fetchall()
                else:
                    rows = conn.execute(
                        "SELECT * FROM consolidation_log ORDER BY timestamp DESC LIMIT ?",
                        (limit,),
                    ).fetchall()
                return [dict(r) for r in rows]
            finally:
                conn.close()
        except Exception:
            return []

    def generate_hipaa_audit_report(self) -> dict:
        """Generate a HIPAA compliance audit report for the consolidation pipeline.

        Returns a summary of:
        - Total memories with PHI scrubbing applied
        - PHI detections by type and source
        - Encryption status
        - Last consolidation timestamp per source
        - Failed PHI gates (data blocked from storage)
        - Consolidation operation counts
        """
        try:
            conn = self._tree._connect()
            try:
                # Total operations
                total_ops = conn.execute(
                    "SELECT COUNT(*) FROM consolidation_log"
                ).fetchone()[0]

                # PHI detections total
                phi_total = conn.execute(
                    "SELECT COUNT(*) FROM consolidation_log WHERE phi_detected = 1"
                ).fetchone()[0]

                # PHI detections by source
                phi_by_source = {
                    r[0]: r[1]
                    for r in conn.execute(
                        "SELECT source, COUNT(*) FROM consolidation_log WHERE phi_detected = 1 GROUP BY source"
                    ).fetchall()
                }

                # Operations by action
                actions = {
                    r[0]: r[1]
                    for r in conn.execute(
                        "SELECT action, COUNT(*) FROM consolidation_log GROUP BY action"
                    ).fetchall()
                }

                # Last consolidation per source
                last_per_source = {
                    r[0]: r[1]
                    for r in conn.execute(
                        "SELECT source, MAX(timestamp) FROM consolidation_log GROUP BY source"
                    ).fetchall()
                }

                # Blocked operations (skip_phi, skip_encryption)
                blocked = conn.execute(
                    "SELECT COUNT(*) FROM consolidation_log WHERE action IN ('skip_phi', 'skip_encryption')"
                ).fetchone()[0]

                # Memories with consolidation source tags
                consolidated_count = conn.execute(
                    "SELECT COUNT(DISTINCT memory_id) FROM consolidation_log WHERE memory_id IS NOT NULL"
                ).fetchone()[0]

                return {
                    "report_generated": datetime.now(timezone.utc).isoformat(),
                    "encryption": {
                        "status": "enabled" if self._encryption_available else "disabled",
                        "algorithm": "Fernet (AES-128-CBC + HMAC-SHA256)" if self._encryption_available else "none",
                        "key_set": bool(self._encryption_available),
                    },
                    "phi_scrubbing": {
                        "status": "active" if self._phi_available else "unavailable",
                        "method": "HIPAA Safe Harbor 18 (full skill + regex fallback)",
                        "total_detections": phi_total,
                        "detections_by_source": phi_by_source,
                    },
                    "consolidation": {
                        "total_operations": total_ops,
                        "consolidated_memories": consolidated_count,
                        "operations_by_action": actions,
                        "last_run_per_source": last_per_source,
                        "blocked_operations": blocked,
                    },
                    "compliance": {
                        "soft_delete_only": True,
                        "audit_trail": True,
                        "no_secrets_in_memory": True,
                        "consent_gating": True,
                        "source_tagging": True,
                    },
                }
            finally:
                conn.close()
        except Exception as e:
            return {"error": str(e)}

    def get_phi_stats(self) -> dict:
        """Get PHI detection statistics from the consolidation log."""
        try:
            conn = self._tree._connect()
            try:
                total = conn.execute(
                    "SELECT COUNT(*) FROM consolidation_log"
                ).fetchone()[0]
                phi_count = conn.execute(
                    "SELECT COUNT(*) FROM consolidation_log WHERE phi_detected = 1"
                ).fetchone()[0]
                by_source = conn.execute(
                    "SELECT source, COUNT(*) FROM consolidation_log WHERE phi_detected = 1 GROUP BY source"
                ).fetchall()
                return {
                    "total_operations": total,
                    "phi_detections": phi_count,
                    "by_source": {r[0]: r[1] for r in by_source},
                }
            finally:
                conn.close()
        except Exception:
            return {"total_operations": 0, "phi_detections": 0, "by_source": {}}
