# Familiar - Self-Hosted AI Companion Platform
# Copyright (c) 2026 George Scott Foley
# ORCID: 0009-0006-4957-0540
# Licensed under the MIT License

"""
Co-op LLM Training Corpus Pipeline

Builds a consent-gated, PHI-scrubbed, quality-scored, semantically-
deduplicated training corpus from Familiar markdown memories for
contribution to co-operative LLM training with Dross token attribution.

Five concerns addressed
-----------------------
1. Quality signal      — weighted multi-signal scorer with Dross usage multiplier
2. PHI/PII gate        — HIPAA Safe Harbor 18 scrub before any corpus contribution
3. Frontmatter schema  — training_eligible, consent_version, domain, pair_type,
                         quality, training_exported_at, contributor_hash
4. Consent versioning  — opt-in gate tied to chain-anchored consent version
5. Semantic dedup      — MinHash/shingling (pure stdlib, no heavy deps)

Dependencies: pure stdlib + familiar.skills.phi_detection (also stdlib-only)
"""

from __future__ import annotations

import hashlib
import json
import logging
import os
import re
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Optional

logger = logging.getLogger(__name__)

# ── Current consent version ── bump this when the consent terms change ──
CURRENT_CONSENT_VERSION = "1.0"

# ── MinHash parameters ── 64 hash functions, k=3 word shingles ──
MINHASH_NUM_HASHES = 64
MINHASH_SHINGLE_K = 3
DEDUP_JACCARD_THRESHOLD = 0.85  # above this = consider duplicate

# ── Quality weights ── must sum to 1.0 ──
QUALITY_WEIGHTS: dict[str, float] = {
    "content_length": 0.20,  # normalized at 500 chars
    "structure": 0.15,  # paragraphs, headers, lists
    "has_examples": 0.10,  # code blocks, numbered lists
    "revision_count": 0.15,  # frontmatter `revision` field
    "mesh_usage": 0.25,  # Dross usage events for this memory
    "completeness": 0.15,  # has topic, created_at, body length
}


# ══════════════════════════════════════════════════════════════
# 1. FRONTMATTER SCHEMA
# ══════════════════════════════════════════════════════════════


@dataclass
class TrainingFrontmatter:
    """YAML frontmatter schema for training-eligible markdown memories."""

    # Core memory fields (may already exist)
    topic: str = ""
    created_at: str = ""
    updated_at: str = ""
    revision: int = 1

    # Taxonomy classification (new structured fields)
    domain: str = ""  # e.g. "practical"  — validated against DOMAIN_TAXONOMY
    subdomain: str = ""  # e.g. "cooking"    — validated against domain's subdomains
    instruction_types: list = field(default_factory=list)  # e.g. ["how_to", "troubleshoot"]
    difficulty: str = ""  # beginner | intermediate | expert
    audience: str = ""  # general | professional | specialist | student
    language: str = "en"

    # Explicit training pairs (list of {instruction, response, instruction_type})
    # When present, one CorpusEntry is produced per pair instead of wrapping the body.
    pairs: list = field(default_factory=list)

    # Training-corpus eligibility fields
    training_eligible: bool = False
    consent_version: str = ""  # must match CURRENT_CONSENT_VERSION to export
    pair_type: str = "document"  # document | instruction | preference
    quality: float = 0.0  # 0.0-1.0, computed by corpus pipeline
    training_exported_at: str = ""  # ISO timestamp, set on export
    contributor_hash: str = ""  # anonymized sha256 of agent node_id

    def __post_init__(self) -> None:
        # Coerce string fields in case YAML parsed them as numeric (e.g. consent_version: 1.0)
        self.consent_version = str(self.consent_version) if self.consent_version else ""
        self.topic = str(self.topic) if self.topic else ""
        self.domain = str(self.domain) if self.domain else ""
        self.subdomain = str(self.subdomain) if self.subdomain else ""
        self.pair_type = str(self.pair_type) if self.pair_type else "document"
        self.difficulty = str(self.difficulty) if self.difficulty else ""
        self.audience = str(self.audience) if self.audience else ""
        self.language = str(self.language) if self.language else "en"
        # Ensure list fields are actually lists
        if not isinstance(self.instruction_types, list):
            self.instruction_types = []
        if not isinstance(self.pairs, list):
            self.pairs = []

    @classmethod
    def from_dict(cls, d: dict) -> "TrainingFrontmatter":
        known = {f.name for f in cls.__dataclass_fields__.values()}  # type: ignore[attr-defined]
        return cls(**{k: v for k, v in d.items() if k in known})

    def to_dict(self) -> dict:
        return asdict(self)


def parse_frontmatter(text: str) -> tuple[TrainingFrontmatter, str]:
    """
    Parse YAML frontmatter from a markdown file.

    Returns (TrainingFrontmatter, body_without_frontmatter).
    If no frontmatter block exists, returns defaults + full text as body.
    """
    if not text.startswith("---"):
        return TrainingFrontmatter(), text

    lines = text.split("\n")
    end_idx = None
    for i, line in enumerate(lines[1:], 1):
        if line.strip() == "---":
            end_idx = i
            break

    if end_idx is None:
        return TrainingFrontmatter(), text

    yaml_block = "\n".join(lines[1:end_idx])
    body = "\n".join(lines[end_idx + 1 :]).lstrip("\n")

    raw: dict = {}
    try:
        import yaml  # optional — pyyaml in base deps

        raw = yaml.safe_load(yaml_block) or {}
    except ImportError:
        # Minimal YAML parser: handles simple key:value scalars only.
        # List fields (pairs, instruction_types) require pyyaml — install pyyaml
        # for full training document support.
        for line in yaml_block.splitlines():
            line = line.strip()
            if not line or line.startswith("#") or ":" not in line:
                continue
            k, _, v = line.partition(":")
            k = k.strip()
            v = v.strip().strip('"').strip("'")
            if v.lower() == "true":
                raw[k] = True
            elif v.lower() == "false":
                raw[k] = False
            elif re.match(r"^-?\d+$", v):
                raw[k] = int(v)
            elif re.match(r"^-?\d+\.\d+$", v):
                raw[k] = float(v)
            elif v:
                raw[k] = v
    except Exception as e:
        logger.debug("Frontmatter YAML parse error: %s", e)

    return TrainingFrontmatter.from_dict(raw), body


def write_frontmatter(fm: TrainingFrontmatter, body: str) -> str:
    """Render a TrainingFrontmatter + body back to markdown string."""
    lines = ["---"]
    d = fm.to_dict()
    # Skip empty lists and empty strings except for always-written keys
    always_write = {"training_eligible"}
    for k, v in d.items():
        if k in always_write:
            lines.append(f"{k}: {str(v).lower()}")
        elif isinstance(v, bool):
            lines.append(f"{k}: {str(v).lower()}")
        elif isinstance(v, list):
            if v:  # only write non-empty lists
                lines.append(f"{k}:")
                for item in v:
                    if isinstance(item, dict):
                        first = True
                        for ik, iv in item.items():
                            prefix = "  - " if first else "    "
                            first = False
                            iv_str = str(iv).replace('"', '\\"')
                            lines.append(f'{prefix}{ik}: "{iv_str}"')
                    else:
                        lines.append(f'  - "{item}"')
        elif isinstance(v, str) and v:
            lines.append(f'{k}: "{v}"')
        elif isinstance(v, int) and v:
            lines.append(f"{k}: {v}")
        elif isinstance(v, float) and v:
            lines.append(f"{k}: {v}")
    lines.append("---")
    lines.append("")
    lines.append(body)
    return "\n".join(lines)


# ══════════════════════════════════════════════════════════════
# 2. PHI / PII GATE
# ══════════════════════════════════════════════════════════════


def phi_scrub(text: str) -> tuple[str, bool]:
    """
    Scrub PHI/PII from text using the full HIPAA Safe Harbor 18 detector.

    Returns (scrubbed_text, phi_was_found).
    Falls back to lightweight regex if phi_detection skill unavailable.
    """
    try:
        from familiar.skills.phi_detection.skill import (
            detect_phi,
            redact_phi,
        )

        result = detect_phi({"text": text, "sensitivity": "standard"})
        phi_found = result.get("phi_detected", False)
        if phi_found:
            scrubbed = redact_phi({"text": text})
            if isinstance(scrubbed, dict):
                scrubbed = scrubbed.get("redacted_text", text)
            return str(scrubbed), True
        return text, False
    except (ImportError, Exception) as e:
        logger.debug("PHI detection skill unavailable, using fallback: %s", e)
        return _phi_scrub_fallback(text)


_PHI_FALLBACK_PATTERNS = [
    (re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"), "[EMAIL]"),
    (re.compile(r"\b\d{3}[-.\s]?\d{3}[-.\s]?\d{4}\b"), "[PHONE]"),
    (re.compile(r"\b\d{3}[-\s]?\d{2}[-\s]?\d{4}\b"), "[SSN]"),
    (re.compile(r"\b(?:\d{4}[-\s]?){3}\d{4}\b"), "[CARD]"),
    (re.compile(r"\b\d{1,2}/\d{1,2}/\d{2,4}\b"), "[DATE]"),
]


def _phi_scrub_fallback(text: str) -> tuple[str, bool]:
    scrubbed = text
    found = False
    for pat, placeholder in _PHI_FALLBACK_PATTERNS:
        if pat.search(scrubbed):
            scrubbed = pat.sub(placeholder, scrubbed)
            found = True
    return scrubbed, found


# ══════════════════════════════════════════════════════════════
# 3. QUALITY SCORER
# ══════════════════════════════════════════════════════════════


def score_memory(
    content: str,
    frontmatter: TrainingFrontmatter,
    usage_count: int = 0,
    dross_earned: float = 0.0,
) -> float:
    """
    Compute a weighted quality score [0.0, 1.0] for a memory.

    Weights (QUALITY_WEIGHTS):
      content_length  0.20 — normalized at 500 chars
      structure       0.15 — paragraphs, headers, lists
      has_examples    0.10 — code blocks, numbered lists
      revision_count  0.15 — number of edits (frontmatter.revision)
      mesh_usage      0.25 — Dross usage events for this memory
      completeness    0.15 — has topic, created_at, non-empty body
    """
    scores: dict[str, float] = {}

    # --- content_length (normalized at 500 chars) ---
    length = len(content)
    scores["content_length"] = min(length / 500.0, 1.0)

    # --- structure: headers, multi-paragraph, lists ---
    structural_signals = 0
    if re.search(r"^#{1,3} ", content, re.MULTILINE):
        structural_signals += 1
    if len([p for p in content.split("\n\n") if p.strip()]) >= 2:
        structural_signals += 1
    if re.search(r"^[-*+] |\n\d+\. ", content, re.MULTILINE):
        structural_signals += 1
    scores["structure"] = min(structural_signals / 3.0, 1.0)

    # --- has_examples: code blocks or numbered steps ---
    scores["has_examples"] = 1.0 if re.search(r"```|^\d+\. ", content, re.MULTILINE) else 0.0

    # --- revision_count: each revision above 1 adds quality ---
    rev = max(frontmatter.revision - 1, 0)
    scores["revision_count"] = min(rev / 5.0, 1.0)  # saturates at 5 revisions

    # --- mesh_usage: Dross earned by peers using this memory ---
    # usage_count and dross_earned are queried from UsageTracker
    # Scale: 0 uses = 0.0, 5+ uses = 0.5, 10+ uses = 0.8, Dross > 1 = bonus
    usage_score = min(usage_count / 10.0, 0.8)
    dross_bonus = min(dross_earned / 2.0, 0.2)
    scores["mesh_usage"] = min(usage_score + dross_bonus, 1.0)

    # --- completeness: topic, created_at, body length >= 100 chars ---
    complete_signals = 0
    if frontmatter.topic:
        complete_signals += 1
    if frontmatter.created_at:
        complete_signals += 1
    if len(content) >= 100:
        complete_signals += 1
    scores["completeness"] = complete_signals / 3.0

    # Weighted sum
    total = sum(QUALITY_WEIGHTS[k] * scores[k] for k in QUALITY_WEIGHTS)
    return round(min(max(total, 0.0), 1.0), 4)


# ══════════════════════════════════════════════════════════════
# 4. SEMANTIC DEDUPLICATOR (MinHash / shingling, pure stdlib)
# ══════════════════════════════════════════════════════════════


def _shingles(text: str, k: int = MINHASH_SHINGLE_K) -> set[str]:
    """Generate k-gram word shingles from text."""
    words = text.lower().split()
    if len(words) < k:
        return {" ".join(words)} if words else set()
    return {" ".join(words[i : i + k]) for i in range(len(words) - k + 1)}


def _minhash_signature(shingle_set: set[str], num_hashes: int = MINHASH_NUM_HASHES) -> list[int]:
    """Compute MinHash signature vector for a shingle set."""
    if not shingle_set:
        return [0] * num_hashes
    sig = []
    for seed in range(num_hashes):
        min_val = min(int(hashlib.md5(f"{seed}:{s}".encode()).hexdigest(), 16) for s in shingle_set)
        sig.append(min_val)
    return sig


def _jaccard_estimate(sig_a: list[int], sig_b: list[int]) -> float:
    """Estimate Jaccard similarity from two MinHash signatures."""
    if not sig_a or not sig_b or len(sig_a) != len(sig_b):
        return 0.0
    matches = sum(a == b for a, b in zip(sig_a, sig_b))
    return matches / len(sig_a)


class SemanticDeduplicator:
    """
    In-memory MinHash deduplicator.

    Maintains a list of (id, signature) pairs. For each new document,
    compute its signature and check against all stored signatures.
    If estimated Jaccard similarity >= DEDUP_JACCARD_THRESHOLD, flag as duplicate.

    For large corpora (>10K docs) a proper LSH index would be faster,
    but this is sufficient for typical Familiar memory stores (100s–low 1000s).
    """

    def __init__(self, threshold: float = DEDUP_JACCARD_THRESHOLD) -> None:
        self.threshold = threshold
        self._signatures: list[tuple[str, list[int]]] = []  # (id, sig)

    def is_duplicate(self, doc_id: str, text: str) -> Optional[str]:
        """
        Check if text is a near-duplicate of an already-seen document.

        Returns the id of the first duplicate found, or None if unique.
        Call add() separately to register new documents.
        """
        sig = _minhash_signature(_shingles(text))
        for seen_id, seen_sig in self._signatures:
            if seen_id == doc_id:
                continue
            if _jaccard_estimate(sig, seen_sig) >= self.threshold:
                return seen_id
        return None

    def add(self, doc_id: str, text: str) -> None:
        """Register a document's signature."""
        sig = _minhash_signature(_shingles(text))
        self._signatures.append((doc_id, sig))

    def size(self) -> int:
        return len(self._signatures)


# ══════════════════════════════════════════════════════════════
# 5. CONSENT REGISTRY
# ══════════════════════════════════════════════════════════════


def _get_training_dir() -> Path:
    try:
        from familiar.core import paths

        d = paths.DATA_DIR / "training"
    except ImportError:
        d = Path.home() / ".familiar" / "data" / "training"
    d.mkdir(parents=True, exist_ok=True)
    return d


def _get_corpus_dir() -> Path:
    d = _get_training_dir() / "corpus"
    d.mkdir(parents=True, exist_ok=True)
    return d


def _consent_registry_path() -> Path:
    return _get_training_dir() / "corpus_consent.json"


def load_consent_registry() -> dict:
    """Load the consent registry (maps user_id → consent record)."""
    path = _consent_registry_path()
    if path.exists():
        try:
            with open(path, encoding="utf-8") as f:
                return json.load(f)
        except (json.JSONDecodeError, OSError):
            pass
    return {}


def _save_consent_registry(registry: dict) -> None:
    path = _consent_registry_path()
    path.parent.mkdir(parents=True, exist_ok=True)
    with open(path, "w", encoding="utf-8") as f:
        json.dump(registry, f, indent=2)
    os.chmod(path, 0o600)


def grant_corpus_consent(user_id: str, consent_version: str = CURRENT_CONSENT_VERSION) -> str:
    """
    Grant corpus contribution consent for a user.

    Anchors the consent record to the genesis chain (if available).
    Returns the chain block hash or '' if chain unavailable.
    """
    registry = load_consent_registry()
    now = datetime.now(timezone.utc).isoformat()

    block_hash = ""
    try:
        from familiar.skills.training.bridge import anchor_consent

        block_hash = anchor_consent(
            {
                "user_id": user_id,
                "consent_version": consent_version,
                "action": "grant_corpus",
                "timestamp": now,
            }
        )
    except Exception as e:
        logger.debug("Chain anchor for corpus consent failed: %s", e)

    registry[user_id] = {
        "consent_version": consent_version,
        "granted_at": now,
        "chain_block_hash": block_hash,
        "revoked": False,
    }
    _save_consent_registry(registry)
    logger.info("Corpus consent granted for %s (v%s)", user_id, consent_version)
    return block_hash


def revoke_corpus_consent(user_id: str) -> None:
    """Revoke corpus contribution consent for a user."""
    registry = load_consent_registry()
    now = datetime.now(timezone.utc).isoformat()

    if user_id in registry:
        registry[user_id]["revoked"] = True
        registry[user_id]["revoked_at"] = now
    else:
        registry[user_id] = {"revoked": True, "revoked_at": now}

    _save_consent_registry(registry)
    logger.info("Corpus consent revoked for %s", user_id)


def check_corpus_consent(user_id: str) -> bool:
    """Return True if user has active consent for the current version."""
    registry = load_consent_registry()
    rec = registry.get(user_id, {})
    if rec.get("revoked"):
        return False
    return rec.get("consent_version") == CURRENT_CONSENT_VERSION


# ══════════════════════════════════════════════════════════════
# 6. CORPUS ENTRY
# ══════════════════════════════════════════════════════════════


@dataclass
class CorpusEntry:
    """A single training-corpus entry produced by the pipeline."""

    id: str  # sha256 of contributor_hash + topic + content
    source_type: str = "corpus_memory"
    instruction: str = ""
    response: str = ""
    instruction_type: str = "how_to"  # from INSTRUCTION_TYPES taxonomy
    domain: str = ""
    subdomain: str = ""
    pair_type: str = "document"
    quality_score: float = 0.0
    quality_signals: dict = field(default_factory=dict)
    phi_scrubbed: bool = False
    consent_version: str = ""
    contributor_hash: str = ""
    provenance: dict = field(default_factory=dict)
    created_at: str = ""

    def to_dict(self) -> dict:
        return asdict(self)


# ══════════════════════════════════════════════════════════════
# 7. CORPUS PIPELINE
# ══════════════════════════════════════════════════════════════


class CorpusPipeline:
    """
    Orchestrates the full co-op training corpus pipeline for markdown memories.

    Steps per memory file:
      1. Parse frontmatter → check training_eligible + consent_version
      2. PHI gate          → scrub; reject if PHI found and not scrubable
      3. Score quality     → weighted multi-signal score with Dross weighting
      4. Semantic dedup    → MinHash; skip near-duplicates
      5. Write CorpusEntry → curated/corpus_memories.jsonl

    Call run() to execute.  Returns a summary dict.
    """

    def __init__(
        self,
        memories_dir: Optional[Path] = None,
        user_id: str = "default",
        min_quality: float = 0.5,
        reject_phi: bool = True,
        usage_tracker: Optional[object] = None,
    ) -> None:
        self.memories_dir = memories_dir or self._default_memories_dir()
        self.user_id = user_id
        self.min_quality = min_quality
        self.reject_phi = reject_phi
        self._usage_tracker = usage_tracker
        self._dedup = SemanticDeduplicator()

    @staticmethod
    def _default_memories_dir() -> Path:
        try:
            from familiar.core.paths import MEMORIES_DIR

            return MEMORIES_DIR
        except ImportError:
            return Path.home() / ".familiar" / "memories"

    def run(self) -> dict:
        """
        Execute the full corpus pipeline.

        Returns a summary dict with keys:
          total_scanned, eligible, phi_rejected, dedup_skipped,
          quality_rejected, exported, errors, output_path
        """
        summary: dict = {
            "total_scanned": 0,
            "eligible": 0,
            "consent_rejected": 0,
            "phi_rejected": 0,
            "phi_rejection_files": [],  # filenames excluded due to PHI (dashboard transparency)
            "dedup_skipped": 0,
            "quality_rejected": 0,
            "exported": 0,
            "errors": 0,
            "output_path": "",
        }

        if not check_corpus_consent(self.user_id):
            return {
                **summary,
                "error": (
                    f"No active corpus consent for user '{self.user_id}' "
                    f"(version {CURRENT_CONSENT_VERSION}). "
                    "Call grant_corpus_consent() or the manage_consent tool."
                ),
            }

        if not self.memories_dir.exists():
            return {**summary, "error": f"Memories directory not found: {self.memories_dir}"}

        # Include all .md files but skip _README.md scaffolding files
        md_files = [p for p in self.memories_dir.rglob("*.md") if not p.name.startswith("_")]
        summary["total_scanned"] = len(md_files)

        entries: list[CorpusEntry] = []

        for md_path in md_files:
            try:
                new_entries = self._process_file(md_path, summary)
                entries.extend(new_entries)
            except Exception as e:
                logger.warning("Corpus pipeline error on %s: %s", md_path.name, e)
                summary["errors"] += 1

        # Write output
        if entries:
            output_path = _get_corpus_dir() / "corpus_memories.jsonl"
            output_path.parent.mkdir(parents=True, exist_ok=True)
            with open(output_path, "a", encoding="utf-8") as f:
                for entry in entries:
                    f.write(json.dumps(entry.to_dict(), default=str) + "\n")
            summary["output_path"] = str(output_path)

        summary["exported"] = len(entries)
        logger.info(
            "Corpus pipeline: scanned=%d eligible=%d exported=%d phi_rejected=%d dedup=%d",
            summary["total_scanned"],
            summary["eligible"],
            summary["exported"],
            summary["phi_rejected"],
            summary["dedup_skipped"],
        )
        return summary

    def _process_file(self, md_path: Path, summary: dict) -> list[CorpusEntry]:
        text = md_path.read_text(encoding="utf-8")
        fm, body = parse_frontmatter(text)

        # Step 1 — eligibility gate
        if not fm.training_eligible:
            return []
        if fm.consent_version != CURRENT_CONSENT_VERSION:
            summary["consent_rejected"] = summary.get("consent_rejected", 0) + 1
            return []

        # Taxonomy validation — log only, don't reject (legacy free-text domains allowed)
        if fm.domain:
            try:
                from familiar.skills.training.taxonomy import validate_domain

                valid, msg = validate_domain(fm.domain, fm.subdomain)
                if not valid:
                    logger.debug("Taxonomy mismatch in %s: %s", md_path.name, msg)
            except ImportError:
                pass

        summary["eligible"] += 1

        # Step 2 — Quality scoring (on body text as signal source)
        usage_count, dross_earned = self._get_usage_stats(md_path.stem, fm)
        quality = score_memory(body, fm, usage_count=usage_count, dross_earned=dross_earned)

        if quality < self.min_quality:
            summary["quality_rejected"] += 1
            return []

        # Step 3 — Route to per-pair or legacy body-wrap
        if fm.pairs:
            return self._entries_from_pairs(
                md_path, fm, body, quality, usage_count, dross_earned, summary
            )
        return self._entry_from_body(md_path, fm, body, quality, usage_count, dross_earned, summary)

    def _entries_from_pairs(
        self,
        md_path: Path,
        fm: "TrainingFrontmatter",
        body: str,
        quality: float,
        usage_count: int,
        dross_earned: float,
        summary: dict,
    ) -> list[CorpusEntry]:
        """Produce one CorpusEntry per explicit pair in the frontmatter."""
        entries: list[CorpusEntry] = []
        contributor_hash = self._contributor_hash()

        for pair in fm.pairs:
            if not isinstance(pair, dict):
                continue
            instruction = str(pair.get("instruction", "")).strip()
            response = str(pair.get("response", "")).strip()
            itype = str(pair.get("instruction_type", "how_to"))

            if not instruction or not response:
                continue

            # PHI gate per pair (check combined to catch cross-field leakage)
            combined = f"{instruction}\n{response}"
            _, phi_found = phi_scrub(combined)
            if phi_found:
                if self.reject_phi:
                    summary["phi_rejected"] += 1
                    summary.setdefault("phi_rejection_files", []).append(md_path.name)
                    logger.info("PHI in pair from %s — excluded", md_path.name)
                    continue
                scrubbed_instruction, _ = phi_scrub(instruction)
                scrubbed_response, _ = phi_scrub(response)
                instruction = scrubbed_instruction
                response = scrubbed_response

            # Semantic dedup on response content
            doc_id = hashlib.sha256(
                (fm.topic + instruction[:100] + response[:200]).encode()
            ).hexdigest()[:16]
            dup_of = self._dedup.is_duplicate(doc_id, response)
            if dup_of:
                summary["dedup_skipped"] += 1
                logger.debug("Dedup: pair from %s near-dup of %s", md_path.name, dup_of)
                continue
            self._dedup.add(doc_id, response)

            entry_id = hashlib.sha256(
                (contributor_hash + fm.topic + instruction[:100]).encode()
            ).hexdigest()[:16]

            entries.append(
                CorpusEntry(
                    id=entry_id,
                    instruction=instruction,
                    response=response,
                    instruction_type=itype,
                    domain=fm.domain or "general",
                    subdomain=fm.subdomain,
                    pair_type="instruction",
                    quality_score=quality,
                    quality_signals={
                        "usage_count": usage_count,
                        "dross_earned": dross_earned,
                        "phi_scrubbed": phi_found,
                        "revision": fm.revision,
                        "explicit_pair": True,
                    },
                    phi_scrubbed=phi_found,
                    consent_version=fm.consent_version,
                    contributor_hash=contributor_hash,
                    provenance={
                        "source": "training_document",
                        "file": md_path.name,
                        "topic": fm.topic,
                        "domain": fm.domain,
                        "subdomain": fm.subdomain,
                        "instruction_type": itype,
                    },
                    created_at=fm.created_at or datetime.now(timezone.utc).isoformat(),
                )
            )

        return entries

    def _entry_from_body(
        self,
        md_path: Path,
        fm: "TrainingFrontmatter",
        body: str,
        quality: float,
        usage_count: int,
        dross_earned: float,
        summary: dict,
    ) -> list[CorpusEntry]:
        """Legacy fallback: wrap the body as a single generic Q&A entry."""
        # PHI gate on full body
        scrubbed_body, phi_found = phi_scrub(body)
        if phi_found:
            if self.reject_phi:
                summary["phi_rejected"] += 1
                summary.setdefault("phi_rejection_files", []).append(md_path.name)
                logger.info("PHI detected in %s — excluded from corpus", md_path.name)
                return []
            body = scrubbed_body

        # Apply 10% quality penalty — unstructured docs are lower-value training data
        adjusted_quality = round(quality * 0.9, 4)
        if adjusted_quality < self.min_quality:
            summary["quality_rejected"] += 1
            return []

        # Semantic dedup
        doc_id = hashlib.sha256((fm.topic + body[:500]).encode()).hexdigest()[:16]
        dup_of = self._dedup.is_duplicate(doc_id, body)
        if dup_of:
            summary["dedup_skipped"] += 1
            logger.debug("Dedup: %s is near-duplicate of %s", md_path.name, dup_of)
            return []
        self._dedup.add(doc_id, body)

        topic = fm.topic or md_path.stem.replace("-", " ")
        contributor_hash = self._contributor_hash()
        entry_id = hashlib.sha256((contributor_hash + topic + body[:200]).encode()).hexdigest()[:16]

        return [
            CorpusEntry(
                id=entry_id,
                instruction=f"What do you know about {topic}?",
                response=body,
                instruction_type="qa",
                domain=fm.domain or "general",
                subdomain=fm.subdomain,
                pair_type=fm.pair_type,
                quality_score=adjusted_quality,
                quality_signals={
                    "usage_count": usage_count,
                    "dross_earned": dross_earned,
                    "phi_scrubbed": phi_found,
                    "revision": fm.revision,
                    "explicit_pair": False,
                },
                phi_scrubbed=phi_found,
                consent_version=fm.consent_version,
                contributor_hash=contributor_hash,
                provenance={
                    "source": "markdown_memory",
                    "file": md_path.name,
                    "topic": topic,
                    "domain": fm.domain,
                },
                created_at=fm.created_at or datetime.now(timezone.utc).isoformat(),
            )
        ]

    def _get_usage_stats(self, memory_key: str, fm: TrainingFrontmatter) -> tuple[int, float]:
        """Query usage tracker for Dross earned by this memory. Returns (count, dross)."""
        if self._usage_tracker is None:
            return 0, 0.0
        try:
            records = self._usage_tracker.get_usage_as_provider(limit=500)
            relevant = [r for r in records if r.get("resource_id") == memory_key]
            count = sum(r.get("usage_count", 1) for r in relevant)
            dross = sum(r.get("dross_amount", 0.0) for r in relevant)
            return count, round(dross, 4)
        except Exception as e:
            logger.debug("Usage tracker query failed: %s", e)
            return 0, 0.0

    def _contributor_hash(self) -> str:
        """Return anonymized sha256 of the user/node identity."""
        try:
            from familiar.core.paths import DATA_DIR

            node_file = DATA_DIR / "genesis_id.txt"
            if node_file.exists():
                node_id = node_file.read_text().strip()
                return hashlib.sha256(node_id.encode()).hexdigest()[:16]
        except Exception:
            pass
        return hashlib.sha256(self.user_id.encode()).hexdigest()[:16]


# ══════════════════════════════════════════════════════════════
# 8. PIPELINE STATUS
# ══════════════════════════════════════════════════════════════


def _get_corpus_run_path() -> Path:
    return _get_corpus_dir() / "corpus_runs.json"


def record_corpus_run(summary: dict) -> None:
    """Append a corpus run summary to the run history."""
    path = _get_corpus_run_path()
    runs: list = []
    if path.exists():
        try:
            with open(path, encoding="utf-8") as f:
                runs = json.load(f)
        except (json.JSONDecodeError, OSError):
            pass

    runs.append({**summary, "timestamp": datetime.now(timezone.utc).isoformat()})
    if len(runs) > 50:
        runs = runs[-50:]

    with open(path, "w", encoding="utf-8") as f:
        json.dump(runs, f, indent=2, default=str)


def get_corpus_status() -> dict:
    """Return corpus pipeline status: latest run, history, output stats."""
    path = _get_corpus_run_path()
    runs: list = []
    if path.exists():
        try:
            with open(path, encoding="utf-8") as f:
                runs = json.load(f)
        except (json.JSONDecodeError, OSError):
            pass

    # Count exported entries
    output_path = _get_corpus_dir() / "corpus_memories.jsonl"
    exported_count = 0
    if output_path.exists():
        with open(output_path, encoding="utf-8") as f:
            exported_count = sum(1 for line in f if line.strip())

    return {
        "total_runs": len(runs),
        "latest_run": runs[-1] if runs else None,
        "total_exported_entries": exported_count,
        "consent_version": CURRENT_CONSENT_VERSION,
        "output_path": str(output_path) if output_path.exists() else None,
        "history": [
            {
                "timestamp": r.get("timestamp"),
                "exported": r.get("exported", 0),
                "phi_rejected": r.get("phi_rejected", 0),
                "dedup_skipped": r.get("dedup_skipped", 0),
            }
            for r in runs[-10:]
        ],
    }


# ══════════════════════════════════════════════════════════════
# 9. PUBLIC API — called from skill.py handlers
# ══════════════════════════════════════════════════════════════


def run_corpus_pipeline(data: dict) -> str:
    """
    Tool handler: Run the co-op corpus pipeline on markdown memories.

    Params: user_id, min_quality (0.0-1.0), reject_phi (bool).
    """
    user_id = data.get("user_id", "default")
    min_quality = float(data.get("min_quality", 0.5))
    reject_phi = bool(data.get("reject_phi", True))

    pipeline = CorpusPipeline(user_id=user_id, min_quality=min_quality, reject_phi=reject_phi)
    summary = pipeline.run()
    record_corpus_run(summary)

    if "error" in summary:
        return f"Corpus pipeline failed: {summary['error']}"

    lines = [
        "Co-op Corpus Pipeline Complete",
        f"  Scanned:          {summary['total_scanned']} memory files",
        f"  Eligible:         {summary['eligible']} (training_eligible=true)",
        f"  PHI rejected:     {summary['phi_rejected']}",
        f"  Dedup skipped:    {summary['dedup_skipped']}",
        f"  Quality rejected: {summary['quality_rejected']}",
        f"  Exported:         {summary['exported']} entries",
        f"  Errors:           {summary['errors']}",
    ]
    if summary.get("output_path"):
        lines.append(f"  Output:           {summary['output_path']}")
    return "\n".join(lines)


def mark_memory_training_eligible(data: dict) -> str:
    """
    Tool handler: Set training_eligible=true and consent_version on a memory file.

    Params: file_path (str), domain (str), pair_type (str).
    """
    file_path_str = data.get("file_path", "")
    if not file_path_str:
        return "Error: file_path is required."

    file_path = Path(file_path_str).expanduser()
    if not file_path.exists():
        return f"Error: file not found: {file_path}"

    text = file_path.read_text(encoding="utf-8")
    fm, body = parse_frontmatter(text)

    fm.training_eligible = True
    fm.consent_version = CURRENT_CONSENT_VERSION
    if data.get("domain"):
        fm.domain = data["domain"]
    if data.get("pair_type"):
        fm.pair_type = data["pair_type"]
    fm.updated_at = datetime.now(timezone.utc).isoformat()

    file_path.write_text(write_frontmatter(fm, body), encoding="utf-8")
    return (
        f"Marked {file_path.name} as training-eligible "
        f"(consent_version={CURRENT_CONSENT_VERSION}, domain={fm.domain or 'general'})."
    )


def corpus_status(data: dict) -> str:
    """Tool handler: Show corpus pipeline status."""
    status = get_corpus_status()
    lines = [
        "Co-op Corpus Pipeline Status",
        f"  Consent version:  {status['consent_version']}",
        f"  Total runs:       {status['total_runs']}",
        f"  Exported entries: {status['total_exported_entries']}",
    ]
    if status["latest_run"]:
        lr = status["latest_run"]
        lines += [
            "  Latest run:",
            f"    Timestamp:   {lr.get('timestamp', '')}",
            f"    Scanned:     {lr.get('total_scanned', 0)}",
            f"    Exported:    {lr.get('exported', 0)}",
            f"    PHI rej.:    {lr.get('phi_rejected', 0)}",
            f"    Dedup skip.: {lr.get('dedup_skipped', 0)}",
        ]
    if status.get("output_path"):
        lines.append(f"  Output: {status['output_path']}")
    return "\n".join(lines)


# ══════════════════════════════════════════════════════════════
# 10. DASHBOARD DATA FUNCTIONS
# ══════════════════════════════════════════════════════════════


def list_memories(memories_dir: Optional[Path] = None) -> list[dict]:
    """
    Return metadata for all markdown memory files (for dashboard memory browser).

    Each dict: filename, path, topic, training_eligible, consent_version,
                domain, pair_type, quality, updated_at, body_length.
    """
    if memories_dir is None:
        memories_dir = CorpusPipeline._default_memories_dir()
    if not memories_dir.exists():
        return []

    result = []
    for md_path in sorted(memories_dir.rglob("*.md")):
        if md_path.name.startswith("_"):
            continue  # skip _README.md scaffold files
        try:
            text = md_path.read_text(encoding="utf-8")
            fm, body = parse_frontmatter(text)
            result.append(
                {
                    "filename": md_path.name,
                    "path": str(md_path),
                    "topic": fm.topic or md_path.stem.replace("-", " "),
                    "training_eligible": fm.training_eligible,
                    "consent_version": fm.consent_version,
                    "domain": fm.domain,
                    "subdomain": fm.subdomain,
                    "pair_count": len(fm.pairs),
                    "pair_type": fm.pair_type,
                    "quality": fm.quality,
                    "updated_at": fm.updated_at,
                    "body_length": len(body),
                }
            )
        except Exception as e:
            logger.debug("Could not read %s: %s", md_path.name, e)
    return result


def list_entries(limit: int = 20, offset: int = 0) -> dict:
    """
    Return paginated corpus entries from corpus_memories.jsonl.

    Returns dict: {entries: [...], total: int}
    """
    output_path = _get_corpus_dir() / "corpus_memories.jsonl"
    if not output_path.exists():
        return {"entries": [], "total": 0}

    all_entries: list[dict] = []
    with open(output_path, encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if line:
                try:
                    all_entries.append(json.loads(line))
                except json.JSONDecodeError:
                    pass

    total = len(all_entries)
    page = all_entries[offset : offset + limit]
    return {"entries": page, "total": total}


def delete_entry(entry_id: str) -> bool:
    """
    Remove a specific entry from corpus_memories.jsonl by id.

    Rewrites the file without the matched entry. Returns True if found and removed.
    """
    output_path = _get_corpus_dir() / "corpus_memories.jsonl"
    if not output_path.exists():
        return False

    kept: list[str] = []
    found = False
    with open(output_path, encoding="utf-8") as f:
        for line in f:
            stripped = line.strip()
            if not stripped:
                continue
            try:
                entry = json.loads(stripped)
                if entry.get("id") == entry_id:
                    found = True
                else:
                    kept.append(stripped)
            except json.JSONDecodeError:
                kept.append(stripped)

    if found:
        with open(output_path, "w", encoding="utf-8") as f:
            for line in kept:
                f.write(line + "\n")
    return found


def toggle_memory_eligible(filename: str, eligible: bool) -> bool:
    """
    Set training_eligible on a specific memory file by filename.

    Searches MEMORIES_DIR for the file and updates its frontmatter in-place.
    Returns True if the file was found and updated.
    """
    memories_dir = CorpusPipeline._default_memories_dir()
    md_path = memories_dir / filename
    if not md_path.exists():
        # Try recursive search
        matches = list(memories_dir.rglob(filename))
        if not matches:
            return False
        md_path = matches[0]

    text = md_path.read_text(encoding="utf-8")
    fm, body = parse_frontmatter(text)
    fm.training_eligible = eligible
    if eligible and not fm.consent_version:
        fm.consent_version = CURRENT_CONSENT_VERSION
    fm.updated_at = datetime.now(timezone.utc).isoformat()
    md_path.write_text(write_frontmatter(fm, body), encoding="utf-8")
    return True


def manage_corpus_consent(data: dict) -> str:
    """Tool handler: Grant or revoke corpus contribution consent."""
    action = data.get("action", "grant")
    user_id = data.get("user_id", "default")

    if action == "grant":
        block_hash = grant_corpus_consent(user_id)
        chain_note = f" (chain: {block_hash[:8]}...)" if block_hash else ""
        return (
            f"Corpus consent granted for '{user_id}' "
            f"(version {CURRENT_CONSENT_VERSION}){chain_note}. "
            "Memories marked with training_eligible=true will now be included "
            "in co-op training corpus exports."
        )
    elif action == "revoke":
        revoke_corpus_consent(user_id)
        return (
            f"Corpus consent revoked for '{user_id}'. "
            "No further memories will be exported to the training corpus."
        )
    else:
        return f"Unknown action '{action}'. Use 'grant' or 'revoke'."
