# Familiar - Self-Hosted AI Companion Platform
# Copyright (c) 2026 George Scott Foley
# ORCID: 0009-0006-4957-0540
# Email: Georgescottfoley@proton.me
# Licensed under the MIT License - see LICENSE file for details

"""
Training Data Curation Skill

Self-curating pipeline for proprietary model training data.
Extracts instruction/response pairs from conversation history and tool calls,
scores quality with heuristics, and exports in standard fine-tuning formats.

All data carries full provenance metadata (author, license, consent, timestamp)
to ensure clean IP lineage for downstream model training.

Dependencies: None (pure stdlib)

v1.1.0 changes:
- SHA-256/16-char IDs replace MD5/8-char (collision resistance)
- User identity threaded through provenance (author field)
- Explicit consent records replace "platform_tos" assertion
- raw_content guard (50KB cap, 4K per-pair cap)
- Watermark updates are now process-safe (fcntl exclusive lock)
- Export fingerprint (SHA-256 of dataset file) added to metadata
- SQL dynamic WHERE eliminated in analytics fallback path
"""

import hashlib
import json
import logging
import re
import sqlite3
from datetime import datetime
from pathlib import Path

try:
    import fcntl
except ImportError:
    fcntl = None

logger = logging.getLogger(__name__)

CURATOR_VERSION = "1.1.0"
MAX_RAW_CONTENT = 50_000  # Maximum raw_content input size (~50KB)
MAX_PAIR_CHARS = 4_000  # Maximum chars per instruction or response


# ── Path helpers (tenant-scoped in Reflection, default in standalone) ──


def _get_data_dir():
    """Get data directory (tenant-scoped in Reflection, default in standalone)."""
    try:
        from familiar.core import paths

        return paths.DATA_DIR
    except ImportError:
        return Path.home() / ".familiar" / "data"


try:
    from familiar.core.utils import atomic_write_json, generate_id, safe_load_json
except ImportError:
    atomic_write_json = None
    safe_load_json = None
    generate_id = None

try:
    from familiar.core.paths import HISTORY_FILE
except ImportError:
    HISTORY_FILE = None


def _get_training_dir():
    """Get training data directory (re-evaluates for tenant scoping)."""
    d = _get_data_dir() / "training"
    d.mkdir(parents=True, exist_ok=True)
    return d


def _get_curated_dir():
    d = _get_training_dir() / "curated"
    d.mkdir(parents=True, exist_ok=True)
    return d


def _get_exports_dir():
    d = _get_training_dir() / "exports"
    d.mkdir(parents=True, exist_ok=True)
    return d


def _get_history_file():
    """Get history file path."""
    if HISTORY_FILE is not None:
        return HISTORY_FILE
    return _get_data_dir() / "history.json"


def _get_state_file():
    return _get_training_dir() / "state.json"


def _get_provenance_file():
    return _get_training_dir() / "provenance.json"


def _get_skill_config():
    """Load this skill's own config.yaml (not the main app config)."""
    cfg_path = Path(__file__).parent / "config.yaml"
    if cfg_path.exists():
        try:
            import yaml

            with open(cfg_path, encoding="utf-8") as f:
                return yaml.safe_load(f) or {}
        except ImportError:
            # PyYAML not available — parse the owner_identity manually
            try:
                text = cfg_path.read_text(encoding="utf-8")
                config = {}
                owner = {}
                in_owner = False
                for line in text.splitlines():
                    stripped = line.strip()
                    if stripped.startswith("owner_identity:"):
                        in_owner = True
                        continue
                    if in_owner and line.startswith("  "):
                        if ":" in stripped:
                            k, v = stripped.split(":", 1)
                            owner[k.strip()] = v.strip().strip('"').strip("'")
                    elif in_owner and not line.startswith(" "):
                        in_owner = False
                config["owner_identity"] = owner
                return config
            except (IOError, OSError) as e:
                logger.debug("I/O error suppressed: %s", e)
        except (IOError, OSError) as e:
            logger.debug("I/O error suppressed: %s", e)
    return {}


def _get_author_identity(data: dict) -> dict:
    """
    Resolve author identity for provenance records.

    Priority order:
      1. UserContext injected by agent (_author key in data dict)
      2. owner_identity from this skill's config.yaml
      3. Fallback to "familiar" (standalone with no config)
    """
    # 1. Multi-user: agent injects UserContext data
    injected = data.get("_author")
    if injected and isinstance(injected, dict):
        return {
            "id": injected.get("user_id", "unknown"),
            "name": injected.get("name", "unknown"),
            "email": injected.get("email", ""),
            "source": "user_context",
        }

    # 2. Standalone: read from skill's own config.yaml
    cfg = _get_skill_config()
    owner = cfg.get("owner_identity", {})
    if owner.get("name"):
        return {
            "id": owner.get("orcid") or owner.get("name"),
            "name": owner["name"],
            "orcid": owner.get("orcid", ""),
            "license": owner.get("license", "proprietary"),
            "source": "config",
        }

    # 3. Fallback
    return {"id": "familiar", "name": "familiar", "source": "default"}


# ── Generic helpers ──


def _gen_id():
    if generate_id:
        return generate_id()
    import secrets

    return secrets.token_hex(4)


def _load_json(filepath, default=None):
    """Load JSON with fallback."""
    if safe_load_json:
        return safe_load_json(filepath, default=default or (lambda: {}))
    if filepath.exists():
        try:
            with open(filepath, encoding="utf-8") as f:
                return json.load(f)
        except (json.JSONDecodeError, IOError) as e:
            logger.debug("JSON parse error suppressed: %s", e)
    return (default() if callable(default) else default) or {}


def _save_json(filepath, data):
    """Save JSON atomically."""
    if atomic_write_json:
        atomic_write_json(filepath, data)
    else:
        filepath.parent.mkdir(parents=True, exist_ok=True)
        with open(filepath, "w", encoding="utf-8") as f:
            json.dump(data, f, indent=2)


def _read_jsonl(filepath):
    """Read JSONL file, skipping corrupt lines."""
    records = []
    if not filepath.exists():
        return records
    with open(filepath, encoding="utf-8") as f:
        for lineno, line in enumerate(f, 1):
            line = line.strip()
            if not line:
                continue
            try:
                records.append(json.loads(line))
            except json.JSONDecodeError:
                logger.warning("Skipping corrupt JSONL line %d in %s", lineno, filepath.name)
    return records


def _append_jsonl(filepath, records):
    """Append records to a JSONL file (crash-safe: worst case one corrupt line)."""
    filepath.parent.mkdir(parents=True, exist_ok=True)
    with open(filepath, "a", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec, default=str) + "\n")


# ── PII detection (regex heuristic) ──

_PII_PATTERNS = [
    re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b"),  # email
    re.compile(r"\b\d{3}[-.\s]?\d{3}[-.\s]?\d{4}\b"),  # US phone
    re.compile(r"\b\d{3}[-\s]?\d{2}[-\s]?\d{4}\b"),  # SSN
    re.compile(r"\b(?:\d{4}[-\s]?){3}\d{4}\b"),  # credit card
]


def _contains_pii(text):
    """Check if text likely contains PII."""
    for pat in _PII_PATTERNS:
        if pat.search(text):
            return True
    return False


# ── Quality scoring (all heuristic, no LLM) ──


def _score_conversation(instruction, response):
    """
    Score an instruction/response pair. Returns (score, signals dict).

    Scoring breakdown:
      Response length          +0.0 to +0.3
      Substantive instruction  +0.0 to +0.2
      Error response penalty   -0.2
      PII detected penalty     -0.3
      Tool call present bonus  +0.1
      Keyword coherence        +0.0 to +0.2
    """
    score = 0.0
    signals = {
        "response_length": len(response),
        "instruction_length": len(instruction),
        "has_tool_call": False,
        "contains_pii_risk": False,
        "is_error_response": False,
        "keyword_overlap": 0.0,
    }

    # Response length: 0-50 chars = 0, 50-200 = 0.1, 200-500 = 0.2, 500+ = 0.3
    rlen = len(response)
    if rlen >= 500:
        score += 0.3
    elif rlen >= 200:
        score += 0.2
    elif rlen >= 50:
        score += 0.1

    # Substantive instruction: reject single-word / trivial
    words = instruction.split()
    if len(words) >= 5:
        score += 0.2
    elif len(words) >= 3:
        score += 0.1
    # Single-word / very short instructions get +0.0

    # Error response penalty
    error_indicators = ["error", "sorry, i can't", "i'm not able to", "failed to", "exception"]
    response_lower = response.lower()
    if any(ind in response_lower for ind in error_indicators):
        score -= 0.2
        signals["is_error_response"] = True

    # PII penalty
    combined_text = instruction + " " + response
    if _contains_pii(combined_text):
        score -= 0.3
        signals["contains_pii_risk"] = True

    # Tool call bonus (look for common tool-call markers)
    tool_markers = ["tool_call", "function_call", "I'll check", "I'll search", "Let me look"]
    if any(m.lower() in response_lower for m in tool_markers):
        score += 0.1
        signals["has_tool_call"] = True

    # Keyword coherence: overlap between instruction words and response words
    inst_words = set(w.lower() for w in words if len(w) > 3)
    resp_words = set(w.lower() for w in response.split() if len(w) > 3)
    if inst_words:
        overlap = len(inst_words & resp_words) / len(inst_words)
        coherence = min(overlap, 1.0) * 0.2
        score += coherence
        signals["keyword_overlap"] = round(overlap, 2)

    return round(max(score, 0.0), 2), signals


def _score_tool_call(skill, action, success, metadata):
    """Score a tool call record. Returns (score, signals dict)."""
    score = 0.0
    signals = {
        "skill": skill or "",
        "action": action or "",
        "success": success,
    }

    # Successful calls score higher
    if success:
        score += 0.4
    else:
        score += 0.1

    # Known high-value skills get a bonus
    high_value = {"knowledge_base", "tasks", "email", "calendar", "nonprofit", "triage"}
    if skill in high_value:
        score += 0.2

    # Has meaningful action name
    if action and len(action) > 2:
        score += 0.2

    return round(min(score, 1.0), 2), signals


# ── Template mapping for synthetic instructions ──

_TOOL_TEMPLATES = {
    "search_knowledge_base": "Search the knowledge base for {query}",
    "save_to_knowledge": "Save this to my knowledge base: {title}",
    "add_task": "Add a new task: {title}",
    "list_tasks": "Show my tasks",
    "complete_task": "Mark task {id} as complete",
    "check_inbox": "Check my email inbox",
    "send_email": "Send an email to {to} about {subject}",
    "search_email": "Search my email for {query}",
    "daily_briefing": "Give me my daily briefing",
    "check_deadlines": "Check for upcoming deadlines",
    "get_events": "Show my calendar events",
    "create_event": "Schedule a meeting: {title}",
    "web_search": "Search the web for {query}",
    "web_read": "Read this webpage: {url}",
    "grant_deadlines": "Show upcoming grant deadlines",
    "donors_needing_thanks": "Which donors need thank-you notes?",
    "heartbeat_check": "Do a quick check-in on my tasks and deadlines",
}

# ── SQL constants for analytics fallback (Task 2: no dynamic WHERE) ──

_SQL_TOOL_CALLS_SUCCESS = (
    "SELECT skill, action, success, timestamp, metadata, error_message "
    "FROM events WHERE event_type = 'tool_call' AND success = 1 "
    "ORDER BY timestamp DESC LIMIT ?"
)
_SQL_TOOL_CALLS_ALL = (
    "SELECT skill, action, success, timestamp, metadata, error_message "
    "FROM events WHERE event_type = 'tool_call' "
    "ORDER BY timestamp DESC LIMIT ?"
)


def _synthesize_instruction(skill, action, metadata):
    """Create a synthetic instruction from skill/action/metadata."""
    key = action or skill
    template = _TOOL_TEMPLATES.get(key)

    if template:
        # Try to fill in template variables from metadata
        try:
            return template.format_map(
                {k: v for k, v in (metadata or {}).items() if isinstance(v, str)}
            )
        except (KeyError, ValueError):
            # Strip unfilled placeholders
            return re.sub(r"\{[^}]+\}", "...", template)

    # Fallback: construct from skill + action names
    parts = []
    if action:
        parts.append(action.replace("_", " ").title())
    elif skill:
        parts.append(f"Use the {skill.replace('_', ' ')} skill")
    return " ".join(parts) if parts else "Perform an action"


# ── Provenance tracking ──


def _get_consent_basis(user_id: str = None) -> str:
    """
    Return a consent basis string for provenance records.

    Stores full consent record in consent.json on first run.
    Returns a lightweight string key for embedding in each record.

    For multi-user: checks user.settings["data_curation_consent"].
    For standalone: auto-grants on first curation run (owner running on own data).
    """
    # Multi-user path: look up user settings
    if user_id and user_id not in ("familiar", "unknown"):
        try:
            from familiar.core.users import get_user_manager

            manager = get_user_manager()
            user = manager.get_user(user_id)
            if user:
                consent_ts = user.settings.get("data_curation_consent")
                if consent_ts:
                    return "explicit_opt_in"
        except Exception as e:
            logger.debug("Error suppressed: %s", e)
    # Standalone path: persist consent record to disk on first run
    consent_file = _get_training_dir() / "consent.json"
    if consent_file.exists():
        record = _load_json(consent_file, default=lambda: {})
        if record.get("granted_at"):
            return record.get("basis", "owner_standalone")

    # First run: create consent record (standalone owner implicitly consents
    # by running curation on their own data)
    record = {
        "basis": "owner_standalone",
        "granted_at": datetime.now().isoformat(),
        "user_id": user_id or "standalone_owner",
        "note": "Auto-granted on first curation run (standalone deployment).",
    }
    _save_json(consent_file, record)
    return record["basis"]


def _extract_author_id(record):
    """Extract author ID from a record, handling both string and dict formats."""
    author = record.get("provenance", {}).get("author", {})
    if isinstance(author, dict):
        return author.get("id", "unknown")
    return str(author) if author else "unknown"


def _extract_unique_authors(records):
    """Collect unique author identities from a list of records."""
    seen = []
    for r in records:
        aid = _extract_author_id(r)
        if aid not in seen:
            seen.append(aid)
    return seen


def _load_provenance():
    return _load_json(
        _get_provenance_file(),
        default=lambda: {
            "version": 1,
            "summary": {"total_examples": 0, "by_source_type": {}, "avg_quality_score": 0.0},
            "chain_of_custody": [],
        },
    )


def _save_provenance(prov):
    _save_json(_get_provenance_file(), prov)


def _record_provenance(action, records_processed, records_accepted, avg_quality=0.0):
    """Append a chain-of-custody entry."""
    prov = _load_provenance()

    custody_entry = {
        "action": action,
        "timestamp": datetime.now().isoformat(),
        "records_processed": records_processed,
        "records_accepted": records_accepted,
        "avg_quality": round(avg_quality, 3),
        "curator_version": CURATOR_VERSION,
    }
    prov["chain_of_custody"].append(custody_entry)

    # Recalculate summary from all JSONL files
    total = 0
    by_source = {}
    quality_sum = 0.0
    for src_name in ("conversations", "tool_calls", "generated"):
        recs = _read_jsonl(_get_curated_dir() / f"{src_name}.jsonl")
        count = len(recs)
        total += count
        by_source[src_name] = count
        quality_sum += sum(r.get("quality_score", 0) for r in recs)

    prov["summary"] = {
        "total_examples": total,
        "by_source_type": by_source,
        "avg_quality_score": round(quality_sum / total, 3) if total else 0.0,
    }

    _save_provenance(prov)

    # Anchor custody entry to genesis block chain (no-op if no genesis exists)
    try:
        from familiar.skills.training.bridge import anchor_provenance

        anchor_provenance(custody_entry)
    except Exception:
        logger.debug("Chain anchor skipped for provenance entry", exc_info=True)


# ── State management (watermarks) ──


def _load_state():
    return _load_json(
        _get_state_file(),
        default=lambda: {
            "conversations_last_ts": None,
            "tool_calls_last_ts": None,
        },
    )


def _save_state(state):
    _save_json(_get_state_file(), state)


def _update_watermark(key: str, value: str):
    """
    Thread/process-safe watermark update using an exclusive file lock.
    Falls back to best-effort on platforms without fcntl (Windows).
    """
    lock_path = _get_state_file().with_suffix(".lock")
    if fcntl is not None:
        try:
            with open(lock_path, "w") as lf:
                fcntl.flock(lf, fcntl.LOCK_EX)
                state = _load_state()
                state[key] = value
                _save_state(state)
                # Lock released on context exit
            return
        except OSError as e:
            logger.debug("I/O error suppressed: %s", e)
    # fcntl not available or lock failed — best-effort
    state = _load_state()
    state[key] = value
    _save_state(state)


# ══════════════════════════════════════════════════════════════
# Tool Handlers
# ══════════════════════════════════════════════════════════════


def curate_conversations(data: dict) -> str:
    """
    Extract instruction/response pairs from conversation history,
    score quality with heuristics, and append to curated JSONL.
    """
    min_quality = data.get("min_quality", 0.5)
    max_examples = data.get("max_examples", 100)
    reprocess = data.get("reprocess", False)

    # Load conversation history
    history_file = _get_history_file()
    if not history_file.exists():
        return "No conversation history found. Chat with Familiar first to generate training data."

    raw_history = _load_json(history_file, default=lambda: [])

    # Normalize history into a flat message list.
    # Familiar stores history as {channel_key: [messages...]} dict.
    history = []
    if isinstance(raw_history, dict):
        if "messages" in raw_history:
            history = raw_history["messages"]
        elif "conversations" in raw_history:
            history = raw_history["conversations"]
        else:
            # Dict of channel_key -> message list
            for channel_key, messages in raw_history.items():
                if isinstance(messages, list):
                    history.extend(messages)
    elif isinstance(raw_history, list):
        history = raw_history

    if not history:
        return "Conversation history is empty."

    # Watermark-based incremental processing
    state = _load_state()
    last_ts = state.get("conversations_last_ts")

    if reprocess:
        last_ts = None

    # Extract user/assistant pairs
    pairs = []
    latest_ts = last_ts

    for i, msg in enumerate(history):
        ts = msg.get("timestamp") or msg.get("created_at") or msg.get("time")

        # Skip already-processed messages
        if last_ts and ts and ts <= last_ts:
            continue

        # Track latest timestamp
        if ts and (latest_ts is None or ts > latest_ts):
            latest_ts = ts

        role = msg.get("role", "").lower()
        if role == "user":
            # Look for the next assistant response
            for j in range(i + 1, min(i + 5, len(history))):
                next_msg = history[j]
                if next_msg.get("role", "").lower() == "assistant":
                    instruction = msg.get("content", "").strip()
                    response = next_msg.get("content", "").strip()

                    if instruction and response and len(instruction) > 2 and len(response) > 10:
                        pairs.append((instruction, response, ts))
                    break

    if not pairs:
        return "No new conversation pairs found to curate."

    # Score and filter
    curated = []
    quality_total = 0.0
    _author = _get_author_identity(data)

    for instruction, response, ts in pairs[:max_examples]:
        score, signals = _score_conversation(instruction, response)

        if score < min_quality:
            continue

        example_id = hashlib.sha256((instruction + response).encode()).hexdigest()[:16]

        record = {
            "id": example_id,
            "source_type": "conversation",
            "instruction": instruction,
            "response": response,
            "quality_score": score,
            "quality_signals": signals,
            "provenance": {
                "author": _author,
                "license": _author.get("license", "proprietary"),
                "consent": _get_consent_basis(_author.get("id")),
                "curated_at": datetime.now().isoformat(),
                "source_timestamp": ts,
                "curator_version": CURATOR_VERSION,
            },
        }
        curated.append(record)
        quality_total += score

    if curated:
        _append_jsonl(_get_curated_dir() / "conversations.jsonl", curated)

    # Update watermark
    if latest_ts:
        _update_watermark("conversations_last_ts", latest_ts)

    # Record provenance
    _record_provenance(
        action="curate_conversations",
        records_processed=len(pairs),
        records_accepted=len(curated),
        avg_quality=quality_total / len(curated) if curated else 0,
    )

    avg_q = round(quality_total / len(curated), 2) if curated else 0
    return (
        f"Curated {len(curated)} conversation examples "
        f"(from {len(pairs)} pairs, min_quality={min_quality}).\n"
        f"Average quality score: {avg_q}\n"
        f"Saved to: curated/conversations.jsonl"
    )


def curate_tool_calls(data: dict) -> str:
    """
    Extract successful tool call events from analytics DB,
    generate synthetic instructions, and append to curated JSONL.
    """
    min_quality = data.get("min_quality", 0.5)
    max_examples = data.get("max_examples", 200)
    success_only = data.get("success_only", True)

    # Try to get analytics store
    try:
        from familiar.core.analytics import EventType, get_analytics_store

        store = get_analytics_store()
    except ImportError:
        # Fallback: try reading analytics.db directly
        db_path = _get_data_dir() / "analytics.db"
        if not db_path.exists():
            return "Analytics database not found. Use Familiar's tools to generate analytics data first."
        store = None

    events = []

    if store:
        raw_events = store.query(
            event_type=EventType.TOOL_CALL,
            limit=max_examples * 2,  # fetch extra to account for filtering
        )
        for ev in raw_events:
            if success_only and not ev.success:
                continue
            events.append(
                {
                    "skill": ev.skill,
                    "action": ev.action,
                    "success": ev.success,
                    "timestamp": ev.timestamp.isoformat() if ev.timestamp else None,
                    "metadata": ev.metadata if isinstance(ev.metadata, dict) else {},
                    "error_message": ev.error_message,
                }
            )
    else:
        # Direct SQLite fallback
        db_path = _get_data_dir() / "analytics.db"
        try:
            conn = sqlite3.connect(str(db_path))
            conn.row_factory = sqlite3.Row
            query = _SQL_TOOL_CALLS_SUCCESS if success_only else _SQL_TOOL_CALLS_ALL
            rows = conn.execute(query, (max_examples * 2,)).fetchall()
            conn.close()
            for row in rows:
                meta = {}
                if row["metadata"]:
                    try:
                        meta = json.loads(row["metadata"])
                    except (json.JSONDecodeError, TypeError) as e:
                        logger.debug("JSON parse error suppressed: %s", e)
                events.append(
                    {
                        "skill": row["skill"],
                        "action": row["action"],
                        "success": bool(row["success"]),
                        "timestamp": row["timestamp"],
                        "metadata": meta,
                        "error_message": row["error_message"],
                    }
                )
        except (sqlite3.Error, OSError) as e:
            return f"Failed to read analytics database: {e}"

    if not events:
        return "No tool call events found in analytics."

    # Score and build curated records
    curated = []
    quality_total = 0.0
    _author = _get_author_identity(data)

    for ev in events[:max_examples]:
        score, signals = _score_tool_call(ev["skill"], ev["action"], ev["success"], ev["metadata"])
        if score < min_quality:
            continue

        instruction = _synthesize_instruction(ev["skill"], ev["action"], ev["metadata"])
        _result_preview = (ev.get("metadata") or {}).get("tool_result_preview", "")
        if ev["action"]:
            _tool_label = ev["action"].replace("_", " ")
            if _result_preview:
                response = f"I'll use the {_tool_label} tool.\n\n{_result_preview}"
            elif ev.get("error_message"):
                response = f"I'll use the {_tool_label} tool. Result: {ev['error_message']}"
            else:
                response = f"I'll use the {_tool_label} tool. Done."
        elif ev["skill"]:
            response = (
                f"I'll use the {ev['skill'].replace('_', ' ')} skill. {_result_preview or 'Done.'}"
            )
        else:
            response = _result_preview or "Done."

        example_id = hashlib.sha256((instruction + ev.get("timestamp", "")).encode()).hexdigest()[
            :16
        ]

        record = {
            "id": example_id,
            "source_type": "tool_call",
            "instruction": instruction,
            "response": response,
            "quality_score": score,
            "quality_signals": signals,
            "provenance": {
                "author": _author,
                "license": _author.get("license", "proprietary"),
                "consent": _get_consent_basis(_author.get("id")),
                "curated_at": datetime.now().isoformat(),
                "source_timestamp": ev.get("timestamp"),
                "curator_version": CURATOR_VERSION,
            },
        }
        curated.append(record)
        quality_total += score

    if curated:
        _append_jsonl(_get_curated_dir() / "tool_calls.jsonl", curated)

    if events and events[0].get("timestamp"):
        _update_watermark("tool_calls_last_ts", events[0]["timestamp"])

    _record_provenance(
        action="curate_tool_calls",
        records_processed=len(events),
        records_accepted=len(curated),
        avg_quality=quality_total / len(curated) if curated else 0,
    )

    avg_q = round(quality_total / len(curated), 2) if curated else 0
    return (
        f"Curated {len(curated)} tool call examples "
        f"(from {len(events)} events, min_quality={min_quality}).\n"
        f"Average quality score: {avg_q}\n"
        f"Saved to: curated/tool_calls.jsonl"
    )


def generate_content(data: dict) -> str:
    """
    Two-step LLM content generation.

    Step 1 (this call): Returns a structured prompt for the agent to send to the LLM.
    Step 2 (agent calls back with content_type="ingest"): Ingests the raw LLM output
    and saves it as curated training data.
    """
    content_type = data.get("content_type", "how_to")
    topic = data.get("topic", "").strip()
    count = min(data.get("count", 5), 20)
    raw_content = data.get("raw_content", "").strip()

    if raw_content and len(raw_content) > MAX_RAW_CONTENT:
        return (
            f"raw_content exceeds the {MAX_RAW_CONTENT:,} character limit "
            f"({len(raw_content):,} chars received). "
            "Split the content into smaller batches and call generate_content separately for each."
        )

    valid_types = {"how_to", "reference", "troubleshooting", "faq", "tutorial", "concept"}
    if content_type not in valid_types:
        return f"Invalid content_type. Choose from: {', '.join(sorted(valid_types))}"

    # Step 2: Ingest raw LLM-generated content
    if raw_content:
        records = []
        quality_total = 0.0
        _author = _get_author_identity(data)

        # Parse the raw content into Q&A pairs (expect "Q: ... A: ..." format)
        pairs = _parse_qa_pairs(raw_content)

        if not pairs:
            # Treat as a single instruction/response if no Q&A structure
            pairs = [("Explain: " + topic, raw_content)]

        # Per-pair length cap
        pairs = [
            (inst, resp)
            for inst, resp in pairs
            if len(inst) <= MAX_PAIR_CHARS and len(resp) <= MAX_PAIR_CHARS
        ]
        if not pairs:
            return "No valid Q&A pairs found after length filtering (max 4,000 chars per instruction/response)."

        for instruction, response in pairs:
            score, signals = _score_conversation(instruction, response)
            example_id = hashlib.sha256((instruction + response).encode()).hexdigest()[:16]

            record = {
                "id": example_id,
                "source_type": "generated",
                "instruction": instruction,
                "response": response,
                "quality_score": score,
                "quality_signals": signals,
                "provenance": {
                    "author": _author,
                    "license": _author.get("license", "proprietary"),
                    "consent": "original_work",
                    "content_type": content_type,
                    "topic": topic,
                    "curated_at": datetime.now().isoformat(),
                    "curator_version": CURATOR_VERSION,
                },
            }
            records.append(record)
            quality_total += score

        if records:
            _append_jsonl(_get_curated_dir() / "generated.jsonl", records)
            _record_provenance(
                action="generate_content",
                records_processed=len(pairs),
                records_accepted=len(records),
                avg_quality=quality_total / len(records) if records else 0,
            )

        avg_q = round(quality_total / len(records), 2) if records else 0
        return (
            f"Ingested {len(records)} generated examples (topic: {topic}, type: {content_type}).\n"
            f"Average quality score: {avg_q}\n"
            f"Saved to: curated/generated.jsonl"
        )

    # Step 1: Return a prompt for the agent to send to the LLM
    if not topic:
        return "Please provide a topic for content generation."

    type_prompts = {
        "how_to": f"Write {count} original how-to Q&A pairs about {topic}. "
        "Format each as 'Q: [question]\\nA: [detailed answer]'. "
        "Make answers practical and actionable, 2-4 sentences each.",
        "reference": f"Write {count} original reference Q&A pairs about {topic}. "
        "Format each as 'Q: [question]\\nA: [detailed answer]'. "
        "Focus on definitions, specifications, and factual details.",
        "troubleshooting": f"Write {count} original troubleshooting Q&A pairs about {topic}. "
        "Format each as 'Q: [problem description]\\nA: [step-by-step solution]'. "
        "Include common error scenarios.",
        "faq": f"Write {count} frequently asked questions and answers about {topic}. "
        "Format each as 'Q: [question]\\nA: [clear answer]'. "
        "Cover the most common user questions.",
        "tutorial": f"Write {count} tutorial-style Q&A pairs about {topic}. "
        "Format each as 'Q: [learning question]\\nA: [instructional answer]'. "
        "Progress from basic to advanced concepts.",
        "concept": f"Write {count} conceptual Q&A pairs explaining {topic}. "
        "Format each as 'Q: [concept question]\\nA: [explanation]'. "
        "Use analogies and examples where helpful.",
    }

    prompt = type_prompts[content_type]

    return (
        f"GENERATE_CONTENT_PROMPT:\n\n{prompt}\n\n"
        f"---\n"
        f"After generating the content, call generate_content again with:\n"
        f'  content_type: "{content_type}"\n'
        f'  topic: "{topic}"\n'
        f'  raw_content: "<paste the generated text here>"'
    )


def _parse_qa_pairs(text):
    """Parse Q&A pairs from text. Expects 'Q: ... A: ...' format."""
    pairs = []
    # Split on Q: markers
    parts = re.split(r"(?:^|\n)Q:\s*", text, flags=re.MULTILINE)

    for part in parts:
        part = part.strip()
        if not part:
            continue

        # Split on A: marker
        qa = re.split(r"\nA:\s*", part, maxsplit=1)
        if len(qa) == 2:
            q = qa[0].strip()
            a = qa[1].strip()
            if q and a:
                pairs.append((q, a))

    return pairs


def export_dataset(data: dict) -> str:
    """
    Combine curated JSONL sources, filter by quality, and export
    in a standard fine-tuning format.
    """
    fmt = data.get("format", "alpaca").lower()
    min_quality = data.get("min_quality", 0.5)
    sources = data.get("sources", ["conversations", "tool_calls", "generated", "memories"])

    valid_formats = {"alpaca", "sharegpt", "openai"}
    if fmt not in valid_formats:
        return f"Invalid format. Choose from: {', '.join(sorted(valid_formats))}"

    if isinstance(sources, str):
        sources = [s.strip() for s in sources.split(",")]

    valid_sources = {"conversations", "tool_calls", "generated", "memories"}
    sources = [s for s in sources if s in valid_sources]

    if not sources:
        return "No valid sources specified."

    # Collect all records
    all_records = []
    for src in sources:
        recs = _read_jsonl(_get_curated_dir() / f"{src}.jsonl")
        all_records.extend(recs)

    if not all_records:
        return "No curated data found. Run curate_conversations or curate_tool_calls first."

    # Filter by quality
    filtered = [r for r in all_records if r.get("quality_score", 0) >= min_quality]

    if not filtered:
        return f"No records meet quality threshold {min_quality}. Try a lower min_quality."

    # Deduplicate by ID
    seen = set()
    unique = []
    for r in filtered:
        rid = r.get("id", "")
        if rid not in seen:
            seen.add(rid)
            unique.append(r)
    filtered = unique

    # Format records
    formatted = []
    for r in filtered:
        instruction = r.get("instruction", "")
        response = r.get("response", "")

        if fmt == "alpaca":
            formatted.append(
                {
                    "instruction": instruction,
                    "input": "",
                    "output": response,
                }
            )
        elif fmt == "sharegpt":
            formatted.append(
                {
                    "conversations": [
                        {"from": "human", "value": instruction},
                        {"from": "gpt", "value": response},
                    ],
                }
            )
        elif fmt == "openai":
            formatted.append(
                {
                    "messages": [
                        {"role": "user", "content": instruction},
                        {"role": "assistant", "content": response},
                    ],
                }
            )

    # Determine version number
    exports_dir = _get_exports_dir()
    existing = list(exports_dir.glob("dataset_v*.jsonl"))
    version = len(existing) + 1

    # Write dataset
    dataset_path = exports_dir / f"dataset_v{version}.jsonl"
    with open(dataset_path, "w", encoding="utf-8") as f:
        for rec in formatted:
            f.write(json.dumps(rec) + "\n")

    # Compute export fingerprint (SHA-256 of the complete dataset file)
    with open(dataset_path, "rb") as _f:
        export_fingerprint = hashlib.sha256(_f.read()).hexdigest()

    # Write metadata
    meta = {
        "version": version,
        "format": fmt,
        "export_fingerprint": export_fingerprint,
        "total_examples": len(formatted),
        "min_quality": min_quality,
        "sources": sources,
        "source_counts": {
            s: sum(1 for r in filtered if r.get("source_type") == s) for s in sources
        },
        "avg_quality_score": round(
            sum(r.get("quality_score", 0) for r in filtered) / len(filtered), 3
        ),
        "exported_at": datetime.now().isoformat(),
        "curator_version": CURATOR_VERSION,
        "provenance_summary": {
            "authors": _extract_unique_authors(filtered),
            "license": "proprietary",
            "all_consent_types": list(
                set(r.get("provenance", {}).get("consent", "unknown") for r in filtered)
            ),
        },
    }
    meta_path = exports_dir / f"dataset_v{version}_meta.json"
    _save_json(meta_path, meta)

    # Anchor export to genesis block chain (no-op if no genesis exists)
    try:
        from familiar.skills.training.bridge import anchor_export

        anchor_export(meta)
    except Exception:
        logger.debug("Chain anchor skipped for export", exc_info=True)

    _record_provenance(
        action="export_dataset",
        records_processed=len(all_records),
        records_accepted=len(formatted),
        avg_quality=meta["avg_quality_score"],
    )

    return (
        f"Exported {len(formatted)} examples (format: {fmt}, v{version}).\n"
        f"Sources: {', '.join(sources)}\n"
        f"Quality threshold: {min_quality}\n"
        f"Average quality: {meta['avg_quality_score']}\n"
        f"Dataset: {dataset_path.name}\n"
        f"Metadata: {meta_path.name}"
    )


def training_stats(data: dict) -> str:
    """Display dataset statistics: totals, quality distribution, provenance, exports."""
    detailed = data.get("detailed", False)

    curated_dir = _get_curated_dir()
    exports_dir = _get_exports_dir()

    # Count records per source
    source_stats = {}
    all_scores = []

    for src_name in ("conversations", "tool_calls", "generated"):
        recs = _read_jsonl(curated_dir / f"{src_name}.jsonl")
        scores = [r.get("quality_score", 0) for r in recs]
        source_stats[src_name] = {
            "count": len(recs),
            "avg_quality": round(sum(scores) / len(scores), 2) if scores else 0,
            "min_quality": round(min(scores), 2) if scores else 0,
            "max_quality": round(max(scores), 2) if scores else 0,
        }
        all_scores.extend(scores)

    total = sum(s["count"] for s in source_stats.values())

    lines = ["Training Data Statistics\n"]
    lines.append(f"  Total curated examples: {total}")

    if total == 0:
        lines.append("\n  No training data curated yet.")
        lines.append("  Run curate_conversations to extract data from your chat history.")
        return "\n".join(lines)

    lines.append(f"  Overall avg quality: {round(sum(all_scores) / len(all_scores), 2)}")
    lines.append("")

    # Per-source breakdown
    lines.append("  By Source:")
    for name, stats in source_stats.items():
        if stats["count"] > 0:
            lines.append(
                f"    {name}: {stats['count']} examples "
                f"(avg: {stats['avg_quality']}, range: {stats['min_quality']}-{stats['max_quality']})"
            )
        else:
            lines.append(f"    {name}: 0 examples")

    # Quality distribution
    if detailed and all_scores:
        lines.append("")
        lines.append("  Quality Distribution:")
        buckets = {"0.0-0.3": 0, "0.3-0.5": 0, "0.5-0.7": 0, "0.7-0.9": 0, "0.9-1.0": 0}
        for s in all_scores:
            if s < 0.3:
                buckets["0.0-0.3"] += 1
            elif s < 0.5:
                buckets["0.3-0.5"] += 1
            elif s < 0.7:
                buckets["0.5-0.7"] += 1
            elif s < 0.9:
                buckets["0.7-0.9"] += 1
            else:
                buckets["0.9-1.0"] += 1
        for bucket, count in buckets.items():
            bar = "#" * int(count / max(len(all_scores), 1) * 30)
            lines.append(f"    {bucket}: {count:>4} {bar}")

    # Exports
    existing_exports = sorted(exports_dir.glob("dataset_v*_meta.json"))
    if existing_exports:
        lines.append("")
        lines.append("  Exports:")
        for meta_path in existing_exports:
            meta = _load_json(meta_path, default=lambda: {})
            v = meta.get("version", "?")
            fmt = meta.get("format", "?")
            count = meta.get("total_examples", "?")
            date = meta.get("exported_at", "")[:10]
            lines.append(f"    v{v}: {count} examples ({fmt}) - {date}")
    else:
        lines.append("\n  No exports yet. Run export_dataset to create one.")

    # Provenance summary
    if detailed:
        prov = _load_provenance()
        chain = prov.get("chain_of_custody", [])
        if chain:
            lines.append("")
            lines.append(f"  Provenance: {len(chain)} curation runs recorded")
            for entry in chain[-5:]:  # Show last 5
                lines.append(
                    f"    {entry.get('action', '?')} @ {entry.get('timestamp', '?')[:16]} "
                    f"({entry.get('records_accepted', 0)} accepted)"
                )

    return "\n".join(lines)


def list_training_data(data: dict) -> str:
    """Browse curated training examples with filtering."""
    source = data.get("source", "all")
    min_quality = data.get("min_quality", 0.0)
    limit = min(data.get("limit", 10), 50)
    search = data.get("search", "").strip().lower()

    curated_dir = _get_curated_dir()

    # Collect records
    all_records = []
    if source == "all":
        sources = ["conversations", "tool_calls", "generated"]
    else:
        sources = [source]

    valid_sources = {"conversations", "tool_calls", "generated", "memories"}
    sources = [s for s in sources if s in valid_sources]

    for src_name in sources:
        recs = _read_jsonl(curated_dir / f"{src_name}.jsonl")
        all_records.extend(recs)

    if not all_records:
        return "No curated training data found. Run curate_conversations first."

    # Filter by quality
    filtered = [r for r in all_records if r.get("quality_score", 0) >= min_quality]

    # Filter by search term
    if search:
        filtered = [
            r
            for r in filtered
            if search in r.get("instruction", "").lower() or search in r.get("response", "").lower()
        ]

    if not filtered:
        return f"No examples match filters (source={source}, min_quality={min_quality}, search='{search}')."

    # Sort by quality descending
    filtered.sort(key=lambda r: r.get("quality_score", 0), reverse=True)

    total = len(filtered)
    showing = filtered[:limit]

    lines = [f"Training Data ({total} total, showing {len(showing)}):\n"]

    for r in showing:
        src = r.get("source_type", "?")
        score = r.get("quality_score", 0)
        inst = r.get("instruction", "")
        resp = r.get("response", "")

        # Truncate for display
        inst_preview = (inst[:80] + "...") if len(inst) > 80 else inst
        resp_preview = (resp[:80] + "...") if len(resp) > 80 else resp

        lines.append(f"  [{r.get('id', '?')}] ({src}) quality={score}")
        lines.append(f"    Q: {inst_preview}")
        lines.append(f"    A: {resp_preview}")
        lines.append("")

    if total > limit:
        lines.append(f"  ... and {total - limit} more (use limit={total} to see all)")

    return "\n".join(lines)


# ══════════════════════════════════════════════════════════════
# Pipeline Tool Handlers
# ══════════════════════════════════════════════════════════════


def run_training_pipeline(data: dict) -> str:
    """Run the full training pipeline: curate → quality gate → export → anchor."""
    from familiar.skills.training.pipeline import PipelineConfig, run_pipeline

    config = PipelineConfig(
        min_records=data.get("min_records", 10),
        min_avg_quality=data.get("min_avg_quality", 0.5),
        min_diversity=data.get("min_diversity", 2),
        export_format=data.get("format", "alpaca"),
        include_memories=data.get("include_memories", True),
        min_quality=data.get("min_quality", 0.5),
    )

    run = run_pipeline(config)

    if run.status == "complete":
        gate = run.stats.get("quality_gate", {})
        return (
            f"Pipeline complete (run_id={run.run_id}).\n"
            f"Records: {gate.get('total_records', 0)}, "
            f"Avg quality: {gate.get('avg_quality', 0)}, "
            f"Sources: {gate.get('distinct_sources', 0)}\n"
            f"Export: {run.stats.get('export', 'n/a')}\n"
            f"Duration: {run.started_at} → {run.completed_at}"
        )
    elif run.status == "failed":
        return f"Pipeline failed: {run.error}"
    else:
        return f"Pipeline ended with status: {run.status}"


def pipeline_status(data: dict) -> str:
    """Show pipeline run status and history."""
    from familiar.skills.training.pipeline import get_pipeline_status

    status = get_pipeline_status()
    total = status.get("total_runs", 0)

    if total == 0:
        return "No pipeline runs yet. Use run_training_pipeline to start."

    latest = status.get("latest_run", {})
    lines = [f"Pipeline Status ({total} total runs)\n"]

    lines.append(f"  Latest: {latest.get('run_id', '?')} — {latest.get('status', '?')}")
    lines.append(f"  Started: {latest.get('started_at', '')[:19]}")
    if latest.get("completed_at"):
        lines.append(f"  Completed: {latest['completed_at'][:19]}")
    if latest.get("error"):
        lines.append(f"  Error: {latest['error']}")

    history = status.get("history", [])
    if len(history) > 1:
        lines.append("\nRecent runs:")
        for h in reversed(history[:-1]):
            err = f" — {h['error'][:40]}" if h.get("error") else ""
            lines.append(f"  {h['run_id']} | {h['status']}{err}")

    return "\n".join(lines)


# ══════════════════════════════════════════════════════════════
# Consent Tool Handlers
# ══════════════════════════════════════════════════════════════


def manage_consent(data: dict) -> str:
    """Grant or revoke consent for training data usage."""
    from familiar.core.consent import ConsentManager, ConsentScope

    action = data.get("action", "grant")
    user_id = data.get("user_id") or data.get("_author", {}).get("user_id", "standalone_owner")
    scopes = data.get("scopes", [ConsentScope.LOCAL_TRAINING.value])

    if isinstance(scopes, str):
        scopes = [s.strip() for s in scopes.split(",")]

    valid_scopes = {s.value for s in ConsentScope}
    scopes = [s for s in scopes if s in valid_scopes]

    if not scopes:
        return f"No valid scopes provided. Options: {', '.join(sorted(valid_scopes))}"

    manager = ConsentManager()

    if action == "grant":
        record = manager.grant_consent(user_id, scopes)
        chain = f" (chain: {record.chain_block_hash[:16]})" if record.chain_block_hash else ""
        return f"Consent granted for {user_id}: {', '.join(record.scopes)}{chain}"
    elif action == "revoke":
        record = manager.revoke_consent(user_id, scopes)
        remaining = f"Remaining: {', '.join(record.scopes)}" if record.scopes else "All revoked"
        return f"Consent revoked for {user_id}. {remaining}"
    else:
        return f"Invalid action '{action}'. Use 'grant' or 'revoke'."


def check_consent_status(data: dict) -> str:
    """Check consent status for a user."""
    from familiar.core.consent import ConsentManager

    user_id = data.get("user_id") or data.get("_author", {}).get("user_id", "standalone_owner")
    manager = ConsentManager()

    record = manager.get_consent(user_id)
    if record is None:
        return f"No consent record found for {user_id}. Use manage_consent to grant consent."

    lines = [f"Consent status for {user_id}:"]
    lines.append(f"  Scopes: {', '.join(record.scopes) if record.scopes else 'none'}")
    lines.append(f"  Granted: {record.granted_at[:19]}")
    if record.revoked_at:
        lines.append(f"  Revoked: {record.revoked_at[:19]}")
    if record.chain_block_hash:
        lines.append(f"  Chain: {record.chain_block_hash[:16]}...")

    # Audit trail
    audit = manager.get_audit_log(user_id, limit=5)
    if audit:
        lines.append("\n  Recent audit:")
        for entry in audit:
            lines.append(f"    {entry['action']} {entry['scopes']} @ {entry['timestamp'][:19]}")

    return "\n".join(lines)


def erase_my_data(data: dict) -> str:
    """Right to erasure: delete all training data for the requesting user."""
    from familiar.core.consent import ConsentManager

    user_id = data.get("user_id") or data.get("_author", {}).get("user_id", "standalone_owner")
    confirm = data.get("confirm", False)

    if not confirm:
        return (
            "This will permanently delete all your training data and revoke consent.\n"
            "To confirm, call erase_my_data with confirm=true."
        )

    manager = ConsentManager()
    result = manager.erase_user_data(user_id)

    chain = ""
    if result.get("chain_block_hash"):
        chain = f"\nChain anchor: {result['chain_block_hash'][:16]}..."

    return (
        f"Data erasure complete for {user_id}.\n"
        f"Records deleted: {result.get('records_deleted', 0)}\n"
        f"Consent revoked. Erasure timestamp: {result.get('erased_at', '')[:19]}"
        f"{chain}"
    )


# ══════════════════════════════════════════════════════════════
# Co-op Tool Handlers
# ══════════════════════════════════════════════════════════════


def create_coop(data: dict) -> str:
    """Create a new training data co-op."""
    from familiar.skills.training.coop import CoopManager, CoopRules

    name = data.get("name", "").strip()
    if not name:
        return "Please provide a name for the co-op."

    rules = CoopRules(
        min_quality=data.get("min_quality", 0.5),
        required_consent=data.get("required_consent", ["coop_sharing"]),
        max_records_per_member=data.get("max_records_per_member", 10000),
    )

    manager = CoopManager()
    manifest = manager.create_coop(
        name=name,
        description=data.get("description", ""),
        rules=rules,
    )

    return (
        f"Co-op created: {manifest.name}\n"
        f"ID: {manifest.coop_id}\n"
        f"Members: {len(manifest.members)}\n"
        f"Share the manifest at ~/.familiar/data/coop/{manifest.coop_id}/manifest.json"
    )


def join_coop(data: dict) -> str:
    """Join an existing co-op by importing its manifest."""
    from familiar.skills.training.coop import CoopManager

    manifest_path = data.get("manifest_path", "").strip()
    if not manifest_path:
        return "Please provide the path to the co-op manifest JSON file."

    manager = CoopManager()
    try:
        result = manager.join_coop(manifest_path)
        if result:
            return f"Successfully joined co-op from {manifest_path}."
        return "Failed to join co-op."
    except (FileNotFoundError, ValueError) as e:
        return str(e)


def contribute_to_coop(data: dict) -> str:
    """Contribute curated data to a co-op."""
    from familiar.skills.training.coop import CoopManager

    coop_id = data.get("coop_id", "").strip()
    if not coop_id:
        # List available co-ops
        manager = CoopManager()
        coops = manager.list_coops()
        if not coops:
            return "No co-ops found. Create one with create_coop or join one with join_coop."
        lines = ["Available co-ops:"]
        for c in coops:
            lines.append(f"  {c['coop_id']} — {c['name']} ({c['members']} members)")
        lines.append("\nSpecify coop_id to contribute.")
        return "\n".join(lines)

    manager = CoopManager()
    result = manager.contribute(coop_id)

    if result.get("error"):
        return f"Contribution failed: {result['error']}"

    return (
        f"Contributed to co-op {coop_id}.\n"
        f"Records: {result.get('records_contributed', 0)}\n"
        f"Avg quality: {result.get('avg_quality', 0)}"
    )


def coop_status(data: dict) -> str:
    """Show co-op membership and contribution status."""
    from familiar.skills.training.coop import CoopManager

    manager = CoopManager()
    coops = manager.list_coops()

    if not coops:
        return "Not a member of any co-ops. Use create_coop or join_coop to start."

    lines = [f"Co-op Membership ({len(coops)} co-ops)\n"]

    for c in coops:
        lines.append(f"  {c['name']} ({c['coop_id']})")
        lines.append(f"    Members: {c['members']}, Created: {c['created_at'][:10]}")

        stats = manager.get_contribution_stats(c["coop_id"])
        cs = stats.get("contribution_stats", {})
        if cs:
            total_records = sum(s.get("records_contributed", 0) for s in cs.values())
            lines.append(f"    Total contributed: {total_records} records from {len(cs)} members")
        lines.append("")

    return "\n".join(lines)


# ══════════════════════════════════════════════════════════════
# Model Training Tool Handlers
# ══════════════════════════════════════════════════════════════


def train_model(data: dict) -> str:
    """Run LoRA fine-tuning on curated data."""
    from familiar.skills.training.model_training import TrainingConfig, TrainingRunner

    config = TrainingConfig(
        base_model=data.get("base_model", "qwen2.5:3b"),
        dataset_path=data.get("dataset_path", ""),
        lora_rank=data.get("lora_rank", 8),
        lora_alpha=data.get("lora_alpha", 16),
        epochs=data.get("epochs", 3),
        learning_rate=data.get("learning_rate", 2e-4),
        batch_size=data.get("batch_size", 4),
        max_seq_length=data.get("max_seq_length", 2048),
    )

    runner = TrainingRunner(config)

    if runner.backend == "stub":
        return runner._get_install_instructions()

    result = runner.train()

    if result.status == "complete":
        chain = ""
        if result.chain_block_hash:
            chain = f"\nChain anchor: {result.chain_block_hash[:16]}..."

        return (
            f"Training complete ({result.backend} backend).\n"
            f"Model: {result.model_path}\n"
            f"Model hash: {result.model_hash[:16]}...\n"
            f"Dataset fingerprint: {result.dataset_fingerprint[:16]}..."
            f"{chain}"
        )
    elif result.status == "stub":
        return result.error
    else:
        return f"Training failed: {result.error}"


def training_run_status(data: dict) -> str:
    """Show training backend detection and readiness status."""
    from familiar.skills.training.model_training import TrainingRunner

    runner = TrainingRunner()

    lines = ["Training Backend Status\n"]
    lines.append(f"  Detected: {runner.backend}")

    if runner.backend == "stub":
        lines.append("")
        lines.append(runner._get_install_instructions())
    elif runner.backend == "ollama":
        lines.append("  Ollama available for model creation.")
    elif runner.backend in ("peft", "unsloth"):
        lines.append(f"  {runner.backend} available for LoRA fine-tuning.")

    # Check for existing datasets
    try:
        from familiar.core.paths import DATA_DIR

        exports_dir = DATA_DIR / "training" / "exports"
        if exports_dir.exists():
            datasets = sorted(exports_dir.glob("dataset_v*.jsonl"))
            if datasets:
                lines.append(f"\n  Available datasets: {len(datasets)}")
                lines.append(f"  Latest: {datasets[-1].name}")
    except ImportError:
        pass

    return "\n".join(lines)


def generate_model_card_tool(data: dict) -> str:
    """Generate a model card with full provenance."""
    from familiar.skills.training.model_card import generate_model_card

    model_name = data.get("model_name", "familiar-custom")

    # Gather metadata
    dataset_meta = None
    chain_status = None

    try:
        from familiar.core.paths import DATA_DIR

        exports_dir = DATA_DIR / "training" / "exports"
        if exports_dir.exists():
            meta_files = sorted(exports_dir.glob("dataset_v*_meta.json"))
            if meta_files:
                with open(meta_files[-1], encoding="utf-8") as f:
                    dataset_meta = json.load(f)
    except Exception:
        pass

    try:
        from familiar.skills.training.bridge import get_chain_status

        chain_status = get_chain_status()
    except Exception:
        pass

    card = generate_model_card(
        model_name=model_name,
        dataset_meta=dataset_meta,
        chain_status=chain_status,
    )

    # Save to disk
    try:
        from familiar.skills.training.model_card import save_model_card

        path = save_model_card(
            model_name=model_name,
            dataset_meta=dataset_meta,
            chain_status=chain_status,
        )
        return f"{card}\n\n---\nSaved to: {path}"
    except Exception:
        return card


# ══════════════════════════════════════════════════════════════
# Tool Definitions
# ══════════════════════════════════════════════════════════════

TOOLS = [
    {
        "name": "curate_conversations",
        "description": (
            "Extract and score instruction/response pairs from conversation history for training data. "
            "Use this to build fine-tuning datasets from real user/assistant interactions. "
            "Incrementally processes only new conversations since last run unless reprocess is set. "
            "Saves results to curated/conversations.jsonl with quality scores and provenance metadata."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "min_quality": {
                    "type": "number",
                    "description": "Minimum quality score (0.0-1.0) to include",
                    "default": 0.5,
                },
                "max_examples": {
                    "type": "integer",
                    "description": "Maximum examples to process per run",
                    "default": 100,
                },
                "reprocess": {
                    "type": "boolean",
                    "description": "If true, reprocess all conversations ignoring watermark",
                    "default": False,
                },
            },
        },
        "handler": curate_conversations,
        "category": "training",
    },
    {
        "name": "curate_tool_calls",
        "description": (
            "Extract tool call events from the analytics database and generate synthetic "
            "instruction/response pairs for training data. "
            "Use this to capture skill usage patterns as fine-tuning examples. "
            "Saves results to curated/tool_calls.jsonl with quality scores and provenance metadata."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "min_quality": {
                    "type": "number",
                    "description": "Minimum quality score (0.0-1.0) to include",
                    "default": 0.5,
                },
                "max_examples": {
                    "type": "integer",
                    "description": "Maximum examples to process",
                    "default": 200,
                },
                "success_only": {
                    "type": "boolean",
                    "description": "Only include successful tool calls",
                    "default": True,
                },
            },
        },
        "handler": curate_tool_calls,
        "category": "training",
    },
    {
        "name": "generate_content",
        "description": (
            "Generate original training content on a topic using a two-step LLM workflow. "
            "First call returns a structured prompt for the agent to send to the LLM. "
            "Second call (with raw_content) ingests the generated Q&A pairs into curated/generated.jsonl. "
            "Supports content types: how_to, reference, troubleshooting, faq, tutorial, concept."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "topic": {
                    "type": "string",
                    "description": "Topic for content generation",
                },
                "content_type": {
                    "type": "string",
                    "enum": [
                        "how_to",
                        "reference",
                        "troubleshooting",
                        "faq",
                        "tutorial",
                        "concept",
                    ],
                    "description": "Type of content to generate",
                    "default": "how_to",
                },
                "count": {
                    "type": "integer",
                    "description": "Number of Q&A pairs to generate (max 20)",
                    "default": 5,
                },
                "raw_content": {
                    "type": "string",
                    "description": "LLM-generated content to ingest (step 2)",
                },
            },
        },
        "handler": generate_content,
        "category": "training",
    },
    {
        "name": "export_dataset",
        "description": (
            "Export curated training data as a versioned JSONL dataset in a standard fine-tuning format. "
            "Supports Alpaca, ShareGPT, and OpenAI message formats. "
            "Deduplicates by ID, filters by quality threshold, and writes a companion metadata JSON. "
            "Run curate_conversations or curate_tool_calls first to populate the curated data."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "format": {
                    "type": "string",
                    "enum": ["alpaca", "sharegpt", "openai"],
                    "description": "Output format",
                    "default": "alpaca",
                },
                "min_quality": {
                    "type": "number",
                    "description": "Minimum quality score for export",
                    "default": 0.5,
                },
                "sources": {
                    "type": "array",
                    "items": {"type": "string"},
                    "description": "Data sources to include",
                    "default": ["conversations", "tool_calls", "generated"],
                },
            },
        },
        "handler": export_dataset,
        "category": "training",
    },
    {
        "name": "training_stats",
        "description": (
            "Show training dataset statistics including example counts by source, quality score ranges, "
            "and a list of existing exports. "
            "Use this to check the health and size of the curated training pipeline. "
            "Set detailed=true for quality distribution histograms and provenance chain entries."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "detailed": {
                    "type": "boolean",
                    "description": "Show detailed quality distribution and provenance chain",
                    "default": False,
                },
            },
        },
        "handler": training_stats,
        "category": "training",
    },
    {
        "name": "list_training_data",
        "description": (
            "Browse curated training examples with filtering by source, quality threshold, "
            "and keyword search. "
            "Use this to inspect individual instruction/response pairs in the training pipeline. "
            "Returns up to 50 examples sorted by quality score descending."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "source": {
                    "type": "string",
                    "enum": ["all", "conversations", "tool_calls", "generated"],
                    "description": "Filter by data source",
                    "default": "all",
                },
                "min_quality": {
                    "type": "number",
                    "description": "Minimum quality score filter",
                    "default": 0.0,
                },
                "limit": {
                    "type": "integer",
                    "description": "Number of examples to show (max 50)",
                    "default": 10,
                },
                "search": {
                    "type": "string",
                    "description": "Search term to filter by",
                    "default": "",
                },
            },
        },
        "handler": list_training_data,
        "category": "training",
    },
    {
        "name": "run_training_pipeline",
        "description": (
            "Run the full automated training pipeline: curate conversations, curate tool calls, "
            "collect memories, quality gate, export dataset, and anchor to genesis chain. "
            "Returns a summary of the pipeline run including stats and any errors."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "format": {
                    "type": "string",
                    "enum": ["alpaca", "sharegpt", "openai"],
                    "description": "Export format",
                    "default": "alpaca",
                },
                "min_quality": {
                    "type": "number",
                    "description": "Minimum quality score (0.0-1.0)",
                    "default": 0.5,
                },
                "min_records": {
                    "type": "integer",
                    "description": "Minimum records to pass quality gate",
                    "default": 10,
                },
                "min_diversity": {
                    "type": "integer",
                    "description": "Minimum distinct source types",
                    "default": 2,
                },
                "include_memories": {
                    "type": "boolean",
                    "description": "Include markdown memories in pipeline",
                    "default": True,
                },
            },
        },
        "handler": run_training_pipeline,
        "category": "training",
    },
    {
        "name": "pipeline_status",
        "description": (
            "Show the status of the training pipeline including the latest run, "
            "run history, and any errors. Use this to check pipeline health."
        ),
        "input_schema": {
            "type": "object",
            "properties": {},
        },
        "handler": pipeline_status,
        "category": "training",
    },
    {
        "name": "manage_consent",
        "description": (
            "Grant or revoke consent for training data usage. "
            "Supports scopes: local_training, coop_sharing, public_dataset. "
            "All consent changes are anchored to the genesis chain for auditability."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "action": {
                    "type": "string",
                    "enum": ["grant", "revoke"],
                    "description": "Whether to grant or revoke consent",
                    "default": "grant",
                },
                "scopes": {
                    "type": "array",
                    "items": {"type": "string"},
                    "description": "Consent scopes to grant/revoke",
                    "default": ["local_training"],
                },
                "user_id": {
                    "type": "string",
                    "description": "User ID (defaults to current user)",
                },
            },
        },
        "handler": manage_consent,
        "category": "training",
    },
    {
        "name": "check_consent_status",
        "description": (
            "Check consent status for a user including granted scopes, "
            "timestamps, chain anchors, and recent audit trail."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "user_id": {
                    "type": "string",
                    "description": "User ID to check (defaults to current user)",
                },
            },
        },
        "handler": check_consent_status,
        "category": "training",
    },
    {
        "name": "erase_my_data",
        "description": (
            "Right to erasure: permanently delete all training data for the requesting user. "
            "Removes curated records, revokes consent, and anchors the erasure to the chain. "
            "Requires confirm=true to execute."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "confirm": {
                    "type": "boolean",
                    "description": "Must be true to confirm erasure",
                    "default": False,
                },
                "user_id": {
                    "type": "string",
                    "description": "User ID to erase (defaults to current user)",
                },
            },
        },
        "handler": erase_my_data,
        "category": "training",
    },
    {
        "name": "create_coop",
        "description": (
            "Create a new training data co-op for collaborative dataset building. "
            "Co-ops allow multiple Familiar instances to pool ethically-sourced training data "
            "with quality and consent rules enforced for every contribution."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "name": {
                    "type": "string",
                    "description": "Human-readable co-op name",
                },
                "description": {
                    "type": "string",
                    "description": "Purpose/description of the co-op",
                    "default": "",
                },
                "min_quality": {
                    "type": "number",
                    "description": "Minimum quality score for contributions",
                    "default": 0.5,
                },
            },
            "required": ["name"],
        },
        "handler": create_coop,
        "category": "training",
    },
    {
        "name": "join_coop",
        "description": (
            "Join an existing co-op by importing its manifest file. "
            "After joining, you can contribute curated data to the cooperative dataset."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "manifest_path": {
                    "type": "string",
                    "description": "Path to the co-op manifest JSON file",
                },
            },
            "required": ["manifest_path"],
        },
        "handler": join_coop,
        "category": "training",
    },
    {
        "name": "contribute_to_coop",
        "description": (
            "Contribute curated training data to a co-op. "
            "Data is filtered by co-op quality rules and consent requirements before contribution."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "coop_id": {
                    "type": "string",
                    "description": "Co-op ID to contribute to (omit to list available)",
                    "default": "",
                },
            },
        },
        "handler": contribute_to_coop,
        "category": "training",
    },
    {
        "name": "coop_status",
        "description": (
            "Show co-op membership, contribution statistics, and member counts. "
            "Use this to check the health and activity of your co-op memberships."
        ),
        "input_schema": {
            "type": "object",
            "properties": {},
        },
        "handler": coop_status,
        "category": "training",
    },
    {
        "name": "train_model",
        "description": (
            "Run LoRA fine-tuning on curated training data. "
            "Automatically detects available backend (unsloth, peft, ollama). "
            "Anchors the training run to the genesis chain with dataset fingerprint and model hash."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "base_model": {
                    "type": "string",
                    "description": "Base model to fine-tune",
                    "default": "qwen2.5:3b",
                },
                "dataset_path": {
                    "type": "string",
                    "description": "Path to dataset JSONL (defaults to latest export)",
                    "default": "",
                },
                "lora_rank": {
                    "type": "integer",
                    "description": "LoRA rank",
                    "default": 8,
                },
                "epochs": {
                    "type": "integer",
                    "description": "Number of training epochs",
                    "default": 3,
                },
            },
        },
        "handler": train_model,
        "category": "training",
    },
    {
        "name": "training_run_status",
        "description": (
            "Show training backend detection, readiness status, "
            "and available datasets for fine-tuning."
        ),
        "input_schema": {
            "type": "object",
            "properties": {},
        },
        "handler": training_run_status,
        "category": "training",
    },
    {
        "name": "generate_model_card",
        "description": (
            "Generate a Markdown model card with full chain-of-provenance information. "
            "Includes contributors, consent types, quality metrics, hyperparameters, "
            "and genesis chain verification status."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "model_name": {
                    "type": "string",
                    "description": "Display name for the model",
                    "default": "familiar-custom",
                },
            },
        },
        "handler": generate_model_card_tool,
        "category": "training",
    },
]
