# Familiar - Self-Hosted AI Companion Platform
# Copyright (c) 2026 George Scott Foley
# ORCID: 0009-0006-4957-0540
# Email: Georgescottfoley@proton.me
# Licensed under the MIT License - see LICENSE file for details

"""
Training Pipeline Orchestrator

Single entry point that sequences existing curation, quality gating,
export, and chain anchoring into an automated pipeline run.

Each run:
  1. Curate conversations (incremental)
  2. Curate tool calls (incremental)
  3. Collect markdown memories
  4. Quality gate (min records, avg quality, diversity)
  5. Export dataset
  6. Anchor export to genesis chain

State persisted to ~/.familiar/data/training/pipeline_state.json.

Dependencies: None (pure stdlib + existing skill functions)
"""

import hashlib
import json
import logging
from dataclasses import asdict, dataclass, field
from datetime import datetime
from pathlib import Path

logger = logging.getLogger(__name__)


def _get_data_dir():
    try:
        from familiar.core import paths

        return paths.DATA_DIR
    except ImportError:
        return Path.home() / ".familiar" / "data"


def _get_pipeline_state_path():
    d = _get_data_dir() / "training"
    d.mkdir(parents=True, exist_ok=True)
    return d / "pipeline_state.json"


@dataclass
class PipelineConfig:
    """Configuration for a pipeline run."""

    min_records: int = 10
    min_avg_quality: float = 0.5
    min_diversity: int = 2  # min distinct source types
    export_format: str = "alpaca"
    include_memories: bool = True
    min_quality: float = 0.5
    max_conversations: int = 100
    max_tool_calls: int = 200


@dataclass
class PipelineRunState:
    """State of a single pipeline run."""

    run_id: str
    status: str  # started|curating_conversations|curating_tool_calls|
    # collecting_memories|quality_gate|exporting|anchoring|complete|failed
    started_at: str
    completed_at: str = ""
    error: str = ""
    stats: dict = field(default_factory=dict)
    config: dict = field(default_factory=dict)


def _load_pipeline_state():
    """Load pipeline state from disk."""
    path = _get_pipeline_state_path()
    if path.exists():
        try:
            with open(path, encoding="utf-8") as f:
                return json.load(f)
        except (json.JSONDecodeError, IOError):
            pass
    return {"runs": [], "latest_run_id": None}


def _save_pipeline_state(state):
    """Save pipeline state to disk."""
    path = _get_pipeline_state_path()
    path.parent.mkdir(parents=True, exist_ok=True)
    with open(path, "w", encoding="utf-8") as f:
        json.dump(state, f, indent=2, default=str)


def _update_run(run: PipelineRunState):
    """Update the current run in persistent state."""
    state = _load_pipeline_state()
    runs = state.get("runs", [])

    # Replace existing run or append
    found = False
    for i, r in enumerate(runs):
        if r.get("run_id") == run.run_id:
            runs[i] = asdict(run)
            found = True
            break
    if not found:
        runs.append(asdict(run))

    # Keep last 50 runs
    if len(runs) > 50:
        runs = runs[-50:]

    state["runs"] = runs
    state["latest_run_id"] = run.run_id
    _save_pipeline_state(state)


def _quality_gate(config: PipelineConfig) -> tuple:
    """
    Check whether curated data meets quality thresholds.

    Returns (passed: bool, details: dict).
    """
    from familiar.skills.training.skill import _get_curated_dir, _read_jsonl

    curated_dir = _get_curated_dir()
    sources = {}
    all_records = []

    for src_name in ("conversations", "tool_calls", "generated", "memories"):
        recs = _read_jsonl(curated_dir / f"{src_name}.jsonl")
        sources[src_name] = len(recs)
        all_records.extend(recs)

    total = len(all_records)
    scores = [r.get("quality_score", 0) for r in all_records]
    avg_quality = sum(scores) / len(scores) if scores else 0.0
    distinct_sources = sum(1 for c in sources.values() if c > 0)

    details = {
        "total_records": total,
        "avg_quality": round(avg_quality, 3),
        "distinct_sources": distinct_sources,
        "source_counts": sources,
        "thresholds": {
            "min_records": config.min_records,
            "min_avg_quality": config.min_avg_quality,
            "min_diversity": config.min_diversity,
        },
    }

    passed = (
        total >= config.min_records
        and avg_quality >= config.min_avg_quality
        and distinct_sources >= config.min_diversity
    )
    details["passed"] = passed

    if not passed:
        reasons = []
        if total < config.min_records:
            reasons.append(f"records {total} < {config.min_records}")
        if avg_quality < config.min_avg_quality:
            reasons.append(f"avg quality {avg_quality:.3f} < {config.min_avg_quality}")
        if distinct_sources < config.min_diversity:
            reasons.append(f"source diversity {distinct_sources} < {config.min_diversity}")
        details["failure_reasons"] = reasons

    return passed, details


def run_pipeline(config: PipelineConfig = None) -> PipelineRunState:
    """
    Run the full training pipeline: curate → quality gate → export → anchor.

    Returns the final PipelineRunState.
    """
    if config is None:
        config = PipelineConfig()

    run_id = hashlib.sha256(datetime.now().isoformat().encode()).hexdigest()[:12]

    run = PipelineRunState(
        run_id=run_id,
        status="started",
        started_at=datetime.now().isoformat(),
        config=asdict(config),
    )
    _update_run(run)

    try:
        # Step 1: Curate conversations
        run.status = "curating_conversations"
        _update_run(run)

        from familiar.skills.training.skill import curate_conversations

        conv_result = curate_conversations(
            {
                "min_quality": config.min_quality,
                "max_examples": config.max_conversations,
            }
        )
        run.stats["conversations"] = conv_result
        _update_run(run)

        # Step 2: Curate tool calls
        run.status = "curating_tool_calls"
        _update_run(run)

        from familiar.skills.training.skill import curate_tool_calls

        tool_result = curate_tool_calls(
            {
                "min_quality": config.min_quality,
                "max_examples": config.max_tool_calls,
            }
        )
        run.stats["tool_calls"] = tool_result
        _update_run(run)

        # Step 3: Collect markdown memories
        if config.include_memories:
            run.status = "collecting_memories"
            _update_run(run)

            memories_result = _collect_memories()
            run.stats["memories"] = memories_result
            _update_run(run)

        # Step 4: Quality gate
        run.status = "quality_gate"
        _update_run(run)

        passed, gate_details = _quality_gate(config)
        run.stats["quality_gate"] = gate_details

        if not passed:
            run.status = "failed"
            run.error = f"Quality gate failed: {', '.join(gate_details.get('failure_reasons', []))}"
            run.completed_at = datetime.now().isoformat()
            _update_run(run)
            return run

        # Step 5: Export dataset
        run.status = "exporting"
        _update_run(run)

        from familiar.skills.training.skill import export_dataset

        sources = ["conversations", "tool_calls", "generated"]
        if config.include_memories:
            sources.append("memories")

        export_result = export_dataset(
            {
                "format": config.export_format,
                "min_quality": config.min_quality,
                "sources": sources,
            }
        )
        run.stats["export"] = export_result
        _update_run(run)

        # Step 6: Anchor to chain
        run.status = "anchoring"
        _update_run(run)

        try:
            from familiar.skills.training.bridge import get_chain_status

            chain_status = get_chain_status()
            run.stats["chain"] = chain_status
        except Exception as e:
            run.stats["chain"] = {"error": str(e)}
            logger.debug("Chain status check failed: %s", e)

        # Complete
        run.status = "complete"
        run.completed_at = datetime.now().isoformat()
        _update_run(run)
        return run

    except Exception as e:
        run.status = "failed"
        run.error = str(e)
        run.completed_at = datetime.now().isoformat()
        _update_run(run)
        logger.exception("Pipeline run %s failed", run_id)
        return run


def _collect_memories():
    """Collect markdown memories and write to curated/memories.jsonl."""
    try:
        from familiar.core.markdown_memory import MarkdownMemoryStore
        from familiar.core.paths import MEMORIES_DIR
    except ImportError:
        return "Markdown memory module not available."

    if not MEMORIES_DIR.exists():
        return "No memories directory found."

    store = MarkdownMemoryStore(MEMORIES_DIR)
    records = store.export_for_training()

    if not records:
        return "No memories to export for training."

    from familiar.skills.training.skill import _append_jsonl, _get_curated_dir

    # Convert memory records to training format
    training_records = []
    for rec in records:
        content = rec.get("content", "").strip()
        topic = rec.get("topic", "")
        if not content or len(content) < 20:
            continue

        training_records.append(
            {
                "id": rec.get("content_hash", "")[:16],
                "source_type": "memories",
                "instruction": f"What do you know about {topic}?",
                "response": content,
                "quality_score": rec.get("quality_score", 0.5),
                "quality_signals": {
                    "source": "markdown_memory",
                    "revision": rec.get("revision", 1),
                },
                "provenance": rec.get("provenance", {}),
            }
        )

    if training_records:
        _append_jsonl(_get_curated_dir() / "memories.jsonl", training_records)

    return f"Collected {len(training_records)} memory records for training."


def get_pipeline_status() -> dict:
    """Return latest run + history summary."""
    state = _load_pipeline_state()
    runs = state.get("runs", [])

    result = {
        "total_runs": len(runs),
        "latest_run": None,
        "history": [],
    }

    if runs:
        result["latest_run"] = runs[-1]
        # Summary of last 10 runs
        for r in runs[-10:]:
            result["history"].append(
                {
                    "run_id": r.get("run_id"),
                    "status": r.get("status"),
                    "started_at": r.get("started_at"),
                    "completed_at": r.get("completed_at"),
                    "error": r.get("error", ""),
                }
            )

    return result
