#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
═══════════════════════════════════════════════════════════════════════════════
 ARGUS — Moteur de Réflexion (Auto-analyse mémoire)
 Audite la santé et la qualité du système de mémoire
═══════════════════════════════════════════════════════════════════════════════

Usage:
  python reflection.py                 # Rapport complet
  python reflection.py --quiet         # Mode silencieux (exit code uniquement)
  python reflection.py --json          # Sortie JSON

Exit codes:
  0 = Mémoire saine
  1 = Problèmes détectés (warnings ou errors)
  2 = Erreur d'exécution

Checks (persona: memory.operations.reflection):
  - Cohérence : contradictions entre souvenirs ?
  - Complétude : zones d'ombre importantes ?
  - Fraîcheur : informations obsolètes ?
  - Connexité : éléments isolés / orphelins ?
  - Équilibre : distribution saine des connaissances ?
"""

import yaml
import sys
import argparse
import json
import re
import logging
from pathlib import Path
from typing import List, Any, Optional
from dataclasses import dataclass, field
from datetime import datetime
from collections import Counter

from config import LOW_CONFIDENCE, CONSOLIDATION_LAG_THRESHOLD

logger = logging.getLogger(__name__)


# ═══════════════════════════════════════════════════════════════════════════════
#  DATA CLASSES
# ═══════════════════════════════════════════════════════════════════════════════

@dataclass
class HealthCheck:
    """Un check de santé individuel."""
    name: str
    category: str               # coherence | completeness | freshness | connectivity | balance
    status: str                 # "ok" | "warning" | "error"
    message: str
    details: Any = None         # Détails optionnels
    recommendation: str = ""

    def to_dict(self) -> dict:
        d = {
            "name": self.name,
            "category": self.category,
            "status": self.status,
            "message": self.message,
        }
        if self.details:
            d["details"] = self.details
        if self.recommendation:
            d["recommendation"] = self.recommendation
        return d


@dataclass
class MemoryStats:
    """Statistiques globales de la mémoire."""
    patterns_count: int = 0
    heuristics_count: int = 0
    episodes_total: int = 0
    episodes_consolidated: int = 0
    episodes_unconsolidated: int = 0
    associations_count: int = 0
    projects_count: int = 0
    avg_pattern_confidence: float = 0.0
    avg_heuristic_confidence: float = 0.0
    low_confidence_patterns: int = 0
    low_confidence_heuristics: int = 0
    patterns_never_seen: int = 0
    oldest_activity: str = ""
    newest_activity: str = ""

    def to_dict(self) -> dict:
        return {
            "patterns_count": self.patterns_count,
            "heuristics_count": self.heuristics_count,
            "episodes_total": self.episodes_total,
            "episodes_consolidated": self.episodes_consolidated,
            "episodes_unconsolidated": self.episodes_unconsolidated,
            "associations_count": self.associations_count,
            "projects_count": self.projects_count,
            "avg_pattern_confidence": round(self.avg_pattern_confidence, 3),
            "avg_heuristic_confidence": round(self.avg_heuristic_confidence, 3),
            "low_confidence_patterns": self.low_confidence_patterns,
            "low_confidence_heuristics": self.low_confidence_heuristics,
            "patterns_never_seen": self.patterns_never_seen,
            "oldest_activity": self.oldest_activity,
            "newest_activity": self.newest_activity,
        }


@dataclass
class ReflectionReport:
    """Rapport de réflexion complet."""
    timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
    stats: MemoryStats = field(default_factory=MemoryStats)
    checks: List[HealthCheck] = field(default_factory=list)
    errors: List[str] = field(default_factory=list)

    @property
    def has_issues(self) -> bool:
        return any(c.status in ("warning", "error") for c in self.checks)

    @property
    def error_count(self) -> int:
        return sum(1 for c in self.checks if c.status == "error")

    @property
    def warning_count(self) -> int:
        return sum(1 for c in self.checks if c.status == "warning")

    @property
    def ok_count(self) -> int:
        return sum(1 for c in self.checks if c.status == "ok")

    def to_dict(self) -> dict:
        return {
            "timestamp": self.timestamp,
            "summary": {
                "total_checks": len(self.checks),
                "ok": self.ok_count,
                "warnings": self.warning_count,
                "errors": self.error_count,
                "health": "HEALTHY" if not self.has_issues else (
                    "DEGRADED" if self.error_count == 0 else "UNHEALTHY"
                ),
            },
            "stats": self.stats.to_dict(),
            "checks": [c.to_dict() for c in self.checks],
            "load_errors": self.errors,
        }


# ═══════════════════════════════════════════════════════════════════════════════
#  REFLECTION ENGINE
# ═══════════════════════════════════════════════════════════════════════════════

# ID regex patterns
PATTERN_ID_RE = re.compile(r'\bPAT-[A-Z]+-\d{3}\b')
HEURISTIC_ID_RE = re.compile(r'\bHEU-[A-Z]+-\d{3}\b')


class ReflectionEngine:
    """
    Moteur de réflexion — auto-analyse de la qualité mémoire.
    'La mémoire qui ne se questionne pas stagne.'
    """

    def __init__(self, base_dir: str, verbose: bool = True,
                 reference_date: Optional[datetime] = None):
        self.base_dir = Path(base_dir)
        self.memory_dir = self.base_dir / "memory"
        self.verbose = verbose
        self.reference_date = reference_date or datetime.now()
        self.report = ReflectionReport()

        # Caches
        self._patterns: List[dict] = []
        self._heuristics: List[dict] = []
        self._episodes: List[dict] = []
        self._episode_files: List[Path] = []
        self._associations: dict = {}
        self._index: dict = {}

    def log(self, message: str, force: bool = False):
        if self.verbose or force:
            logger.info(message)

    def _load_yaml(self, path: Path) -> Optional[dict]:
        try:
            return yaml.safe_load(path.read_text(encoding="utf-8"))
        except Exception as e:
            self.report.errors.append(f"Cannot load {path}: {e}")
            return None

    # ─────────────────────────────────────────────────────────────────────────
    #  DATA LOADING
    # ─────────────────────────────────────────────────────────────────────────

    def _load_all(self):
        """Charge toutes les données mémoire en cache.
        
        Charge une seule fois tous les fichiers nécessaires aux health checks :
        index, patterns, heuristiques, épisodes, associations et projets.
        Les données sont stockées dans les attributs self._* pour éviter
        les relectures multiples pendant l'audit.
        """
        # Index
        index_path = self.memory_dir / "_index.yaml"
        if index_path.exists():
            self._index = self._load_yaml(index_path) or {}

        # Patterns
        patterns_path = self.memory_dir / "semantic" / "defect_patterns.yaml"
        if patterns_path.exists():
            data = self._load_yaml(patterns_path)
            if data and "patterns" in data:
                self._patterns = data["patterns"]

        # Heuristics
        heuristics_path = self.memory_dir / "semantic" / "testing_heuristics.yaml"
        if heuristics_path.exists():
            data = self._load_yaml(heuristics_path)
            if data:
                for key, value in data.items():
                    if isinstance(value, list):
                        for h in value:
                            if isinstance(h, dict) and "id" in h:
                                self._heuristics.append(h)

        # Episodes
        episodic_dir = self.memory_dir / "episodic"
        if episodic_dir.exists():
            for subdir in ["analyses", "incidents", "decisions", "executions"]:
                dir_path = episodic_dir / subdir
                if not dir_path.exists():
                    continue
                for f in sorted(dir_path.glob("*.yaml")):
                    if f.name.startswith("_template"):
                        continue
                    data = self._load_yaml(f)
                    if data:
                        self._episodes.append(data)
                        self._episode_files.append(f)

        # Associations
        assoc_path = self.memory_dir / "relational" / "associations.yaml"
        if assoc_path.exists():
            self._associations = self._load_yaml(assoc_path) or {}

        # Projects
        projects_dir = self.memory_dir / "projects"
        project_count = 0
        if projects_dir.exists():
            for f in projects_dir.glob("*.yaml"):
                if not f.name.startswith("_template"):
                    project_count += 1
        self.report.stats.projects_count = project_count

    # ─────────────────────────────────────────────────────────────────────────
    #  STATISTICS
    # ─────────────────────────────────────────────────────────────────────────

    def _compute_stats(self):
        """Calcule les statistiques de la mémoire."""
        stats = self.report.stats

        # Patterns
        stats.patterns_count = len(self._patterns)
        if self._patterns:
            confidences = [p.get("confidence", 0.5) for p in self._patterns]
            stats.avg_pattern_confidence = sum(confidences) / len(confidences)
            stats.low_confidence_patterns = sum(1 for c in confidences if c < LOW_CONFIDENCE)
            stats.patterns_never_seen = sum(
                1 for p in self._patterns if not p.get("last_seen")
            )

        # Heuristics
        stats.heuristics_count = len(self._heuristics)
        if self._heuristics:
            confidences = [h.get("confidence", 0.5) for h in self._heuristics]
            stats.avg_heuristic_confidence = sum(confidences) / len(confidences)
            stats.low_confidence_heuristics = sum(1 for c in confidences if c < LOW_CONFIDENCE)

        # Episodes
        stats.episodes_total = len(self._episodes)
        stats.episodes_consolidated = sum(
            1 for ep in self._episodes
            if isinstance(ep.get("consolidation"), dict) and ep["consolidation"].get("consolidated")
        )
        stats.episodes_unconsolidated = stats.episodes_total - stats.episodes_consolidated

        # Associations
        total_assoc = 0
        for key in ["file_defects", "change_impact", "component_coupling",
                     "developer_area", "test_coverage_map", "defect_patterns",
                     "decision_outcomes"]:
            section = self._associations.get(key, {})
            if isinstance(section, dict):
                for list_key in ["entries", "correlations", "couplings",
                                 "mappings", "associations", "tracking"]:
                    items = section.get(list_key, [])
                    if isinstance(items, list):
                        total_assoc += len(items)
        stats.associations_count = total_assoc

    # ─────────────────────────────────────────────────────────────────────────
    #  HEALTH CHECKS
    # ─────────────────────────────────────────────────────────────────────────

    def _check_coherence(self):
        """Vérifie la cohérence interne de la mémoire.
        
        Checks :
        1. Les compteurs de _index.yaml reflètent les fichiers réels
        2. Les références croisées (source_episodes dans patterns) pointent
           vers des épisodes existants
        """
        # Check 1: Index counts vs actual counts
        index_stats = self._index.get("statistics", {}).get("totals", {})

        actual_episodes = len(self._episodes)
        indexed_episodes = index_stats.get("episodes", 0)
        if actual_episodes != indexed_episodes:
            self.report.checks.append(HealthCheck(
                name="index_episode_count",
                category="coherence",
                status="warning",
                message=f"Index says {indexed_episodes} episodes, found {actual_episodes}",
                recommendation="Run sync_counters.py --fix",
            ))
        else:
            self.report.checks.append(HealthCheck(
                name="index_episode_count",
                category="coherence",
                status="ok",
                message=f"Episode count matches ({actual_episodes})",
            ))

        actual_patterns = len(self._patterns)
        indexed_patterns = index_stats.get("patterns", 0)
        if actual_patterns != indexed_patterns:
            self.report.checks.append(HealthCheck(
                name="index_pattern_count",
                category="coherence",
                status="warning",
                message=f"Index says {indexed_patterns} patterns, found {actual_patterns}",
                recommendation="Run sync_counters.py --fix",
            ))
        else:
            self.report.checks.append(HealthCheck(
                name="index_pattern_count",
                category="coherence",
                status="ok",
                message=f"Pattern count matches ({actual_patterns})",
            ))

        actual_heuristics = len(self._heuristics)
        indexed_heuristics = index_stats.get("heuristics", 0)
        if actual_heuristics != indexed_heuristics:
            self.report.checks.append(HealthCheck(
                name="index_heuristic_count",
                category="coherence",
                status="warning",
                message=f"Index says {indexed_heuristics} heuristics, found {actual_heuristics}",
                recommendation="Run sync_counters.py --fix",
            ))
        else:
            self.report.checks.append(HealthCheck(
                name="index_heuristic_count",
                category="coherence",
                status="ok",
                message=f"Heuristic count matches ({actual_heuristics})",
            ))

        # Check 2: Cross-reference validity (patterns referenced in heuristics)
        orphan_refs = []

        for p in self._patterns:
            for ep in p.get("source_episodes", []) or []:
                ep_ids = {e.get("id") or e.get("metadata", {}).get("id", "") for e in self._episodes}
                if ep and ep not in ep_ids:
                    orphan_refs.append(f"{p.get('id')}.source_episodes → {ep}")

        if orphan_refs:
            self.report.checks.append(HealthCheck(
                name="cross_reference_validity",
                category="coherence",
                status="warning",
                message=f"{len(orphan_refs)} orphan reference(s) found",
                details=orphan_refs[:10],
                recommendation="Run validate_references.py to diagnose",
            ))
        else:
            self.report.checks.append(HealthCheck(
                name="cross_reference_validity",
                category="coherence",
                status="ok",
                message="All cross-references are valid",
            ))

    def _check_completeness(self):
        """Vérifie les lacunes dans la couverture mémoire.
        
        Checks :
        1. Diversité des types d'épisodes (analyses, incidents, décisions, exécutions)
        2. Peuplement de la mémoire relationnelle (associations)
        3. Patterns théoriques vs observés (last_seen renseigné ?)
        """
        # Check: Episode type distribution
        type_counts = Counter()
        for ep_path in self._episode_files:
            parts = ep_path.parts
            if "analyses" in parts:
                type_counts["analyses"] += 1
            elif "incidents" in parts:
                type_counts["incidents"] += 1
            elif "decisions" in parts:
                type_counts["decisions"] += 1
            elif "executions" in parts:
                type_counts["executions"] += 1

        empty_types = [t for t in ["analyses", "incidents", "decisions", "executions"]
                       if type_counts[t] == 0]

        if empty_types:
            self.report.checks.append(HealthCheck(
                name="episode_type_coverage",
                category="completeness",
                status="warning",
                message=f"No episodes in: {', '.join(empty_types)}",
                details=dict(type_counts),
                recommendation="Record decisions and incidents to build experience",
            ))
        else:
            self.report.checks.append(HealthCheck(
                name="episode_type_coverage",
                category="completeness",
                status="ok",
                message=f"All episode types have data: {dict(type_counts)}",
            ))

        # Check: Relational memory population
        stats = self.report.stats
        if stats.associations_count == 0 and stats.episodes_total > 0:
            self.report.checks.append(HealthCheck(
                name="relational_memory_population",
                category="completeness",
                status="warning",
                message="Relational memory is empty despite having episodes",
                recommendation="Run consolidation.py --apply to populate associations",
            ))
        elif stats.associations_count < 3:
            self.report.checks.append(HealthCheck(
                name="relational_memory_population",
                category="completeness",
                status="warning",
                message=f"Relational memory is sparse ({stats.associations_count} associations)",
                recommendation="Analyze more projects to build associations",
            ))
        else:
            self.report.checks.append(HealthCheck(
                name="relational_memory_population",
                category="completeness",
                status="ok",
                message=f"Relational memory has {stats.associations_count} associations",
            ))

        # Check: Patterns all theoretical (never seen in practice)
        if self.report.stats.patterns_never_seen == self.report.stats.patterns_count and \
           self.report.stats.patterns_count > 0:
            self.report.checks.append(HealthCheck(
                name="patterns_field_validation",
                category="completeness",
                status="warning",
                message="All patterns are theoretical (never seen in real episodes)",
                details={"never_seen": self.report.stats.patterns_never_seen},
                recommendation="Link patterns to incidents/analyses via consolidation",
            ))
        elif self.report.stats.patterns_never_seen > 0:
            self.report.checks.append(HealthCheck(
                name="patterns_field_validation",
                category="completeness",
                status="ok",
                message=f"{self.report.stats.patterns_never_seen}/{self.report.stats.patterns_count} "
                        f"patterns have no real-world observations yet",
            ))
        else:
            self.report.checks.append(HealthCheck(
                name="patterns_field_validation",
                category="completeness",
                status="ok",
                message="All patterns have real-world observations",
            ))

    def _check_freshness(self):
        """Vérifie la fraîcheur des données.
        
        Checks :
        1. Date de dernière consolidation (> 30 jours = warning)
        2. Backlog d'épisodes non consolidés (>= CONSOLIDATION_LAG_THRESHOLD = 5)
        3. Items à basse confidence (< LOW_CONFIDENCE = 0.40)
        """
        # Check: Last consolidation
        last_consolidation = self._index.get("statistics", {}).get("activity", {}).get("last_consolidation")
        if not last_consolidation:
            if self.report.stats.episodes_total > 0:
                self.report.checks.append(HealthCheck(
                    name="consolidation_recency",
                    category="freshness",
                    status="warning",
                    message="No consolidation has ever been run",
                    recommendation="Run consolidation.py --apply",
                ))
            else:
                self.report.checks.append(HealthCheck(
                    name="consolidation_recency",
                    category="freshness",
                    status="ok",
                    message="No consolidation needed (no episodes yet)",
                ))
        else:
            try:
                last_date = datetime.strptime(str(last_consolidation), "%Y-%m-%d")
                days_ago = (self.reference_date - last_date).days
                if days_ago > 30:
                    self.report.checks.append(HealthCheck(
                        name="consolidation_recency",
                        category="freshness",
                        status="warning",
                        message=f"Last consolidation was {days_ago} days ago",
                        recommendation="Run consolidation.py --apply",
                    ))
                else:
                    self.report.checks.append(HealthCheck(
                        name="consolidation_recency",
                        category="freshness",
                        status="ok",
                        message=f"Last consolidation {days_ago} days ago",
                    ))
            except ValueError:
                self.report.checks.append(HealthCheck(
                    name="consolidation_recency",
                    category="freshness",
                    status="warning",
                    message=f"Cannot parse consolidation date: {last_consolidation}",
                ))

        # Check: Unconsolidated episodes backlog
        if self.report.stats.episodes_unconsolidated >= CONSOLIDATION_LAG_THRESHOLD:
            self.report.checks.append(HealthCheck(
                name="consolidation_backlog",
                category="freshness",
                status="warning",
                message=f"{self.report.stats.episodes_unconsolidated} unconsolidated episodes",
                recommendation="Run consolidation.py --apply to process backlog",
            ))
        else:
            self.report.checks.append(HealthCheck(
                name="consolidation_backlog",
                category="freshness",
                status="ok",
                message=f"{self.report.stats.episodes_unconsolidated} unconsolidated episode(s)",
            ))

        # Check: Low confidence items
        total_low = (self.report.stats.low_confidence_patterns +
                     self.report.stats.low_confidence_heuristics)
        if total_low > 0:
            self.report.checks.append(HealthCheck(
                name="low_confidence_items",
                category="freshness",
                status="warning",
                message=f"{total_low} items below confidence threshold ({LOW_CONFIDENCE})",
                details={
                    "low_confidence_patterns": self.report.stats.low_confidence_patterns,
                    "low_confidence_heuristics": self.report.stats.low_confidence_heuristics,
                },
                recommendation="Run decay.py to review; reinforce or remove stale knowledge",
            ))
        else:
            self.report.checks.append(HealthCheck(
                name="low_confidence_items",
                category="freshness",
                status="ok",
                message="All knowledge items above confidence threshold",
            ))

    def _check_balance(self):
        """Vérifie l'équilibre du système de connaissances.
        
        Checks :
        1. Diversité des catégories de patterns (< 3 catégories pour > 5 patterns = warning)
        2. Ratio connaissances / expérience (beaucoup de théorie sans épisodes = warning)
        """
        # Check: Pattern category distribution
        categories = Counter(p.get("category", "unknown") for p in self._patterns)
        if len(categories) < 3 and len(self._patterns) > 5:
            self.report.checks.append(HealthCheck(
                name="pattern_category_diversity",
                category="balance",
                status="warning",
                message=f"Low category diversity ({len(categories)} categories for "
                        f"{len(self._patterns)} patterns)",
                details=dict(categories),
                recommendation="Consider adding patterns in underrepresented categories",
            ))
        else:
            self.report.checks.append(HealthCheck(
                name="pattern_category_diversity",
                category="balance",
                status="ok",
                message=f"{len(categories)} categories across {len(self._patterns)} patterns",
                details=dict(categories),
            ))

        # Check: Knowledge vs experience ratio
        knowledge_items = self.report.stats.patterns_count + self.report.stats.heuristics_count
        experience_items = self.report.stats.episodes_total
        if knowledge_items > 20 and experience_items == 0:
            self.report.checks.append(HealthCheck(
                name="knowledge_experience_ratio",
                category="balance",
                status="warning",
                message=f"{knowledge_items} knowledge items but 0 episodes (all theoretical)",
                recommendation="Start recording real analyses and incidents",
            ))
        elif knowledge_items > 0 and experience_items > 0:
            ratio = experience_items / knowledge_items
            self.report.checks.append(HealthCheck(
                name="knowledge_experience_ratio",
                category="balance",
                status="ok",
                message=f"Knowledge/experience ratio: {knowledge_items} items / {experience_items} episodes ({ratio:.1f})",
            ))
        else:
            self.report.checks.append(HealthCheck(
                name="knowledge_experience_ratio",
                category="balance",
                status="ok",
                message=f"{knowledge_items} knowledge items, {experience_items} episodes",
            ))

    # ─────────────────────────────────────────────────────────────────────────
    #  MAIN WORKFLOW
    # ─────────────────────────────────────────────────────────────────────────

    def reflect(self) -> ReflectionReport:
        """Exécute la réflexion complète."""
        self.log("\n═══ ARGUS — Memory Reflection ═══\n")

        # Load all data
        self._load_all()
        self._compute_stats()

        # Run all checks
        self._check_coherence()
        self._check_completeness()
        self._check_freshness()
        self._check_balance()

        # Print summary
        self.log(f"\n  ── Statistics ──")
        self.log(f"  Patterns: {self.report.stats.patterns_count} "
                 f"(avg confidence: {self.report.stats.avg_pattern_confidence:.2f})")
        self.log(f"  Heuristics: {self.report.stats.heuristics_count} "
                 f"(avg confidence: {self.report.stats.avg_heuristic_confidence:.2f})")
        self.log(f"  Episodes: {self.report.stats.episodes_total} "
                 f"({self.report.stats.episodes_consolidated} consolidated)")
        self.log(f"  Associations: {self.report.stats.associations_count}")
        self.log(f"  Projects: {self.report.stats.projects_count}")

        self.log(f"\n  ── Health Checks ({len(self.report.checks)}) ──")
        for c in self.report.checks:
            icon = {"ok": "✓", "warning": "⚠", "error": "✗"}[c.status]
            self.log(f"  {icon} [{c.category}] {c.name}: {c.message}")
            if c.recommendation and c.status != "ok":
                self.log(f"    → {c.recommendation}")

        health = "HEALTHY" if not self.report.has_issues else (
            "DEGRADED" if self.report.error_count == 0 else "UNHEALTHY"
        )
        self.log(f"\n  Overall: {health} "
                 f"({self.report.ok_count} ok, {self.report.warning_count} warnings, "
                 f"{self.report.error_count} errors)")

        return self.report


# ═══════════════════════════════════════════════════════════════════════════════
#  CLI
# ═══════════════════════════════════════════════════════════════════════════════

def main():
    logging.basicConfig(level=logging.INFO, format='%(message)s')
    parser = argparse.ArgumentParser(
        description="Argus — Memory Reflection Engine"
    )
    parser.add_argument(
        "--quiet", action="store_true",
        help="Suppress output (exit code only)"
    )
    parser.add_argument(
        "--json", action="store_true",
        help="Output JSON report"
    )
    parser.add_argument(
        "--base-dir", type=str, default=None,
        help="Project root directory"
    )
    parser.add_argument(
        "--reference-date", type=str, default=None,
        help="Reference date (YYYY-MM-DD). Default: today"
    )
    args = parser.parse_args()

    base_dir = args.base_dir or str(Path(__file__).parent.parent)
    verbose = not args.quiet and not args.json

    ref_date = None
    if args.reference_date:
        ref_date = datetime.strptime(args.reference_date, "%Y-%m-%d")

    engine = ReflectionEngine(base_dir, verbose=verbose, reference_date=ref_date)

    try:
        report = engine.reflect()
    except Exception as e:
        if args.json:
            print(json.dumps({"error": str(e)}, indent=2))
        elif not args.quiet:
            print(f"\n  ✗ Error: {e}", file=sys.stderr)
        sys.exit(2)

    if args.json:
        print(json.dumps(report.to_dict(), indent=2, ensure_ascii=False))

    sys.exit(1 if report.has_issues else 0)


if __name__ == "__main__":
    main()
