# Familiar - Self-Hosted AI Companion Platform
# Copyright (c) 2026 George Scott Foley
# Licensed under the MIT License

"""
Tests for familiar/skills/training/corpus.py

Covers:
  - Frontmatter parse/write round-trip
  - PHI gate (scrub + reject)
  - Quality scorer weights and Dross boost
  - MinHash deduplicator (unique / near-dup / exact-dup)
  - Consent registry (grant / revoke / version check)
  - CorpusPipeline (end-to-end with temp dir)
  - Tool handlers (smoke tests)
"""

import json
import os
import shutil
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch

# ── Module under test ──
from familiar.skills.training.corpus import (
    CURRENT_CONSENT_VERSION,
    DEDUP_JACCARD_THRESHOLD,
    CorpusEntry,
    CorpusPipeline,
    SemanticDeduplicator,
    TrainingFrontmatter,
    _jaccard_estimate,
    _minhash_signature,
    _shingles,
    check_corpus_consent,
    corpus_status,
    get_corpus_status,
    grant_corpus_consent,
    manage_corpus_consent,
    mark_memory_training_eligible,
    parse_frontmatter,
    phi_scrub,
    record_corpus_run,
    revoke_corpus_consent,
    run_corpus_pipeline,
    score_memory,
    write_frontmatter,
)

# ══════════════════════════════════════════════════════════════
# Helpers
# ══════════════════════════════════════════════════════════════


def _temp_dir():
    return Path(tempfile.mkdtemp())


def _write_memory(directory: Path, name: str, content: str) -> Path:
    p = directory / name
    p.write_text(content, encoding="utf-8")
    return p


def _eligible_md(topic: str = "test topic", body: str = "x" * 200) -> str:
    return (
        f"---\n"
        f'topic: "{topic}"\n'
        f"training_eligible: true\n"
        f'consent_version: "{CURRENT_CONSENT_VERSION}"\n'
        f'domain: "general"\n'
        f'pair_type: "document"\n'
        f"revision: 2\n"
        f"---\n\n{body}"
    )


# ══════════════════════════════════════════════════════════════
# 1. Frontmatter parse / write
# ══════════════════════════════════════════════════════════════


class TestFrontmatterParse(unittest.TestCase):
    def test_no_frontmatter(self):
        fm, body = parse_frontmatter("Just some text.")
        self.assertFalse(fm.training_eligible)
        self.assertEqual(body, "Just some text.")

    def test_basic_frontmatter(self):
        text = (
            "---\n"
            "topic: cooking\n"
            "training_eligible: true\n"
            f"consent_version: {CURRENT_CONSENT_VERSION}\n"
            "revision: 3\n"
            "---\n\nbody text"
        )
        fm, body = parse_frontmatter(text)
        self.assertTrue(fm.training_eligible)
        self.assertEqual(fm.consent_version, CURRENT_CONSENT_VERSION)
        self.assertEqual(fm.revision, 3)
        self.assertEqual(body, "body text")

    def test_quoted_values(self):
        text = '---\ntopic: "my topic"\ndomain: "legal"\n---\nbody'
        fm, body = parse_frontmatter(text)
        self.assertEqual(fm.topic, "my topic")
        self.assertEqual(fm.domain, "legal")

    def test_write_roundtrip(self):
        original = (
            "---\n"
            'topic: "recipes"\n'
            "training_eligible: true\n"
            f'consent_version: "{CURRENT_CONSENT_VERSION}"\n'
            'domain: "cooking"\n'
            'pair_type: "document"\n'
            "---\n\nsome body"
        )
        fm, body = parse_frontmatter(original)
        roundtrip = write_frontmatter(fm, body)
        fm2, body2 = parse_frontmatter(roundtrip)
        self.assertEqual(fm.topic, fm2.topic)
        self.assertEqual(fm.training_eligible, fm2.training_eligible)
        self.assertEqual(body, body2)

    def test_unclosed_frontmatter(self):
        text = "---\ntopic: x\nbody without close"
        fm, body = parse_frontmatter(text)
        self.assertFalse(fm.training_eligible)
        self.assertEqual(body, text)

    def test_write_includes_training_eligible_false(self):
        fm = TrainingFrontmatter(topic="t")
        result = write_frontmatter(fm, "body")
        self.assertIn("training_eligible: false", result)


# ══════════════════════════════════════════════════════════════
# 2. PHI gate
# ══════════════════════════════════════════════════════════════


class TestPhiScrub(unittest.TestCase):
    def test_no_phi(self):
        text = "This is a recipe for chicken soup with vegetables."
        scrubbed, found = phi_scrub(text)
        # phi_detection skill may or may not be available; fallback must not corrupt text
        self.assertIsInstance(scrubbed, str)
        self.assertIsInstance(found, bool)

    def test_email_detected_by_fallback(self):
        # Force fallback by patching out the phi_detection import
        with patch.dict("sys.modules", {"familiar.skills.phi_detection.skill": None}):
            text = "Contact us at doctor@hospital.org for info."
            scrubbed, found = phi_scrub(text)
            self.assertTrue(found)
            self.assertNotIn("doctor@hospital.org", scrubbed)

    def test_phone_detected_by_fallback(self):
        with patch.dict("sys.modules", {"familiar.skills.phi_detection.skill": None}):
            text = "Call 555-867-5309 for an appointment."
            scrubbed, found = phi_scrub(text)
            self.assertTrue(found)
            self.assertNotIn("555-867-5309", scrubbed)

    def test_clean_text_not_modified(self):
        with patch.dict("sys.modules", {"familiar.skills.phi_detection.skill": None}):
            text = "The quick brown fox jumps over the lazy dog."
            scrubbed, found = phi_scrub(text)
            self.assertFalse(found)
            self.assertEqual(scrubbed, text)


# ══════════════════════════════════════════════════════════════
# 3. Quality scorer
# ══════════════════════════════════════════════════════════════


class TestScoreMemory(unittest.TestCase):
    def test_short_empty_memory_is_low(self):
        fm = TrainingFrontmatter(revision=1)
        score = score_memory("hi", fm)
        self.assertLess(score, 0.3)

    def test_rich_memory_scores_higher(self):
        body = (
            "# How to make pasta\n\n"
            "1. Boil water\n2. Add pasta\n3. Cook 8 minutes\n\n"
            "```bash\npip install pasta\n```\n\n" + "Detailed notes. " * 30
        )
        fm = TrainingFrontmatter(topic="pasta", created_at="2026-01-01", revision=4)
        score = score_memory(body, fm)
        self.assertGreater(score, 0.5)

    def test_dross_usage_boosts_score(self):
        body = "Some content " * 20
        fm = TrainingFrontmatter(topic="foo")
        score_no_usage = score_memory(body, fm, usage_count=0, dross_earned=0.0)
        score_with_usage = score_memory(body, fm, usage_count=10, dross_earned=2.0)
        self.assertGreater(score_with_usage, score_no_usage)

    def test_score_clamped_to_unit_interval(self):
        body = "# Big doc\n\n```python\nprint('hello')\n```\n\n" + "Long body. " * 100
        fm = TrainingFrontmatter(topic="t", created_at="2026-01-01", revision=10)
        score = score_memory(body, fm, usage_count=20, dross_earned=5.0)
        self.assertLessEqual(score, 1.0)
        self.assertGreaterEqual(score, 0.0)


# ══════════════════════════════════════════════════════════════
# 4. MinHash deduplicator
# ══════════════════════════════════════════════════════════════


class TestMinHash(unittest.TestCase):
    def test_shingles_short_text(self):
        s = _shingles("hello world")
        self.assertTrue(len(s) >= 1)

    def test_shingles_k_gram(self):
        s = _shingles("the quick brown fox jumps", k=3)
        self.assertIn("the quick brown", s)
        self.assertIn("quick brown fox", s)

    def test_signature_length(self):
        sig = _minhash_signature({"a b c", "d e f"}, num_hashes=32)
        self.assertEqual(len(sig), 32)

    def test_identical_texts_same_signature(self):
        text = "hello world this is a test"
        shingles = _shingles(text)
        sig1 = _minhash_signature(shingles)
        sig2 = _minhash_signature(shingles)
        self.assertEqual(sig1, sig2)

    def test_jaccard_identical(self):
        text = "the quick brown fox jumps over the lazy dog"
        sig = _minhash_signature(_shingles(text))
        j = _jaccard_estimate(sig, sig)
        self.assertAlmostEqual(j, 1.0, places=1)

    def test_jaccard_very_different(self):
        sig_a = _minhash_signature(_shingles("hello world foo bar baz qux"))
        sig_b = _minhash_signature(_shingles("xyz abc def ghi jkl mno pqr"))
        j = _jaccard_estimate(sig_a, sig_b)
        self.assertLess(j, 0.3)


class TestSemanticDeduplicator(unittest.TestCase):
    def test_unique_documents(self):
        dedup = SemanticDeduplicator()
        text_a = "This is a recipe for chocolate cake with flour and butter " * 5
        text_b = "Python programming tutorial on decorators and metaclasses " * 5
        dedup.add("a", text_a)
        result = dedup.is_duplicate("b", text_b)
        self.assertIsNone(result)

    def test_near_duplicate_detected(self):
        dedup = SemanticDeduplicator()
        # Use a long document where one word changes — >85% shingles remain identical
        words = (
            "the quick brown fox jumped over the lazy dog near the river bank "
            "and then ran through the forest looking for food and shelter "
        ) * 6  # 168 words → ~165 shingles
        text_a = words
        # Change just one word — >95% shingles stay the same
        text_b = words.replace("jumped", "leaped", 1)
        dedup.add("a", text_a)
        result = dedup.is_duplicate("b", text_b)
        self.assertEqual(result, "a")

    def test_same_id_not_self_duplicate(self):
        dedup = SemanticDeduplicator()
        text = "hello world foo bar baz " * 10
        dedup.add("doc1", text)
        result = dedup.is_duplicate("doc1", text)
        self.assertIsNone(result)

    def test_size_tracking(self):
        dedup = SemanticDeduplicator()
        self.assertEqual(dedup.size(), 0)
        dedup.add("a", "some text here and there")
        self.assertEqual(dedup.size(), 1)


# ══════════════════════════════════════════════════════════════
# 5. Consent registry
# ══════════════════════════════════════════════════════════════


class TestConsentRegistry(unittest.TestCase):
    def setUp(self):
        self._tmp = _temp_dir()
        # Patch training dir to tmp
        self._patch = patch(
            "familiar.skills.training.corpus._get_training_dir",
            return_value=self._tmp,
        )
        self._patch.start()

    def tearDown(self):
        self._patch.stop()
        shutil.rmtree(self._tmp, ignore_errors=True)

    def test_no_consent_by_default(self):
        self.assertFalse(check_corpus_consent("user1"))

    def test_grant_then_check(self):
        grant_corpus_consent("user1")
        self.assertTrue(check_corpus_consent("user1"))

    def test_revoke_removes_consent(self):
        grant_corpus_consent("user1")
        revoke_corpus_consent("user1")
        self.assertFalse(check_corpus_consent("user1"))

    def test_wrong_version_not_active(self):
        grant_corpus_consent("user1", consent_version="0.9")
        self.assertFalse(check_corpus_consent("user1"))

    def test_multiple_users_independent(self):
        grant_corpus_consent("alice")
        self.assertTrue(check_corpus_consent("alice"))
        self.assertFalse(check_corpus_consent("bob"))


# ══════════════════════════════════════════════════════════════
# 6. CorpusPipeline end-to-end
# ══════════════════════════════════════════════════════════════


class TestCorpusPipeline(unittest.TestCase):
    def setUp(self):
        self._tmp = _temp_dir()
        self._memories = self._tmp / "memories"
        self._memories.mkdir()
        self._corpus = self._tmp / "corpus"

        # Patch training dir and memories dir
        self._p1 = patch(
            "familiar.skills.training.corpus._get_training_dir",
            return_value=self._tmp,
        )
        self._p1.start()

    def tearDown(self):
        self._p1.stop()
        shutil.rmtree(self._tmp, ignore_errors=True)

    def _pipeline(self, **kwargs) -> CorpusPipeline:
        grant_corpus_consent("default")
        return CorpusPipeline(
            memories_dir=self._memories,
            user_id="default",
            **kwargs,
        )

    def test_empty_memories_dir(self):
        p = self._pipeline()
        summary = p.run()
        self.assertEqual(summary["total_scanned"], 0)
        self.assertEqual(summary["exported"], 0)

    def test_ineligible_memory_skipped(self):
        _write_memory(self._memories, "a.md", "---\ntopic: x\ntraining_eligible: false\n---\nbody")
        p = self._pipeline()
        summary = p.run()
        self.assertEqual(summary["eligible"], 0)

    def test_eligible_memory_exported(self):
        _write_memory(self._memories, "good.md", _eligible_md(body="x" * 300))
        p = self._pipeline(min_quality=0.0)
        summary = p.run()
        self.assertEqual(summary["eligible"], 1)
        self.assertEqual(summary["exported"], 1)

    def test_phi_rejected_when_reject_phi_true(self):
        body = "Call Dr Smith at 555-867-5309 for appointments. " + "x" * 200
        _write_memory(self._memories, "phi.md", _eligible_md(body=body))
        with patch.dict("sys.modules", {"familiar.skills.phi_detection.skill": None}):
            p = self._pipeline(reject_phi=True, min_quality=0.0)
            summary = p.run()
        self.assertEqual(summary["phi_rejected"], 1)
        self.assertEqual(summary["exported"], 0)

    def test_phi_scrubbed_when_reject_phi_false(self):
        body = "Email us at secret@example.com for more info. " + "x" * 200
        _write_memory(self._memories, "phi2.md", _eligible_md(body=body))
        with patch.dict("sys.modules", {"familiar.skills.phi_detection.skill": None}):
            p = self._pipeline(reject_phi=False, min_quality=0.0)
            summary = p.run()
        # Should be exported with PHI scrubbed out
        self.assertEqual(summary["exported"], 1)

    def test_quality_gate(self):
        _write_memory(self._memories, "short.md", _eligible_md(body="too short"))
        p = self._pipeline(min_quality=0.8)
        summary = p.run()
        self.assertEqual(summary["quality_rejected"], 1)

    def test_dedup_skips_near_duplicate(self):
        # Long body with minimal word change so >85% shingles are identical
        base_words = (
            "the quick brown fox jumped over the lazy dog near the river bank "
            "and then ran through the forest looking for food and shelter "
        ) * 6
        body_a = base_words
        body_b = base_words.replace("jumped", "leaped", 1)
        _write_memory(self._memories, "a.md", _eligible_md(topic="doc a", body=body_a))
        _write_memory(self._memories, "b.md", _eligible_md(topic="doc b", body=body_b))
        p = self._pipeline(min_quality=0.0)
        summary = p.run()
        # One should be exported, one deduped
        self.assertEqual(summary["exported"] + summary["dedup_skipped"], 2)
        self.assertGreaterEqual(summary["dedup_skipped"], 1)

    def test_no_consent_returns_error(self):
        # Revoke any existing consent
        revoke_corpus_consent("default")
        p = CorpusPipeline(memories_dir=self._memories, user_id="default")
        summary = p.run()
        self.assertIn("error", summary)
        self.assertIn("consent", summary["error"].lower())

    def test_output_jsonl_written(self):
        _write_memory(self._memories, "doc.md", _eligible_md(body="x" * 300))
        p = self._pipeline(min_quality=0.0)
        p.run()
        output = self._tmp / "corpus" / "corpus_memories.jsonl"
        self.assertTrue(output.exists())
        with open(output) as f:
            lines = [l for l in f if l.strip()]
        self.assertEqual(len(lines), 1)
        entry = json.loads(lines[0])
        self.assertEqual(entry["source_type"], "corpus_memory")
        self.assertIn("quality_score", entry)
        self.assertEqual(entry["consent_version"], CURRENT_CONSENT_VERSION)

    def test_corpus_entry_has_contributor_hash(self):
        _write_memory(self._memories, "doc.md", _eligible_md(body="y" * 300))
        p = self._pipeline(min_quality=0.0)
        p.run()
        output = self._tmp / "corpus" / "corpus_memories.jsonl"
        entry = json.loads(output.read_text().strip())
        self.assertTrue(entry.get("contributor_hash"))
        self.assertNotEqual(entry["contributor_hash"], "default")  # must be hashed


# ══════════════════════════════════════════════════════════════
# 7. Status and run tracking
# ══════════════════════════════════════════════════════════════


class TestCorpusStatus(unittest.TestCase):
    def setUp(self):
        self._tmp = _temp_dir()
        self._p = patch(
            "familiar.skills.training.corpus._get_training_dir",
            return_value=self._tmp,
        )
        self._p.start()

    def tearDown(self):
        self._p.stop()
        shutil.rmtree(self._tmp, ignore_errors=True)

    def test_status_no_runs(self):
        status = get_corpus_status()
        self.assertEqual(status["total_runs"], 0)
        self.assertIsNone(status["latest_run"])

    def test_record_and_retrieve_run(self):
        record_corpus_run({"exported": 5, "phi_rejected": 1, "dedup_skipped": 2})
        status = get_corpus_status()
        self.assertEqual(status["total_runs"], 1)
        self.assertEqual(status["latest_run"]["exported"], 5)

    def test_corpus_status_tool_handler_smoke(self):
        result = corpus_status({})
        self.assertIn("Corpus Pipeline Status", result)
        self.assertIn(CURRENT_CONSENT_VERSION, result)


# ══════════════════════════════════════════════════════════════
# 8. Tool handler smoke tests
# ══════════════════════════════════════════════════════════════


class TestToolHandlers(unittest.TestCase):
    def setUp(self):
        self._tmp = _temp_dir()
        self._p = patch(
            "familiar.skills.training.corpus._get_training_dir",
            return_value=self._tmp,
        )
        self._p.start()

    def tearDown(self):
        self._p.stop()
        shutil.rmtree(self._tmp, ignore_errors=True)

    def test_manage_corpus_consent_grant(self):
        result = manage_corpus_consent({"action": "grant", "user_id": "alice"})
        self.assertIn("granted", result)
        self.assertIn(CURRENT_CONSENT_VERSION, result)

    def test_manage_corpus_consent_revoke(self):
        manage_corpus_consent({"action": "grant", "user_id": "alice"})
        result = manage_corpus_consent({"action": "revoke", "user_id": "alice"})
        self.assertIn("revoked", result)

    def test_manage_corpus_consent_invalid_action(self):
        result = manage_corpus_consent({"action": "delete"})
        self.assertIn("Unknown action", result)

    def test_mark_memory_training_eligible_missing_file(self):
        result = mark_memory_training_eligible({"file_path": "/nonexistent/path.md"})
        self.assertIn("Error", result)

    def test_mark_memory_training_eligible_creates_frontmatter(self):
        md = self._tmp / "note.md"
        md.write_text("Just a plain note without any frontmatter.", encoding="utf-8")
        result = mark_memory_training_eligible({"file_path": str(md), "domain": "cooking"})
        self.assertIn("training-eligible", result)
        updated = md.read_text()
        self.assertIn("training_eligible: true", updated)
        self.assertIn(CURRENT_CONSENT_VERSION, updated)
        self.assertIn("cooking", updated)

    def test_run_corpus_pipeline_no_consent(self):
        result = run_corpus_pipeline({"user_id": "nobody"})
        self.assertIn("failed", result.lower())

    def test_run_corpus_pipeline_with_consent(self):
        grant_corpus_consent("default")
        memories = self._tmp / "memories"
        memories.mkdir()
        (memories / "doc.md").write_text(_eligible_md(body="z" * 300), encoding="utf-8")
        with patch(
            "familiar.skills.training.corpus.CorpusPipeline._default_memories_dir",
            return_value=memories,
        ):
            result = run_corpus_pipeline({"user_id": "default", "min_quality": "0.0"})
        self.assertIn("Corpus Pipeline", result)


if __name__ == "__main__":
    unittest.main()
