"""Comprehensive end-to-end test for PipelineOrchestrator.

This test validates the pipeline execution infrastructure by running the complete
8-stage production pipeline through the orchestrator. It covers:

INFRASTRUCTURE TESTED:
- PipelineOrchestrator.execute() - orchestrates stage execution
- PipelineStage.run() - full stage lifecycle (prepare/compute/accumulate/finalise/save)
- Channel - connects stages via ports
- Phase callbacks - tracks execution phases (prepare/compute/finalise/save)
- Cache persistence - atomic_pickle_dump saves results
- Cache loading - atomic_pickle_load restores results (second run)
- Progress tracking - orchestrator monitors stage progress
- Port binding - OutputPort → Channel → InputPort flow
- Review data - both photos (ComputeIdentical) and sequences (similarity stages)

PRODUCTION PIPELINE (8 stages):
Stage 0: ComputeShaBins - Hash files and bin by SHA256
Stage 1: ComputeIdentical - Find byte-identical duplicates (produces photo review data)
Stage 2: ComputeTemplates - Bin photos by filename template
Stage 3: ComputeVersions - Detect version patterns in filenames
Stage 4: ComputeTemplateSimilarity - Match photos with similar templates (produces sequence review data)
Stage 5: ComputeIndices - Find sequences with overlapping indices
Stage 6: ComputePerceptualHash - Compute perceptual hashes and bin
Stage 7: ComputePerceptualMatch - Match photos by perceptual hash similarity

This is NOT testing stage logic (comprehensive tests do that).
This IS testing the orchestration framework that runs stages.
"""

import shutil
import tempfile
import time
from pathlib import Path

from PIL import Image

from orchestrator import build_pipeline
from utils import InputPort, atomic_pickle_load


def create_test_photo_files(photo_dir: Path) -> None:
    """Create minimal test photos as actual files on disk.

    Creates test images with patterns that exercise the full pipeline:
    - 2 identical files → triggers ComputeIdentical (photo review)
    - 12 photos in similar templates → triggers TemplateSimilarity (sequence review)
    - 4 version sequences → triggers Compute Versions

    Args:
        photo_dir: Directory to create test photos in
    """
    # Create minimal 32x32 test images

    # Pattern 1: Two identical files (triggers ComputeIdentical review)
    img1 = Image.new("RGB", (32, 32), color=(255, 0, 0))
    img1.save(photo_dir / "red_001.jpg", quality=95)
    img1.save(photo_dir / "red_002.jpg", quality=95)  # Identical content

    # Pattern 2: Similar templates (triggers TemplateSimilarity)
    # Create 3 sequences with 4 photos each (12 total photos)
    for seq_num in range(1, 5):
        # Template 1: IMG_###.jpg
        img = Image.new("RGB", (32, 32), color=(seq_num * 60, 0, 0))
        img.save(photo_dir / f"IMG_{seq_num:03d}.jpg", quality=95)

        # Template 2: IMAGE_###.jpg
        img = Image.new("RGB", (32, 32), color=(0, seq_num * 60, 0))
        img.save(photo_dir / f"IMAGE_{seq_num:03d}.jpg", quality=95)

        # Template 3: PIC_###.jpg
        img = Image.new("RGB", (32, 32), color=(0, 0, seq_num * 60))
        img.save(photo_dir / f"PIC_{seq_num:03d}.jpg", quality=95)

    # Pattern 3: Version sequences (triggers ComputeVersions)
    # Create 3 versions of the same photo
    for i in range(3):
        img = Image.new("RGB", (32, 32), color=(128, 128, 128))
        img.save(photo_dir / f"photo_001_v{i}.jpg", quality=95)


def test_orchestrator_fresh_execution() -> None:
    """Test orchestrator executing complete pipeline without cached data.

    This test validates:
    - PipelineOrchestrator.execute() runs all 8 stages
    - Each stage.run() completes full lifecycle
    - Phase callbacks track execution phases
    - Channels connect stages correctly
    - Cache files are created (atomic_pickle_dump called)
    - Both review types available (photos + sequences)
    """
    print("\n" + "=" * 70)
    print("Orchestrator Test 1: Fresh Execution (No Cache)")
    print("=" * 70)

    # Create temporary directories
    photo_dir = Path(tempfile.mkdtemp(prefix="orchestrator_photos_"))
    work_dir = Path(tempfile.mkdtemp(prefix="orchestrator_cache_"))
    print(f"\nPhoto directory: {photo_dir}")
    print(f"Cache directory: {work_dir}")

    try:
        # Create test photos as actual files
        print("\nCreating test photo files...")
        create_test_photo_files(photo_dir)
        photo_count = len(list(photo_dir.glob("*.jpg")))
        print(f"  Created {photo_count} test photos")

        # Build production pipeline
        print("\nBuilding production pipeline (8 stages)...")
        orchestrator = build_pipeline(photo_dir)
        print(f"  Pipeline has {len(orchestrator.graph.get_stages_in_order())} stages")

        # Override all cache paths to use test work directory
        for stage in orchestrator.graph.get_stages_in_order():
            stage.path = work_dir / f"{stage.stage_name.replace(' ', '_').lower()}.pkl"

        # Track phase callbacks
        phases_seen: list[str] = []

        def track_phases(phase: str) -> None:
            phases_seen.append(phase)

        original_update = orchestrator._update_phase

        def tracked_update(phase: str) -> None:
            track_phases(phase)
            original_update(phase)

        orchestrator._update_phase = tracked_update  # type: ignore[method-assign]

        # Execute pipeline
        print("\n[1/4] Executing pipeline (fresh, no cache)...")
        start_time = time.perf_counter()
        orchestrator.execute()
        elapsed = time.perf_counter() - start_time
        print(f"  Pipeline completed in {elapsed:.2f}s")

        # Validate execution status
        print("\n[2/4] Validating execution status...")
        assert not orchestrator.failed, "Pipeline should not have failed"
        assert orchestrator._is_pipeline_complete(), "Pipeline should be complete"
        print("  [OK] Pipeline completed successfully")

        # Validate phase callbacks were called
        print("\n[3/4] Validating phase callbacks...")
        expected_phases = ["prepare", "compute", "finalise", "save"]
        num_stages = len(orchestrator.graph.get_stages_in_order())
        assert len(phases_seen) == len(expected_phases) * num_stages, (
            f"Expected {len(expected_phases) * num_stages} phase callbacks ({num_stages} stages x 4 phases), "
            f"got {len(phases_seen)}: {phases_seen}"
        )

        # Check each stage went through expected phases
        for stage_idx in range(num_stages):
            stage_phases = phases_seen[stage_idx * 4 : (stage_idx + 1) * 4]
            assert stage_phases == expected_phases, f"Stage {stage_idx} phases incorrect: {stage_phases}"
        print(f"  [OK] All phase callbacks fired: {len(phases_seen)} total")

        # Validate cache files created
        print("\n[4/7] Validating cache files...")
        cache_files = list(work_dir.glob("*.pkl"))
        assert len(cache_files) == num_stages, f"Expected {num_stages} cache files, found {len(cache_files)}"
        print(f"  [OK] Cache files created: {len(cache_files)} files")

        # Get stages for validation
        stages = orchestrator.graph.get_stages_in_order()

        # Validate cache persistence (Priority 1: atomic_pickle_dump/load coverage)
        print("\n[5/7] Validating cache persistence...")

        for stage in stages:
            assert stage.path.exists(), f"{stage.stage_name} cache not created"

            # Load cache to verify structure (tests atomic_pickle_load)
            loaded: tuple[object, ...] = atomic_pickle_load(stage.path)

            # Verify structure is correct 7-tuple (result, seq_review, id_review, photos, seqs, elapsed, throughput)
            assert isinstance(loaded, tuple), f"{stage.stage_name} cache is not a tuple"
            assert len(loaded) == 7, f"{stage.stage_name} cache tuple has {len(loaded)} items, expected 7"

            _result, seq_review, id_review, photos_final, seqs_final, elapsed_seconds, throughput = loaded

            # Verify types (sufficient to prove caching worked)
            assert isinstance(seq_review, list), f"{stage.stage_name} seq_review wrong type"
            assert isinstance(id_review, list), f"{stage.stage_name} id_review wrong type"
            assert photos_final is None or isinstance(photos_final, int), f"{stage.stage_name} photos_final wrong type"
            assert seqs_final is None or isinstance(seqs_final, int), f"{stage.stage_name} seqs_final wrong type"

        print(f"  [OK] All {len(stages)} cache files verified loadable")

        # Validate port binding (Priority 2: channel/port infrastructure)
        print("\n[6/7] Validating port binding...")

        ports_checked = 0
        for stage in stages[1:]:  # Skip stage 0 (ComputeShaBins has no inputs)
            # Find all InputPort attributes
            for attr_name in dir(stage):
                if attr_name.startswith("_"):
                    continue
                attr = getattr(stage, attr_name)
                if isinstance(attr, InputPort):
                    assert attr.is_bound(), f"{stage.stage_name}.{attr_name} not bound"
                    assert attr._source is not None, f"{stage.stage_name}.{attr_name} has no source"
                    ports_checked += 1

        print(f"  [OK] All {ports_checked} input ports bound correctly")

        # Validate both review types
        print("\n[7/7] Validating review data...")

        # Stage 1 (ComputeIdentical) should have photo review data
        identical_stage = stages[1]
        assert identical_stage.needs_review() == "photos", (
            f"ComputeIdentical should produce photo review, got {identical_stage.needs_review()}"
        )
        assert identical_stage.has_review_data(), "ComputeIdentical should have review data"
        print(f"  Photo review groups: {len(identical_stage.identical_review_result)}")

        # Check if any stage has sequence review data (not required - depends on photo similarity)
        sequence_stages = [
            stage for stage in stages if stage.needs_review() == "sequences" and stage.has_review_data()
        ]
        if sequence_stages:
            print(f"  Sequence review groups: {len(sequence_stages[0].sequence_review_result)}")
        else:
            print("  No sequence reviews produced (photos not similar enough to merge)")

        print("\n" + "=" * 70)
        print("Test 1 Complete: Fresh Execution")
        print("=" * 70)
        print(f"[PASS] Orchestrator executed {num_stages} stages successfully")

    finally:
        # Cleanup
        print("\nCleaning up directories...")
        shutil.rmtree(photo_dir, ignore_errors=True)
        shutil.rmtree(work_dir, ignore_errors=True)


def test_orchestrator_cached_execution() -> None:
    """Test orchestrator executing pipeline with cached data.

    This test validates:
    - Cached stages load via atomic_pickle_load
    - Stages skip work when cache is valid
    - Phase callback "cache_load" is fired
    - Results are identical to fresh execution
    - Cache validity checking works correctly
    """
    print("\n" + "=" * 70)
    print("Orchestrator Test 2: Cached Execution (Load from Cache)")
    print("=" * 70)

    # Create temporary directories
    photo_dir = Path(tempfile.mkdtemp(prefix="orchestrator_photos_"))
    work_dir = Path(tempfile.mkdtemp(prefix="orchestrator_cache_"))
    print(f"\nPhoto directory: {photo_dir}")
    print(f"Cache directory: {work_dir}")

    try:
        # Create test photos as actual files
        print("\nCreating test photo files...")
        create_test_photo_files(photo_dir)
        photo_count = len(list(photo_dir.glob("*.jpg")))
        print(f"  Created {photo_count} test photos")

        # First execution: Create cache
        print("\n[1/4] First execution: Creating cache...")
        orchestrator1 = build_pipeline(photo_dir)

        # Override cache paths
        for stage in orchestrator1.graph.get_stages_in_order():
            stage.path = work_dir / f"{stage.stage_name.replace(' ', '_').lower()}.pkl"

        orchestrator1.execute()
        print("  [OK] Cache created")

        # Capture first results
        stages1 = orchestrator1.graph.get_stages_in_order()
        num_stages = len(stages1)

        # Store results from all stages for comparison
        results1 = [stage.result for stage in stages1]  # type: ignore[attr-defined]

        # Second execution: Load from cache
        print("\n[2/4] Second execution: Loading from cache...")
        orchestrator2 = build_pipeline(photo_dir)

        # Override cache paths (same as first run)
        for stage in orchestrator2.graph.get_stages_in_order():
            stage.path = work_dir / f"{stage.stage_name.replace(' ', '_').lower()}.pkl"

        # Track phase callbacks
        phases_seen: list[str] = []

        def track_phases(phase: str) -> None:
            phases_seen.append(phase)

        original_update = orchestrator2._update_phase

        def tracked_update(phase: str) -> None:
            track_phases(phase)
            original_update(phase)

        orchestrator2._update_phase = tracked_update  # type: ignore[method-assign]

        start_time = time.perf_counter()
        orchestrator2.execute()
        elapsed = time.perf_counter() - start_time
        print(f"  Pipeline completed in {elapsed:.2f}s (should be fast - loaded from cache)")

        # Validate cache_load phases
        print("\n[3/4] Validating cache loading...")
        cache_load_count = phases_seen.count("cache_load")
        assert cache_load_count == num_stages, (
            f"Expected {num_stages} 'cache_load' phase callbacks (one per stage), got {cache_load_count}"
        )
        print(f"  [OK] All {num_stages} stages loaded from cache")

        # Should NOT see prepare/compute/finalise/save (stages skipped work)
        work_phases = ["prepare", "compute", "finalise", "save"]
        for phase in work_phases:
            count = phases_seen.count(phase)
            assert count == 0, f"Phase '{phase}' should not fire when loading from cache, but fired {count} times"
        print("  [OK] No work phases fired (stages used cache)")

        # Validate results loaded (checking structure, not deep equality)
        print("\n[4/4] Validating cached results loaded...")
        stages2 = orchestrator2.graph.get_stages_in_order()

        # Verify all stages have results (proves cache was loaded)
        for stage in stages2:
            assert stage.result is not None, f"{stage.stage_name} has no result after cache load"  # type: ignore[attr-defined]

        # Verify result types match (sufficient proof of cache loading)
        assert isinstance(stages2[0].result, type(results1[0])), "Stage 0 result type mismatch"  # type: ignore[attr-defined]
        assert len(stages2) == len(results1), "Stage count mismatch"

        print(f"  [OK] All {len(stages2)} stages loaded results from cache")

        print("\n" + "=" * 70)
        print("Test 2 Complete: Cached Execution")
        print("=" * 70)
        print(f"[PASS] Orchestrator loaded {num_stages} stages from cache successfully")

    finally:
        # Cleanup
        print("\nCleaning up directories...")
        shutil.rmtree(photo_dir, ignore_errors=True)
        shutil.rmtree(work_dir, ignore_errors=True)


def test_orchestrator_review_api_simulation() -> None:
    """Test REST API review data access patterns.

    Simulates the complete flow used by /api/review/* endpoints:
    1. Review availability discovery
    2. Review data loading (per-stage)
    3. Session storage
    4. Pagination
    5. Review type handling (photos vs sequences)
    """
    print("\n" + "=" * 70)
    print("Orchestrator Test 3: Review API Simulation")
    print("=" * 70)

    # Create temporary directories
    photo_dir = Path(tempfile.mkdtemp(prefix="review_test_"))
    work_dir = Path(tempfile.mkdtemp(prefix="review_cache_"))
    print(f"\nPhoto directory: {photo_dir}")
    print(f"Cache directory: {work_dir}")

    try:
        # Create test photos and build pipeline
        print("\nCreating test photo files...")
        create_test_photo_files(photo_dir)
        photo_count = len(list(photo_dir.glob("*.jpg")))
        print(f"  Created {photo_count} test photos")

        print("\nBuilding production pipeline...")
        orchestrator = build_pipeline(photo_dir)

        # Override cache paths
        for stage in orchestrator.graph.get_stages_in_order():
            stage.path = work_dir / f"{stage.stage_name.replace(' ', '_').lower()}.pkl"

        # Execute pipeline
        print("\nExecuting pipeline...")
        orchestrator.execute()
        stages = orchestrator.graph.get_stages_in_order()

        # [PART 1] Simulate /api/review/availability
        print("\n[1/5] Testing review availability endpoint...")

        availability_map = {}
        for stage in stages:
            review_type = stage.needs_review()
            if review_type != "none":
                availability_map[stage.stage_id] = {
                    "available": stage.has_review_data(),
                    "review_type": review_type,
                }

        # Verify both review types present
        photo_stages = [sid for sid, info in availability_map.items() if info["review_type"] == "photos"]
        seq_stages = [sid for sid, info in availability_map.items() if info["review_type"] == "sequences"]

        assert len(photo_stages) >= 1, "Should have at least 1 photo review stage"
        assert len(seq_stages) >= 1, "Should have at least 1 sequence review stage"
        print(f"  Found {len(photo_stages)} photo stages, {len(seq_stages)} sequence stages")

        # [PART 2] Simulate /api/review/load for ComputeIdentical (photos)
        print("\n[2/5] Testing photo review loading (ComputeIdentical)...")

        # Find ComputeIdentical stage
        identical_stage = next(s for s in stages if s.stage_name == "Byte-identical detection")
        assert identical_stage.needs_review() == "photos", (
            f"ComputeIdentical should produce photo review, got {identical_stage.needs_review()}"
        )
        assert identical_stage.has_review_data(), "ComputeIdentical should have review data"

        # Load review data (simulates endpoint logic)
        photo_groups = identical_stage.identical_review_result
        assert len(photo_groups) > 0, "Should have identical groups from test photos"
        print(f"  Loaded {len(photo_groups)} identical groups")

        # Create mock session storage
        review_sessions = {}
        review_sessions[identical_stage.stage_id] = {
            "identical_groups": photo_groups,
            "sequence_groups": [],
            "review_type": "photos",
            "stage_name": identical_stage.stage_name,
        }

        # [PART 3] Simulate /api/review/identical/groups pagination
        print("\n[3/5] Testing photo review pagination...")

        # Test pagination edge cases
        total_groups = len(photo_groups)
        page_size = 100

        # Page 0 (first page)
        page = 0
        start_idx = page * page_size
        end_idx = min(start_idx + page_size, total_groups)
        page_groups = photo_groups[start_idx:end_idx]
        has_more = end_idx < total_groups

        assert len(page_groups) == min(page_size, total_groups)
        print(f"  Page 0: {len(page_groups)} groups, has_more={has_more}")

        # [PART 4] Simulate /api/review/load for sequence stage
        print("\n[4/5] Testing sequence review loading...")

        # Find first stage with actual non-empty sequence review data
        seq_stage = next(
            (s for s in stages if s.needs_review() == "sequences" and len(s.sequence_review_result) > 0),
            None,
        )

        # It's okay if no stages have sequence review data (depends on test data)
        # What matters is the test verifies the API pattern works when data exists
        if seq_stage is not None:
            # Load sequence groups
            seq_groups = seq_stage.sequence_review_result
            assert len(seq_groups) > 0, "Should have sequence groups"
            print(f"  Loaded {len(seq_groups)} sequence groups from {seq_stage.stage_name}")

            # Store in sessions (simulate REST API pattern)
            review_sessions[seq_stage.stage_id] = {
                "identical_groups": [],
                "sequence_groups": seq_groups,
                "review_type": "sequences",
                "stage_name": seq_stage.stage_name,
            }
        else:
            # Log that no sequence review data was generated
            # This is valid - not all pipelines produce sequence groups for review
            print("  [SKIP] No sequence stages produced review data (test data dependent)")

        # [PART 5] Test photofiles access (shared globally)
        print("\n[5/5] Testing photofiles access...")

        photofiles = orchestrator.get_photofiles()
        assert isinstance(photofiles, dict), "get_photofiles() should return dict"
        assert len(photofiles) > 0, "Should have photofiles"
        print(f"  Loaded {len(photofiles)} photofiles globally")

        # Verify photofiles structure
        first_photo = next(iter(photofiles.values()))
        assert hasattr(first_photo, "path"), "PhotoFile should have path"
        assert hasattr(first_photo, "id"), "PhotoFile should have id"

        print("\n" + "=" * 70)
        print("Test 3 Complete: Review API Simulation")
        print("=" * 70)
        print("[PASS] All review API patterns verified")

    finally:
        # Cleanup
        print("\nCleaning up directories...")
        shutil.rmtree(photo_dir, ignore_errors=True)
        shutil.rmtree(work_dir, ignore_errors=True)


def test_orchestrator_review_error_cases() -> None:
    """Test review data edge cases and error handling.

    Validates that orchestrator gracefully handles:
    1. Accessing review data before pipeline execution
    2. Stages with no review data
    3. Empty review results
    """
    print("\n" + "=" * 70)
    print("Orchestrator Test 4: Review Error Cases")
    print("=" * 70)

    # Case 1: Access review data before pipeline execution
    photo_dir = Path(tempfile.mkdtemp(prefix="error_test_"))
    try:
        print("\n[1/3] Testing review data before execution...")
        create_test_photo_files(photo_dir)
        orchestrator = build_pipeline(photo_dir)

        # Before execute() - stages should have no review data
        stages = orchestrator.graph.get_stages_in_order()
        for stage in stages:
            if stage.needs_review() != "none":
                # has_review_data() should handle empty case gracefully
                assert not stage.has_review_data(), f"{stage.stage_name} should have no review data before execution"

        print("  [OK] Empty review data handled correctly")

        # Case 2: Verify stages with needs_review() == "none" never have review data
        print("\n[2/3] Testing stages with no review requirement...")
        orchestrator.execute()

        no_review_stages = [s for s in stages if s.needs_review() == "none"]
        for stage in no_review_stages:
            assert not stage.has_review_data(), f"{stage.stage_name} should never produce review data"
            assert len(stage.identical_review_result) == 0, (
                f"{stage.stage_name} should have empty identical_review_result"
            )
            assert len(stage.sequence_review_result) == 0, (
                f"{stage.stage_name} should have empty sequence_review_result"
            )

        print(f"  [OK] Verified {len(no_review_stages)} stages with no review requirement")

        # Case 3: Verify get_photofiles() works after execution
        print("\n[3/3] Testing photofiles access after execution...")

        # After execution, photofiles should be available
        photofiles = orchestrator.get_photofiles()
        assert isinstance(photofiles, dict), "get_photofiles() should return dict"
        assert len(photofiles) > 0, "Should have photofiles after execution"

        print(f"  [OK] Photofiles accessible after execution: {len(photofiles)} files")

        print("\n" + "=" * 70)
        print("Test 4 Complete: Review Error Cases")
        print("=" * 70)
        print("[PASS] All error cases handled gracefully")

    finally:
        # Cleanup
        print("\nCleaning up directories...")
        shutil.rmtree(photo_dir, ignore_errors=True)


if __name__ == "__main__":
    test_orchestrator_fresh_execution()
    test_orchestrator_cached_execution()
    test_orchestrator_review_api_simulation()
    test_orchestrator_review_error_cases()
