"""
#exonware/xwnode/examples/x5/data_operations/test_9_index_operations.py
INDEX Operations Test Suite
Tests all INDEX operations (performance optimization) for V2 (Indexed) implementation.
Note: V1 (Streaming) does not support indexing, so these tests focus on V2.
All tests are fully implemented at production level with no TODOs.
Company: eXonware.com
Author: eXonware Backend Team
Email: connect@exonware.com
Version: 0.0.1
Generation Date: 11-Oct-2025
"""

import sys
import os
import json
import time
from pathlib import Path
from typing import Any
# Import test helpers
sys.path.insert(0, str(Path(__file__).parent))
from test_helpers import (
    create_test_file,
    cleanup_test_file,
    get_all_matching_v1,
    get_all_matching_v2,
    count_records_v1,
    count_records_v2,
)
# Import from parent
sys.path.insert(0, str(Path(__file__).parent.parent))
from json_utils import (
    match_by_id,
    stream_read,
)
from json_utils_indexed import (
    build_index,
    ensure_index,
    load_index,
    indexed_get_by_line,
    indexed_get_by_id,
    get_page,
    JsonIndex,
)
# ============================================================================
# 9.1 Index Creation
# ============================================================================


def test_9_1_1_build_index():
    """
    Full Test Name: test_9_1_1_build_index
    Test: Create index for file
    """
    test_data = [{"id": str(i), "name": f"User{i}"} for i in range(10)]
    file_path = create_test_file(test_data)
    try:
        # V1: No index support (baseline)
        v1_start = time.perf_counter()
        v1_count = count_records_v1(file_path)
        v1_time = time.perf_counter() - v1_start
        # V2: Build index
        v2_start = time.perf_counter()
        index = build_index(file_path, id_field="id")
        v2_time = time.perf_counter() - v2_start
        assert index is not None
        assert len(index.line_offsets) == 10
        assert index.id_index is not None
        assert len(index.id_index) == 10
        assert v1_count == len(index.line_offsets)
        return True, v1_time, v2_time
    finally:
        cleanup_test_file(file_path)


def test_9_1_2_build_id_index():
    """
    Full Test Name: test_9_1_2_build_id_index
    Test: Create index on ID field
    """
    test_data = [{"id": str(i), "name": f"User{i}"} for i in range(10)]
    file_path = create_test_file(test_data)
    try:
        # V2: Build ID index
        v2_start = time.perf_counter()
        index = build_index(file_path, id_field="id")
        v2_time = time.perf_counter() - v2_start
        assert index.id_index is not None
        assert "0" in index.id_index
        assert "5" in index.id_index
        assert "9" in index.id_index
        assert index.id_index["0"] == 0
        assert index.id_index["9"] == 9
        # Test lookup using index
        result = indexed_get_by_id(file_path, "5", id_field="id", index=index)
        assert result["id"] == "5"
        return True, 0.0, v2_time
    finally:
        cleanup_test_file(file_path)


def test_9_1_3_build_field_index():
    """
    Full Test Name: test_9_1_3_build_field_index
    Test: Create index on specific field
    """
    test_data = [
        {"id": "1", "role": "admin"},
        {"id": "2", "role": "user"},
        {"id": "3", "role": "admin"}
    ]
    file_path = create_test_file(test_data)
    try:
        # V2: Build field index (using role as id_field for indexing)
        v2_start = time.perf_counter()
        index = build_index(file_path, id_field="role")
        v2_time = time.perf_counter() - v2_start
        assert index.id_index is not None
        assert "admin" in index.id_index
        assert "user" in index.id_index
        # Note: Multiple records with same role will only index the first one
        # This is expected behavior for id_index
        return True, 0.0, v2_time
    finally:
        cleanup_test_file(file_path)


def test_9_1_4_build_composite_index():
    """
    Full Test Name: test_9_1_4_build_composite_index
    Test: Create index on multiple fields
    """
    test_data = [
        {"id": "1", "user_id": "u1", "session_id": "s1"},
        {"id": "2", "user_id": "u1", "session_id": "s2"},
        {"id": "3", "user_id": "u2", "session_id": "s1"}
    ]
    file_path = create_test_file(test_data)
    try:
        # V2: Build composite index (simulate by creating composite key)
        # Since build_index only supports single id_field, we simulate composite
        # by creating a combined key in the data
        test_data_composite = [
            {"id": "1", "composite_key": "u1_s1", "user_id": "u1", "session_id": "s1"},
            {"id": "2", "composite_key": "u1_s2", "user_id": "u1", "session_id": "s2"},
            {"id": "3", "composite_key": "u2_s1", "user_id": "u2", "session_id": "s1"}
        ]
        cleanup_test_file(file_path)
        file_path = create_test_file(test_data_composite)
        v2_start = time.perf_counter()
        index = build_index(file_path, id_field="composite_key")
        v2_time = time.perf_counter() - v2_start
        assert index.id_index is not None
        assert "u1_s1" in index.id_index
        assert "u1_s2" in index.id_index
        assert "u2_s1" in index.id_index
        return True, 0.0, v2_time
    finally:
        cleanup_test_file(file_path)


def test_9_1_5_build_partial_index():
    """
    Full Test Name: test_9_1_5_build_partial_index
    Test: Create index on filtered subset
    """
    test_data = [
        {"id": "1", "role": "admin", "active": True},
        {"id": "2", "role": "user", "active": True},
        {"id": "3", "role": "admin", "active": False}
    ]
    file_path = create_test_file(test_data)
    try:
        # V2: Build partial index (only on active records)
        # Since build_index doesn't support filtering, we simulate by
        # building index and then filtering results
        v2_start = time.perf_counter()
        index = build_index(file_path, id_field="id")
        # Filter to only active records
        active_indices = []
        for i in range(len(index.line_offsets)):
            record = indexed_get_by_line(file_path, i, index=index)
            if record.get("active"):
                active_indices.append(i)
        v2_time = time.perf_counter() - v2_start
        assert len(active_indices) == 2
        assert index is not None
        return True, 0.0, v2_time
    finally:
        cleanup_test_file(file_path)
# ============================================================================
# 9.2 Index Maintenance
# ============================================================================


def test_9_2_1_rebuild_index():
    """
    Full Test Name: test_9_2_1_rebuild_index
    Test: Recreate index from scratch
    """
    test_data = [{"id": str(i), "name": f"User{i}"} for i in range(10)]
    file_path = create_test_file(test_data)
    try:
        # V2: Rebuild index
        index1 = build_index(file_path, id_field="id")
        v2_start = time.perf_counter()
        index2 = build_index(file_path, id_field="id")  # Rebuild
        v2_time = time.perf_counter() - v2_start
        assert index2 is not None
        assert len(index2.line_offsets) == len(index1.line_offsets)
        assert index2.id_index == index1.id_index
        return True, 0.0, v2_time
    finally:
        cleanup_test_file(file_path)


def test_9_2_2_update_index():
    """
    Full Test Name: test_9_2_2_update_index
    Test: Incrementally update index
    """
    test_data = [{"id": str(i), "name": f"User{i}"} for i in range(5)]
    file_path = create_test_file(test_data)
    try:
        # V2: Update index (rebuild after changes)
        index1 = build_index(file_path, id_field="id")
        # Add new record
        from test_helpers import append_record_v2
        append_record_v2(file_path, {"id": "5", "name": "User5"})
        v2_start = time.perf_counter()
        index2 = build_index(file_path, id_field="id")  # Rebuild to update
        v2_time = time.perf_counter() - v2_start
        assert len(index2.line_offsets) == 6
        assert "5" in index2.id_index
        assert len(index2.id_index) == 6
        return True, 0.0, v2_time
    finally:
        cleanup_test_file(file_path)


def test_9_2_3_validate_index():
    """
    Full Test Name: test_9_2_3_validate_index
    Test: Check index integrity
    """
    test_data = [{"id": str(i), "name": f"User{i}"} for i in range(10)]
    file_path = create_test_file(test_data)
    try:
        # V2: Validate index
        index = build_index(file_path, id_field="id")
        v2_start = time.perf_counter()
        # Validate by checking consistency
        is_valid = (
            index is not None and
            len(index.line_offsets) == 10 and
            index.id_index is not None and
            len(index.id_index) == 10 and
            all(str(i) in index.id_index for i in range(10))
        )
        v2_time = time.perf_counter() - v2_start
        assert is_valid
        # Test load_index validation
        loaded_index = load_index(file_path, strict=True)
        assert loaded_index is not None
        assert len(loaded_index.line_offsets) == 10
        return True, 0.0, v2_time
    finally:
        cleanup_test_file(file_path)


def test_9_2_4_drop_index():
    """
    Full Test Name: test_9_2_4_drop_index
    Test: Remove index
    """
    test_data = [{"id": str(i), "name": f"User{i}"} for i in range(10)]
    file_path = create_test_file(test_data)
    try:
        # V2: Drop index
        index = build_index(file_path, id_field="id")
        idx_path = file_path + ".idx.json"
        assert os.path.exists(idx_path)
        v2_start = time.perf_counter()
        if os.path.exists(idx_path):
            os.remove(idx_path)
        v2_time = time.perf_counter() - v2_start
        assert not os.path.exists(idx_path)
        # Verify index is gone
        loaded_index = load_index(file_path, strict=True)
        assert loaded_index is None
        return True, 0.0, v2_time
    finally:
        cleanup_test_file(file_path)


def test_9_2_5_index_statistics():
    """
    Full Test Name: test_9_2_5_index_statistics
    Test: Get index usage statistics
    """
    test_data = [{"id": str(i), "name": f"User{i}"} for i in range(10)]
    file_path = create_test_file(test_data)
    try:
        # V2: Index statistics
        index = build_index(file_path, id_field="id")
        v2_start = time.perf_counter()
        stats = {
            "total_records": len(index.line_offsets),
            "indexed_ids": len(index.id_index) if index.id_index else 0,
            "index_size_bytes": os.path.getsize(file_path + ".idx.json") if os.path.exists(file_path + ".idx.json") else 0,
            "has_id_index": index.id_index is not None
        }
        v2_time = time.perf_counter() - v2_start
        assert stats["total_records"] == 10
        assert stats["indexed_ids"] == 10
        assert stats["has_id_index"] is True
        assert stats["index_size_bytes"] > 0
        return True, 0.0, v2_time
    finally:
        cleanup_test_file(file_path)
# ============================================================================
# 9.3 Index Usage
# ============================================================================


def test_9_3_1_use_index_for_lookup():
    """
    Full Test Name: test_9_3_1_use_index_for_lookup
    Test: Leverage index for fast access
    """
    test_data = [{"id": str(i), "name": f"User{i}"} for i in range(100)]
    file_path = create_test_file(test_data)
    try:
        # V1: No index (baseline)
        v1_start = time.perf_counter()
        v1_result = stream_read(file_path, match_by_id("id", "50"))
        v1_time = time.perf_counter() - v1_start
        # V2: Use index for lookup
        index = build_index(file_path, id_field="id")
        v2_start = time.perf_counter()
        v2_result = indexed_get_by_id(file_path, "50", id_field="id", index=index)
        v2_time = time.perf_counter() - v2_start
        assert v1_result == v2_result
        assert v1_result["id"] == "50"
        return True, v1_time, v2_time
    finally:
        cleanup_test_file(file_path)


def test_9_3_2_use_index_for_range():
    """
    Full Test Name: test_9_3_2_use_index_for_range
    Test: Use index for range queries
    """
    test_data = [{"id": str(i), "value": i * 10} for i in range(20)]
    file_path = create_test_file(test_data)
    try:
        # V2: Use index for range queries
        index = build_index(file_path, id_field="id")
        v2_start = time.perf_counter()
        # Get records in range using indexed line access
        range_results = []
        for i in range(5, 15):  # Range 5-14
            record = indexed_get_by_line(file_path, i, index=index)
            if 50 <= record.get("value", 0) <= 140:
                range_results.append(record)
        v2_time = time.perf_counter() - v2_start
        assert len(range_results) == 10
        assert all(50 <= r.get("value", 0) <= 140 for r in range_results)
        return True, 0.0, v2_time
    finally:
        cleanup_test_file(file_path)


def test_9_3_3_use_index_for_sorting():
    """
    Full Test Name: test_9_3_3_use_index_for_sorting
    Test: Use index for sorted results
    """
    test_data = [
        {"id": "3", "name": "Charlie"},
        {"id": "1", "name": "Alice"},
        {"id": "2", "name": "Bob"}
    ]
    file_path = create_test_file(test_data)
    try:
        # V2: Use index for sorting
        index = build_index(file_path, id_field="id")
        v2_start = time.perf_counter()
        # Get records in sorted order using indexed access
        sorted_results = []
        for i in range(len(index.line_offsets)):
            record = indexed_get_by_line(file_path, i, index=index)
            sorted_results.append(record)
        # Sort by ID
        sorted_results.sort(key=lambda x: x.get("id", ""))
        v2_time = time.perf_counter() - v2_start
        assert sorted_results[0]["id"] == "1"
        assert sorted_results[1]["id"] == "2"
        assert sorted_results[2]["id"] == "3"
        return True, 0.0, v2_time
    finally:
        cleanup_test_file(file_path)


def test_9_3_4_index_hint():
    """
    Full Test Name: test_9_3_4_index_hint
    Test: Force use of specific index
    """
    test_data = [{"id": str(i), "name": f"User{i}"} for i in range(10)]
    file_path = create_test_file(test_data)
    try:
        # V2: Index hint (force use of index by explicitly passing it)
        index = build_index(file_path, id_field="id")
        v2_start = time.perf_counter()
        # Force use of index by explicitly passing it
        result = indexed_get_by_id(file_path, "5", id_field="id", index=index)
        v2_time = time.perf_counter() - v2_start
        assert result["id"] == "5"
        assert index is not None
        return True, 0.0, v2_time
    finally:
        cleanup_test_file(file_path)
