#!/usr/bin/env python3
"""
Comprehensive RAG Pipeline Test Suite

This test suite comprehensively tests the entire RAG (Retrieval-Augmented Generation)
pipeline using EndeeVectorStore. It covers all major functions and use cases from
vectorstores.py including:

1. Vector Store Creation & Configuration
   - Creating new indexes with different configurations
   - Connecting to existing indexes
   - Index validation and recreation

2. Document Management
   - Adding texts and documents
   - Batch operations
   - Text truncation handling
   - Metadata management

3. Search Operations
   - Similarity search (with and without scores)
   - Search by vector (with and without scores)
   - Filtered searches
   - Hybrid search (dense + sparse)

4. CRUD Operations
   - Create (add_texts, add_documents)
   - Read (get_by_ids, similarity_search)
   - Update (via upsert)
   - Delete (by IDs and filters)

5. Advanced Features
   - Retriever integration
   - Multiple filter conditions
   - Precision levels
   - Distance metrics
   - Batch processing

6. Factory Methods
   - from_texts()
   - from_documents()
   - from_existing_index()

This test suite validates the complete RAG workflow from document ingestion
to retrieval and demonstrates production-ready usage patterns.
"""

import unittest
import uuid
import time
from typing import List

from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings

from langchain_endee import EndeeVectorStore, RetrievalMode, Precision
from langchain_endee.sparse_embeddings import SparseEmbeddings, SparseVector
from tests.setup_class import EndeeLangChainTestSetup


# ==============================================================================
# Mock Embeddings for Testing
# ==============================================================================

class MockDenseEmbeddings(Embeddings):
    """Mock dense embeddings for testing."""

    def __init__(self, dimension: int = 384):
        self.dimension = dimension

    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Generate deterministic embeddings based on text hash."""
        embeddings = []
        for text in texts:
            hash_val = hash(text)
            embedding = [(hash_val + i) % 100 / 100.0 for i in range(self.dimension)]
            embeddings.append(embedding)
        return embeddings

    def embed_query(self, text: str) -> List[float]:
        """Generate embedding for query."""
        return self.embed_documents([text])[0]


class MockSparseEmbeddings(SparseEmbeddings):
    """Mock sparse embeddings for testing hybrid search."""

    def __init__(self, sparse_dim: int = 1000):
        self.sparse_dim = sparse_dim

    def embed_documents(self, texts: List[str]) -> List[SparseVector]:
        """Generate mock sparse embeddings."""
        sparse_vectors = []
        for text in texts:
            words = text.lower().split()
            indices = sorted(list(set([hash(word) % self.sparse_dim for word in words[:20]])))
            values = [1.0 / (i + 1) for i in range(len(indices))]
            sparse_vectors.append(SparseVector(indices=indices, values=values))
        return sparse_vectors

    def embed_query(self, text: str) -> SparseVector:
        """Generate sparse embedding for query."""
        return self.embed_documents([text])[0]

    def get_sparse_dim(self) -> int:
        """Get the sparse dimensionality."""
        return self.sparse_dim


# ==============================================================================
# Test Data - Rich RAG Dataset
# ==============================================================================

RAG_TEST_DOCUMENTS = [
    # Technology & Programming
    "Python is a high-level, interpreted programming language known for its simplicity and readability. It supports multiple programming paradigms including object-oriented, imperative, and functional programming.",

    "JavaScript is the programming language of the web, enabling interactive web pages. Node.js extends JavaScript to server-side development, making it a full-stack language.",

    "Rust is a systems programming language focused on safety, speed, and concurrency. It achieves memory safety without garbage collection through its ownership system.",

    # AI & Machine Learning
    "Machine learning is a subset of artificial intelligence that enables systems to automatically learn and improve from experience without being explicitly programmed.",

    "Deep learning uses artificial neural networks with multiple layers to progressively extract higher-level features from raw input data. It powers modern AI applications.",

    "Natural language processing (NLP) enables computers to understand, interpret, and generate human language. It combines linguistics, computer science, and AI.",

    "Transformer models revolutionized NLP by using self-attention mechanisms to process sequential data in parallel. GPT, BERT, and T5 are based on transformers.",

    # Vector Databases & RAG
    "Vector databases are specialized systems designed to store and retrieve high-dimensional vector embeddings efficiently. They enable semantic search and similarity matching.",

    "Retrieval-Augmented Generation (RAG) combines retrieval-based methods with generative models. It retrieves relevant documents from a knowledge base to generate accurate responses.",

    "Embeddings are numerical representations of text in continuous vector space. They capture semantic meaning, allowing similar items to be close together in the space.",

    "Hybrid search combines dense embeddings (semantic search) with sparse embeddings (keyword search) for improved retrieval accuracy. It balances semantic understanding with exact matching.",

    # Data Science & Engineering
    "Data engineering involves designing and building systems for collecting, storing, and analyzing data at scale. It enables data science and analytics workflows.",

    "Feature engineering is the process of selecting, manipulating, and transforming raw data into features for machine learning models. Good features improve model performance.",

    # Cloud & Infrastructure
    "Docker containers package software with all dependencies, ensuring consistency across different environments. They enable portable and reproducible deployments.",

    "Kubernetes orchestrates containerized applications, automating deployment, scaling, and management across clusters. It's the de facto standard for container orchestration.",
]

RAG_TEST_METADATAS = [
    # Technology & Programming
    {"category": "programming", "language": "python", "difficulty": "beginner", "tags": ["coding", "general-purpose"], "year": 2023},
    {"category": "programming", "language": "javascript", "difficulty": "beginner", "tags": ["web", "full-stack"], "year": 2023},
    {"category": "programming", "language": "rust", "difficulty": "advanced", "tags": ["systems", "performance"], "year": 2023},

    # AI & Machine Learning
    {"category": "ai", "subcategory": "machine_learning", "difficulty": "intermediate", "tags": ["ml", "basics"], "year": 2023},
    {"category": "ai", "subcategory": "deep_learning", "difficulty": "advanced", "tags": ["neural-networks", "dl"], "year": 2023},
    {"category": "ai", "subcategory": "nlp", "difficulty": "intermediate", "tags": ["language", "text"], "year": 2023},
    {"category": "ai", "subcategory": "transformers", "difficulty": "advanced", "tags": ["nlp", "architecture"], "year": 2023},

    # Vector Databases & RAG
    {"category": "database", "subcategory": "vector_db", "difficulty": "intermediate", "tags": ["search", "embeddings"], "year": 2023},
    {"category": "ai", "subcategory": "rag", "difficulty": "advanced", "tags": ["retrieval", "generation"], "year": 2023},
    {"category": "ai", "subcategory": "embeddings", "difficulty": "intermediate", "tags": ["representation", "vectors"], "year": 2023},
    {"category": "database", "subcategory": "hybrid_search", "difficulty": "advanced", "tags": ["search", "retrieval"], "year": 2023},

    # Data Science & Engineering
    {"category": "data", "subcategory": "engineering", "difficulty": "intermediate", "tags": ["infrastructure", "pipelines"], "year": 2023},
    {"category": "data", "subcategory": "feature_engineering", "difficulty": "advanced", "tags": ["ml", "preprocessing"], "year": 2023},

    # Cloud & Infrastructure
    {"category": "devops", "subcategory": "containers", "difficulty": "intermediate", "tags": ["docker", "deployment"], "year": 2023},
    {"category": "devops", "subcategory": "orchestration", "difficulty": "advanced", "tags": ["kubernetes", "scaling"], "year": 2023},
]


# ==============================================================================
# Complete RAG Pipeline Test Suite
# ==============================================================================

class TestCompleteRAGPipeline(EndeeLangChainTestSetup):
    """Comprehensive test suite for the complete RAG pipeline."""

    @classmethod
    def setUpClass(cls):
        """Set up test fixtures."""
        super().setUpClass()
        cls.dense_embeddings = MockDenseEmbeddings(dimension=cls.dimension)
        cls.sparse_embeddings = MockSparseEmbeddings(sparse_dim=1000)
        cls.rag_texts = RAG_TEST_DOCUMENTS
        cls.rag_metadatas = RAG_TEST_METADATAS

    # ==========================================================================
    # PART 1: Vector Store Creation & Configuration
    # ==========================================================================

    def test_01_create_vector_store_basic(self):
        """Test basic vector store creation."""
        print("\n=== Test 01: Basic Vector Store Creation ===")

        index_name = f"{self.test_index_name}_rag_01"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore(
            api_token=self.endee_api_token,
            index_name=index_name,
            embedding=self.dense_embeddings,
            dimension=self.dimension,
            space_type=self.space_type,
            precision=Precision.INT8D,
            force_recreate=True,
        )

        self.assertIsNotNone(vector_store)
        self.assertEqual(vector_store.index_name, index_name)
        self.assertEqual(vector_store.dimension, self.dimension)
        print("✅ Basic vector store created successfully")

    def test_02_create_with_different_precision(self):
        """Test creating vector store with different precision levels."""
        print("\n=== Test 02: Different Precision Levels ===")

        precisions = [
            (Precision.INT8D, "int8d"),
            (Precision.FLOAT16, "float16"),
            (Precision.FLOAT32, "float32")
        ]

        for precision, precision_name in precisions:
            index_name = f"{self.test_index_name}_rag_02_{precision_name}"
            self.test_indexes.add(index_name)

            vector_store = EndeeVectorStore(
                api_token=self.endee_api_token,
                index_name=index_name,
                embedding=self.dense_embeddings,
                dimension=self.dimension,
                space_type=self.space_type,
                precision=precision,
                force_recreate=True,
            )

            self.assertIsNotNone(vector_store)
            print(f"✅ Created index with precision: {precision_name}")

    def test_03_test_properties(self):
        """Test vector store properties."""
        print("\n=== Test 03: Vector Store Properties ===")

        index_name = f"{self.test_index_name}_rag_03"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore(
            api_token=self.endee_api_token,
            index_name=index_name,
            embedding=self.dense_embeddings,
            dimension=self.dimension,
            force_recreate=True,
        )

        # Test properties
        self.assertIsNotNone(vector_store.embeddings)
        self.assertIsNotNone(vector_store.client)
        self.assertIsNotNone(vector_store.index)

        print("✅ All properties accessible")
        print(f"   - Embeddings: {type(vector_store.embeddings).__name__}")
        print(f"   - Client: {type(vector_store.client).__name__}")
        print(f"   - Index: {type(vector_store.index).__name__}")

    # ==========================================================================
    # PART 2: Document Management - Adding Texts
    # ==========================================================================

    def test_04_add_texts_basic(self):
        """Test adding texts to vector store."""
        print("\n=== Test 04: Add Texts (Basic) ===")

        index_name = f"{self.test_index_name}_rag_04"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore(
            api_token=self.endee_api_token,
            index_name=index_name,
            embedding=self.dense_embeddings,
            dimension=self.dimension,
            force_recreate=True,
        )

        texts = self.rag_texts[:5]
        metadatas = self.rag_metadatas[:5]

        ids = vector_store.add_texts(texts=texts, metadatas=metadatas)

        self.assertEqual(len(ids), len(texts))
        print(f"✅ Added {len(ids)} texts successfully")

    def test_05_add_texts_with_custom_ids(self):
        """Test adding texts with custom IDs."""
        print("\n=== Test 05: Add Texts with Custom IDs ===")

        index_name = f"{self.test_index_name}_rag_05"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore(
            api_token=self.endee_api_token,
            index_name=index_name,
            embedding=self.dense_embeddings,
            dimension=self.dimension,
            force_recreate=True,
        )

        texts = self.rag_texts[:3]
        metadatas = self.rag_metadatas[:3]
        custom_ids = [f"custom_id_{i}" for i in range(len(texts))]

        ids = vector_store.add_texts(
            texts=texts,
            metadatas=metadatas,
            ids=custom_ids
        )

        self.assertEqual(ids, custom_ids)
        print(f"✅ Added texts with custom IDs: {custom_ids}")

    def test_06_add_texts_batch_processing(self):
        """Test batch processing when adding many texts."""
        print("\n=== Test 06: Batch Processing ===")

        index_name = f"{self.test_index_name}_rag_06"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore(
            api_token=self.endee_api_token,
            index_name=index_name,
            embedding=self.dense_embeddings,
            dimension=self.dimension,
            force_recreate=True,
        )

        # Add all documents in batches
        ids = vector_store.add_texts(
            texts=self.rag_texts,
            metadatas=self.rag_metadatas,
            batch_size=5,
            embedding_chunk_size=3
        )

        self.assertEqual(len(ids), len(self.rag_texts))
        print(f"✅ Added {len(ids)} texts in batches")

    # ==========================================================================
    # PART 3: Factory Methods
    # ==========================================================================

    def test_07_from_texts_factory(self):
        """Test from_texts factory method."""
        print("\n=== Test 07: from_texts Factory Method ===")

        index_name = f"{self.test_index_name}_rag_07"
        self.test_indexes.add(index_name)

        texts = self.rag_texts[:5]
        metadatas = self.rag_metadatas[:5]

        vector_store = EndeeVectorStore.from_texts(
            texts=texts,
            embedding=self.dense_embeddings,
            metadatas=metadatas,
            api_token=self.endee_api_token,
            index_name=index_name,
            dimension=self.dimension,
            space_type=self.space_type,
            force_recreate=True,
        )

        self.assertIsNotNone(vector_store)
        print(f"✅ Created vector store from {len(texts)} texts")

    def test_08_from_documents_factory(self):
        """Test from_documents factory method."""
        print("\n=== Test 08: from_documents Factory Method ===")

        index_name = f"{self.test_index_name}_rag_08"
        self.test_indexes.add(index_name)

        documents = [
            Document(page_content=text, metadata=meta)
            for text, meta in zip(self.rag_texts[:5], self.rag_metadatas[:5])
        ]

        vector_store = EndeeVectorStore.from_documents(
            documents=documents,
            embedding=self.dense_embeddings,
            api_token=self.endee_api_token,
            index_name=index_name,
            dimension=self.dimension,
            force_recreate=True,
        )

        self.assertIsNotNone(vector_store)
        print(f"✅ Created vector store from {len(documents)} documents")

    def test_09_from_existing_index(self):
        """Test connecting to existing index."""
        print("\n=== Test 09: from_existing_index ===")

        index_name = f"{self.test_index_name}_rag_09"
        self.test_indexes.add(index_name)

        # First create an index with data
        vector_store1 = EndeeVectorStore.from_texts(
            texts=self.rag_texts[:3],
            embedding=self.dense_embeddings,
            metadatas=self.rag_metadatas[:3],
            api_token=self.endee_api_token,
            index_name=index_name,
            dimension=self.dimension,
            force_recreate=True,
        )

        # Now connect to it
        vector_store2 = EndeeVectorStore.from_existing_index(
            index_name=index_name,
            embedding=self.dense_embeddings,
            api_token=self.endee_api_token,
        )

        self.assertIsNotNone(vector_store2)
        print("✅ Connected to existing index successfully")

    # ==========================================================================
    # PART 4: Search Operations - Similarity Search
    # ==========================================================================

    def test_10_similarity_search_basic(self):
        """Test basic similarity search."""
        print("\n=== Test 10: Basic Similarity Search ===")

        index_name = f"{self.test_index_name}_rag_10"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore.from_texts(
            texts=self.rag_texts,
            embedding=self.dense_embeddings,
            metadatas=self.rag_metadatas,
            api_token=self.endee_api_token,
            index_name=index_name,
            dimension=self.dimension,
            force_recreate=True,
        )

        query = "machine learning and artificial intelligence"
        results = vector_store.similarity_search(query, k=3)

        self.assertIsNotNone(results)
        self.assertGreater(len(results), 0)
        self.assertLessEqual(len(results), 3)

        print(f"✅ Found {len(results)} results for query: '{query}'")
        for i, doc in enumerate(results, 1):
            print(f"   {i}. {doc.page_content[:60]}...")

    def test_11_similarity_search_with_score(self):
        """Test similarity search with scores."""
        print("\n=== Test 11: Similarity Search with Scores ===")

        index_name = f"{self.test_index_name}_rag_11"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore.from_texts(
            texts=self.rag_texts,
            embedding=self.dense_embeddings,
            metadatas=self.rag_metadatas,
            api_token=self.endee_api_token,
            index_name=index_name,
            dimension=self.dimension,
            force_recreate=True,
        )

        query = "vector databases and embeddings"
        results = vector_store.similarity_search_with_score(query, k=5)

        self.assertIsNotNone(results)
        self.assertGreater(len(results), 0)

        # Verify all results have scores
        for doc, score in results:
            self.assertIsInstance(doc, Document)
            self.assertIsInstance(score, float)
            self.assertGreaterEqual(score, 0.0)

        print(f"✅ Found {len(results)} results with scores")
        for i, (doc, score) in enumerate(results, 1):
            print(f"   {i}. [Score: {score:.4f}] {doc.page_content[:50]}...")

    def test_12_similarity_search_by_vector(self):
        """Test search by pre-computed embedding vector."""
        print("\n=== Test 12: Search by Vector ===")

        index_name = f"{self.test_index_name}_rag_12"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore.from_texts(
            texts=self.rag_texts[:8],
            embedding=self.dense_embeddings,
            metadatas=self.rag_metadatas[:8],
            api_token=self.endee_api_token,
            index_name=index_name,
            dimension=self.dimension,
            force_recreate=True,
        )

        # Generate query embedding
        query_text = "programming languages and development"
        query_embedding = self.dense_embeddings.embed_query(query_text)

        # Search by vector
        results = vector_store.similarity_search_by_vector(query_embedding, k=3)

        self.assertIsNotNone(results)
        self.assertGreater(len(results), 0)

        print(f"✅ Found {len(results)} results using pre-computed vector")

    def test_13_similarity_search_by_vector_with_score(self):
        """Test search by vector with scores."""
        print("\n=== Test 13: Search by Vector with Scores ===")

        index_name = f"{self.test_index_name}_rag_13"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore.from_texts(
            texts=self.rag_texts[:8],
            embedding=self.dense_embeddings,
            metadatas=self.rag_metadatas[:8],
            api_token=self.endee_api_token,
            index_name=index_name,
            dimension=self.dimension,
            force_recreate=True,
        )

        query_text = "containers and orchestration"
        query_embedding = self.dense_embeddings.embed_query(query_text)

        results = vector_store.similarity_search_by_vector_with_score(
            query_embedding,
            k=3
        )

        self.assertIsNotNone(results)
        for doc, score in results:
            self.assertIsInstance(doc, Document)
            self.assertIsInstance(score, float)

        print(f"✅ Found {len(results)} results with scores")

    # ==========================================================================
    # PART 5: Filtered Search
    # ==========================================================================

    def test_14_filtered_search_single_condition(self):
        """Test search with single filter condition."""
        print("\n=== Test 14: Filtered Search (Single Condition) ===")

        index_name = f"{self.test_index_name}_rag_14"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore.from_texts(
            texts=self.rag_texts,
            embedding=self.dense_embeddings,
            metadatas=self.rag_metadatas,
            api_token=self.endee_api_token,
            index_name=index_name,
            dimension=self.dimension,
            force_recreate=True,
        )

        query = "technology and systems"
        filter_condition = [{"category": {"$eq": "programming"}}]

        results = vector_store.similarity_search(
            query,
            k=5,
            filter=filter_condition
        )

        self.assertIsNotNone(results)

        # Verify all results match filter
        for doc in results:
            self.assertEqual(doc.metadata.get("category"), "programming")

        print(f"✅ Found {len(results)} results matching filter: {filter_condition}")

    def test_15_filtered_search_multiple_conditions(self):
        """Test search with multiple filter conditions (AND logic)."""
        print("\n=== Test 15: Filtered Search (Multiple Conditions) ===")

        index_name = f"{self.test_index_name}_rag_15"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore.from_texts(
            texts=self.rag_texts,
            embedding=self.dense_embeddings,
            metadatas=self.rag_metadatas,
            api_token=self.endee_api_token,
            index_name=index_name,
            dimension=self.dimension,
            force_recreate=True,
        )

        query = "advanced concepts"
        filter_conditions = [
            {"category": {"$eq": "ai"}},
            {"difficulty": {"$eq": "advanced"}}
        ]

        results = vector_store.similarity_search(
            query,
            k=10,
            filter=filter_conditions
        )

        self.assertIsNotNone(results)

        # Verify all results match both filters
        for doc in results:
            self.assertEqual(doc.metadata.get("category"), "ai")
            self.assertEqual(doc.metadata.get("difficulty"), "advanced")

        print(f"✅ Found {len(results)} results matching multiple filters")
        print(f"   All results: category='ai' AND difficulty='advanced'")

    def test_16_filtered_search_with_in_operator(self):
        """Test search with $in filter operator."""
        print("\n=== Test 16: Filtered Search ($in operator) ===")

        index_name = f"{self.test_index_name}_rag_16"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore.from_texts(
            texts=self.rag_texts,
            embedding=self.dense_embeddings,
            metadatas=self.rag_metadatas,
            api_token=self.endee_api_token,
            index_name=index_name,
            dimension=self.dimension,
            force_recreate=True,
        )

        query = "learning and development"
        filter_condition = [{"category": {"$in": ["ai", "data"]}}]

        results = vector_store.similarity_search(
            query,
            k=10,
            filter=filter_condition
        )

        self.assertIsNotNone(results)

        # Verify results match filter
        for doc in results:
            self.assertIn(doc.metadata.get("category"), ["ai", "data"])

        print(f"✅ Found {len(results)} results with $in operator")

    # ==========================================================================
    # PART 6: CRUD Operations
    # ==========================================================================

    def test_17_get_by_ids(self):
        """Test retrieving documents by IDs."""
        print("\n=== Test 17: Get Documents by IDs ===")

        index_name = f"{self.test_index_name}_rag_17"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore(
            api_token=self.endee_api_token,
            index_name=index_name,
            embedding=self.dense_embeddings,
            dimension=self.dimension,
            force_recreate=True,
        )

        texts = self.rag_texts[:5]
        metadatas = self.rag_metadatas[:5]
        custom_ids = [f"doc_{i}" for i in range(len(texts))]

        # Add with custom IDs
        added_ids = vector_store.add_texts(
            texts=texts,
            metadatas=metadatas,
            ids=custom_ids
        )

        # Retrieve by IDs
        retrieved_docs = vector_store.get_by_ids(custom_ids[:3])

        self.assertEqual(len(retrieved_docs), 3)

        print(f"✅ Retrieved {len(retrieved_docs)} documents by IDs")
        for doc in retrieved_docs:
            print(f"   - ID: {doc.metadata.get('_id')}")

    def test_18_delete_by_ids(self):
        """Test deleting documents by IDs."""
        print("\n=== Test 18: Delete by IDs ===")

        index_name = f"{self.test_index_name}_rag_18"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore(
            api_token=self.endee_api_token,
            index_name=index_name,
            embedding=self.dense_embeddings,
            dimension=self.dimension,
            force_recreate=True,
        )

        texts = self.rag_texts[:5]
        metadatas = self.rag_metadatas[:5]
        ids = vector_store.add_texts(texts=texts, metadatas=metadatas)

        # Delete first document
        delete_id = ids[0]
        result = vector_store.delete(ids=[delete_id])

        self.assertTrue(result)

        # Verify deletion
        retrieved = vector_store.get_by_ids([delete_id])
        self.assertEqual(len(retrieved), 0)

        # Verify remaining documents
        remaining = vector_store.get_by_ids(ids[1:])
        self.assertEqual(len(remaining), len(ids) - 1)

        print(f"✅ Deleted document with ID: {delete_id}")
        print(f"   Verified {len(remaining)} documents remain")

    def test_19_delete_by_filter(self):
        """Test deleting documents by filter."""
        print("\n=== Test 19: Delete by Filter ===")

        index_name = f"{self.test_index_name}_rag_19"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore.from_texts(
            texts=self.rag_texts[:8],
            embedding=self.dense_embeddings,
            metadatas=self.rag_metadatas[:8],
            api_token=self.endee_api_token,
            index_name=index_name,
            dimension=self.dimension,
            force_recreate=True,
        )

        # Delete all programming documents
        filter_condition = [{"category": {"$eq": "programming"}}]
        result = vector_store.delete(filter=filter_condition)

        self.assertTrue(result)

        # Verify deletion - search should return no programming docs
        results = vector_store.similarity_search(
            "programming",
            k=10,
            filter=filter_condition
        )

        # Should have 0 or very few results now
        print(f"✅ Deleted documents by filter: {filter_condition}")
        print(f"   Remaining matching documents: {len(results)}")

    # ==========================================================================
    # PART 7: Retriever Integration
    # ==========================================================================

    def test_20_as_retriever_basic(self):
        """Test using vector store as a retriever."""
        print("\n=== Test 20: As Retriever (Basic) ===")

        index_name = f"{self.test_index_name}_rag_20"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore.from_texts(
            texts=self.rag_texts,
            embedding=self.dense_embeddings,
            metadatas=self.rag_metadatas,
            api_token=self.endee_api_token,
            index_name=index_name,
            dimension=self.dimension,
            force_recreate=True,
        )

        # Create retriever
        retriever = vector_store.as_retriever(search_kwargs={"k": 3})

        # Use retriever
        query = "natural language processing and transformers"
        results = retriever.invoke(query)

        self.assertIsNotNone(results)
        self.assertGreater(len(results), 0)
        self.assertLessEqual(len(results), 3)

        print(f"✅ Retriever returned {len(results)} results")
        for i, doc in enumerate(results, 1):
            print(f"   {i}. {doc.page_content[:50]}...")

    def test_21_as_retriever_with_filter(self):
        """Test retriever with filter."""
        print("\n=== Test 21: As Retriever with Filter ===")

        index_name = f"{self.test_index_name}_rag_21"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore.from_texts(
            texts=self.rag_texts,
            embedding=self.dense_embeddings,
            metadatas=self.rag_metadatas,
            api_token=self.endee_api_token,
            index_name=index_name,
            dimension=self.dimension,
            force_recreate=True,
        )

        # Create retriever with filter
        retriever = vector_store.as_retriever(
            search_kwargs={
                "k": 5,
                "filter": [{"category": {"$eq": "database"}}]
            }
        )

        query = "search and retrieval systems"
        results = retriever.invoke(query)

        self.assertIsNotNone(results)

        # Verify all results match filter
        for doc in results:
            self.assertEqual(doc.metadata.get("category"), "database")

        print(f"✅ Filtered retriever returned {len(results)} results")

    # ==========================================================================
    # PART 8: Hybrid Search (Dense + Sparse)
    # ==========================================================================

    def test_22_hybrid_search_basic(self):
        """Test basic hybrid search."""
        print("\n=== Test 22: Hybrid Search (Basic) ===")

        index_name = f"{self.test_index_name}_rag_22"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore(
            api_token=self.endee_api_token,
            index_name=index_name,
            embedding=self.dense_embeddings,
            sparse_embedding=self.sparse_embeddings,
            dimension=self.dimension,
            space_type=self.space_type,
            retrieval_mode=RetrievalMode.HYBRID,
            force_recreate=True,
        )

        # Add documents
        vector_store.add_texts(
            texts=self.rag_texts[:10],
            metadatas=self.rag_metadatas[:10]
        )

        # Perform hybrid search
        query = "machine learning and deep learning"
        results = vector_store.similarity_search(query, k=5)

        self.assertIsNotNone(results)
        self.assertGreater(len(results), 0)

        print(f"✅ Hybrid search returned {len(results)} results")

    def test_23_hybrid_search_with_scores(self):
        """Test hybrid search with scores."""
        print("\n=== Test 23: Hybrid Search with Scores ===")

        index_name = f"{self.test_index_name}_rag_23"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore.from_texts(
            texts=self.rag_texts[:10],
            embedding=self.dense_embeddings,
            metadatas=self.rag_metadatas[:10],
            api_token=self.endee_api_token,
            index_name=index_name,
            dimension=self.dimension,
            retrieval_mode=RetrievalMode.HYBRID,
            sparse_embedding=self.sparse_embeddings,
            force_recreate=True,
        )

        query = "vector embeddings and semantic search"
        results = vector_store.similarity_search_with_score(query, k=5)

        self.assertIsNotNone(results)

        for doc, score in results:
            self.assertIsInstance(doc, Document)
            self.assertIsInstance(score, float)

        print(f"✅ Hybrid search with scores returned {len(results)} results")
        for i, (doc, score) in enumerate(results, 1):
            print(f"   {i}. [Score: {score:.4f}] {doc.page_content[:50]}...")

    def test_24_hybrid_vs_dense_comparison(self):
        """Compare hybrid search with dense-only search."""
        print("\n=== Test 24: Hybrid vs Dense Comparison ===")

        # Create dense index
        dense_index = f"{self.test_index_name}_rag_24_dense"
        self.test_indexes.add(dense_index)

        dense_store = EndeeVectorStore.from_texts(
            texts=self.rag_texts[:10],
            embedding=self.dense_embeddings,
            metadatas=self.rag_metadatas[:10],
            api_token=self.endee_api_token,
            index_name=dense_index,
            dimension=self.dimension,
            retrieval_mode=RetrievalMode.DENSE,
            force_recreate=True,
        )

        # Create hybrid index
        hybrid_index = f"{self.test_index_name}_rag_24_hybrid"
        self.test_indexes.add(hybrid_index)

        hybrid_store = EndeeVectorStore.from_texts(
            texts=self.rag_texts[:10],
            embedding=self.dense_embeddings,
            metadatas=self.rag_metadatas[:10],
            api_token=self.endee_api_token,
            index_name=hybrid_index,
            dimension=self.dimension,
            retrieval_mode=RetrievalMode.HYBRID,
            sparse_embedding=self.sparse_embeddings,
            force_recreate=True,
        )

        query = "Kubernetes container orchestration"

        dense_results = dense_store.similarity_search_with_score(query, k=3)
        hybrid_results = hybrid_store.similarity_search_with_score(query, k=3)

        self.assertIsNotNone(dense_results)
        self.assertIsNotNone(hybrid_results)

        print("✅ Comparison complete")
        print(f"   Dense results: {len(dense_results)}")
        print(f"   Hybrid results: {len(hybrid_results)}")

    # ==========================================================================
    # PART 9: Full RAG Pipeline End-to-End
    # ==========================================================================

    def test_25_complete_rag_pipeline(self):
        """Test complete RAG pipeline from ingestion to retrieval."""
        print("\n=== Test 25: Complete RAG Pipeline ===")

        index_name = f"{self.test_index_name}_rag_25_complete"
        self.test_indexes.add(index_name)

        # Step 1: Create vector store
        print("   Step 1: Creating vector store...")
        vector_store = EndeeVectorStore(
            api_token=self.endee_api_token,
            index_name=index_name,
            embedding=self.dense_embeddings,
            dimension=self.dimension,
            space_type="cosine",
            precision=Precision.INT8D,
            force_recreate=True,
        )

        # Step 2: Ingest documents
        print("   Step 2: Ingesting documents...")
        ids = vector_store.add_texts(
            texts=self.rag_texts,
            metadatas=self.rag_metadatas,
            batch_size=5
        )
        self.assertEqual(len(ids), len(self.rag_texts))

        # Step 3: Perform searches
        print("   Step 3: Performing searches...")

        # Basic search
        results1 = vector_store.similarity_search(
            "What is machine learning?",
            k=3
        )
        self.assertGreater(len(results1), 0)

        # Filtered search
        results2 = vector_store.similarity_search(
            "programming and development",
            k=5,
            filter=[{"category": {"$eq": "programming"}}]
        )
        self.assertGreater(len(results2), 0)

        # Search with scores
        results3 = vector_store.similarity_search_with_score(
            "vector databases and embeddings",
            k=3
        )
        self.assertGreater(len(results3), 0)

        # Step 4: Use as retriever
        print("   Step 4: Using as retriever...")
        retriever = vector_store.as_retriever(search_kwargs={"k": 3})
        retrieved = retriever.invoke("transformers and NLP")
        self.assertGreater(len(retrieved), 0)

        # Step 5: Document management
        print("   Step 5: Testing document management...")
        test_docs = vector_store.get_by_ids(ids[:2])
        self.assertEqual(len(test_docs), 2)

        print("✅ Complete RAG pipeline executed successfully!")
        print(f"   - Documents ingested: {len(ids)}")
        print(f"   - Basic search results: {len(results1)}")
        print(f"   - Filtered search results: {len(results2)}")
        print(f"   - Scored search results: {len(results3)}")
        print(f"   - Retrieved via retriever: {len(retrieved)}")

    def test_26_rag_with_metadata_enrichment(self):
        """Test RAG with rich metadata and filtering."""
        print("\n=== Test 26: RAG with Metadata Enrichment ===")

        index_name = f"{self.test_index_name}_rag_26"
        self.test_indexes.add(index_name)

        vector_store = EndeeVectorStore.from_texts(
            texts=self.rag_texts,
            embedding=self.dense_embeddings,
            metadatas=self.rag_metadatas,
            api_token=self.endee_api_token,
            index_name=index_name,
            dimension=self.dimension,
            force_recreate=True,
        )

        # Search and examine metadata
        query = "advanced AI concepts"
        results = vector_store.similarity_search_with_score(
            query,
            k=5,
            filter=[{"difficulty": {"$eq": "advanced"}}]
        )

        print(f"✅ Found {len(results)} advanced documents")
        for i, (doc, score) in enumerate(results, 1):
            meta = doc.metadata
            print(f"   {i}. [Score: {score:.4f}]")
            print(f"      Category: {meta.get('category')}")
            print(f"      Subcategory: {meta.get('subcategory', 'N/A')}")
            print(f"      Tags: {meta.get('tags', [])}")


# ==============================================================================
# Test Runner
# ==============================================================================

if __name__ == '__main__':
    # Run tests with verbose output
    unittest.main(verbosity=2)
