import asyncio
import hashlib
import io
import time
from dataclasses import dataclass
from enum import Enum
from functools import cached_property
from io import BytesIO
from os.path import basename
from pathlib import Path
from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast, overload
from uuid import uuid4

from httpx import AsyncClient

from mielto.db.base import BaseDb
from mielto.db.schemas.knowledge import KnowledgeRow
from mielto.knowledge.content import Content, ContentAuth, ContentStatus, FileData
from mielto.knowledge.document import Document
from mielto.knowledge.reader import Reader, ReaderFactory
from mielto.knowledge.remote_content.remote_content import GCSContent, RemoteContent, S3Content
from mielto.utils.common import generate_prefix_ulid
from mielto.utils.http import async_fetch_with_retry
from mielto.utils.log import log_debug, log_error, log_info, log_warning
from mielto.vectordb import VectorDb

ContentDict = Dict[str, Union[str, Dict[str, str]]]


class KnowledgeContentOrigin(Enum):
    PATH = "path"
    URL = "url"
    TOPIC = "topic"
    CONTENT = "content"


@dataclass
class Knowledge:
    """Knowledge class"""

    name: Optional[str] = None
    description: Optional[str] = None
    vector_db: Optional[VectorDb] = None
    contents_db: Optional[BaseDb] = None
    max_results: int = 10
    readers: Optional[Dict[str, Reader]] = None
    provider: Optional[str] = "native"

    def __post_init__(self):
        if self.vector_db and not self.vector_db.exists():
            self.vector_db.create()

        self.construct_readers()
        self.valid_metadata_filters = set()

    # --- SDK Specific Methods ---

    # --- Add Contents ---
    @overload
    async def add_contents_async(self, contents: List[ContentDict]) -> None: ...

    @overload
    async def add_contents_async(
        self,
        *,
        paths: Optional[List[str]] = None,
        urls: Optional[List[str]] = None,
        metadata: Optional[Dict[str, str]] = None,
        include: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
        upsert: bool = False,
        skip_if_exists: bool = False,
        remote_content: Optional[RemoteContent] = None,
    ) -> None: ...

    async def add_contents_async(self, *args, **kwargs) -> None:
        if args and isinstance(args[0], list):
            arguments = args[0]
            for argument in arguments:
                await self.add_content_async(
                    name=argument.get("name"),
                    description=argument.get("description"),
                    path=argument.get("path"),
                    url=argument.get("url"),
                    metadata=argument.get("metadata"),
                    topics=argument.get("topics"),
                    workspace_id=argument.get("workspace_id"),
                    collection_id=argument.get("collection_id"),
                    reader=argument.get("reader"),
                    include=argument.get("include"),
                    exclude=argument.get("exclude"),
                    upsert=argument.get("upsert", False),
                    skip_if_exists=argument.get("skip_if_exists", False),
                    remote_content=argument.get("remote_content", None),
                )

        elif kwargs:
            name = kwargs.get("name", [])
            metadata = kwargs.get("metadata", {})
            description = kwargs.get("description", [])
            topics = kwargs.get("topics", [])
            paths = kwargs.get("paths", [])
            urls = kwargs.get("urls", [])
            include = kwargs.get("include")
            exclude = kwargs.get("exclude")
            workspace_id = kwargs.get("workspace_id")
            collection_id = kwargs.get("collection_id")
            upsert = kwargs.get("upsert", False)
            skip_if_exists = kwargs.get("skip_if_exists", False)
            remote_content = kwargs.get("remote_content", None)

            for path in paths:
                await self.add_content_async(
                    name=name,
                    description=description,
                    path=path,
                    metadata=metadata,
                    workspace_id=workspace_id,
                    collection_id=collection_id,
                    include=include,
                    exclude=exclude,
                    upsert=upsert,
                    skip_if_exists=skip_if_exists,
                )
            for url in urls:
                await self.add_content_async(
                    name=name,
                    description=description,
                    url=url,
                    metadata=metadata,
                    workspace_id=workspace_id,
                    collection_id=collection_id,
                    include=include,
                    exclude=exclude,
                    upsert=upsert,
                    skip_if_exists=skip_if_exists,
                )
            if topics:
                await self.add_content_async(
                    name=name,
                    description=description,
                    topics=topics,
                    metadata=metadata,
                    workspace_id=workspace_id,
                    collection_id=collection_id,
                    include=include,
                    exclude=exclude,
                    upsert=upsert,
                    skip_if_exists=skip_if_exists,
                )

            if remote_content:
                await self.add_content_async(
                    name=name,
                    metadata=metadata,
                    workspace_id=workspace_id,
                    collection_id=collection_id,
                    description=description,
                    remote_content=remote_content,
                    upsert=upsert,
                    skip_if_exists=skip_if_exists,
                )

        else:
            raise ValueError("Invalid usage of add_contents.")

    @overload
    def add_contents(self, contents: List[ContentDict]) -> None: ...

    @overload
    def add_contents(
        self,
        *,
        paths: Optional[List[str]] = None,
        urls: Optional[List[str]] = None,
        metadata: Optional[Dict[str, str]] = None,
        include: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
        upsert: bool = False,
        skip_if_exists: bool = False,
    ) -> None: ...

    def add_contents(self, *args, **kwargs) -> None:
        """
        Synchronously add multiple content items to the knowledge base.

        This method wraps the asynchronous add_contents method

        Supports two usage patterns:
        1. Pass a list of content dictionaries as first argument
        2. Pass keyword arguments with paths, urls, metadata, etc.

        Args:
            contents: List of content dictionaries (when used as first overload)
            paths: Optional list of file paths to load content from
            urls: Optional list of URLs to load content from
            metadata: Optional metadata dictionary to apply to all content
            include: Optional list of file patterns to include
            exclude: Optional list of file patterns to exclude
            upsert: Whether to update existing content if it already exists
            skip_if_exists: Whether to skip adding content if it already exists
        """
        asyncio.run(self.add_contents_async(*args, **kwargs))

    # --- Add Content ---

    @overload
    async def add_content_async(
        self,
        *,
        workspace_id: Optional[str] = None,
        collection_id: Optional[str] = None,
        path: Optional[str] = None,
        url: Optional[str] = None,
        text_content: Optional[str] = None,
        metadata: Optional[Dict[str, str]] = None,
        include: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
        upsert: bool = False,
        skip_if_exists: bool = False,
        reader: Optional[Reader] = None,
        auth: Optional[ContentAuth] = None,
    ) -> Any: ...

    @overload
    async def add_content_async(self, *args, **kwargs) -> Any: ...

    async def add_content_async(
        self,
        workspace_id: Optional[str] = None,
        collection_id: Optional[str] = None,
        name: Optional[str] = None,
        description: Optional[str] = None,
        path: Optional[str] = None,
        url: Optional[str] = None,
        text_content: Optional[str] = None,
        metadata: Optional[Dict[str, Any]] = None,
        topics: Optional[List[str]] = None,
        remote_content: Optional[RemoteContent] = None,
        reader: Optional[Reader] = None,
        include: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
        upsert: bool = True,
        skip_if_exists: bool = True,
        auth: Optional[ContentAuth] = None,
    ) -> Any:
        # Validation: At least one of the parameters must be provided
        if all(argument is None for argument in [path, url, text_content, topics, remote_content]):
            log_info("At least one of 'path', 'url', 'text_content', 'topics', or 'remote_content' must be provided.")
            return

        if not skip_if_exists:
            log_info("skip_if_exists is disabled, disabling upsert")
            upsert = False

        content = None
        file_data = None
        if text_content:
            file_data = FileData(content=text_content, type="text/plain")

        content = Content(
            id=generate_prefix_ulid("con"),
            name=name,
            workspace_id=workspace_id,
            collection_id=collection_id,
            description=description,
            path=path,
            url=url,
            file_data=file_data if file_data else None,
            metadata=metadata,
            topics=topics,
            remote_content=remote_content,
            reader=reader,
            auth=auth,
        )

        await self._load_content(content, upsert, skip_if_exists, include, exclude)
        return content


    async def insert_content(self, content: Content) -> None:
        await self._load_content(content, upsert=True, skip_if_exists=True, include=None, exclude=None)


    @overload
    def add_content(
        self,
        *,
        workspace_id: Optional[str] = None,
        collection_id: Optional[str] = None,
        path: Optional[str] = None,
        url: Optional[str] = None,
        text_content: Optional[str] = None,
        metadata: Optional[Dict[str, str]] = None,
        include: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
        upsert: bool = False,
        skip_if_exists: bool = False,
        reader: Optional[Reader] = None,
        auth: Optional[ContentAuth] = None,
    ) -> None: ...

    @overload
    def add_content(self, *args, **kwargs) -> None: ...

    def add_content(
        self,
        name: Optional[str] = None,
        description: Optional[str] = None,
        path: Optional[str] = None,
        url: Optional[str] = None,
        workspace_id: Optional[str] = None,
        collection_id: Optional[str] = None,
        text_content: Optional[str] = None,
        metadata: Optional[Dict[str, Any]] = None,
        topics: Optional[List[str]] = None,
        remote_content: Optional[RemoteContent] = None,
        reader: Optional[Reader] = None,
        include: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
        upsert: bool = True,
        skip_if_exists: bool = True,
        auth: Optional[ContentAuth] = None,
    ) -> None:
        """
        Synchronously add content to the knowledge base.

        Args:
            name: Optional name for the content
            description: Optional description for the content
            path: Optional file path to load content from
            url: Optional URL to load content from
            text_content: Optional text content to add directly
            metadata: Optional metadata dictionary
            topics: Optional list of topics
            config: Optional cloud storage configuration
            reader: Optional custom reader for processing the content
            include: Optional list of file patterns to include
            exclude: Optional list of file patterns to exclude
            upsert: Whether to update existing content if it already exists
            skip_if_exists: Whether to skip adding content if it already exists
        """
        asyncio.run(
            self.add_content_async(
                workspace_id=workspace_id,
                collection_id=collection_id,
                name=name,
                description=description,
                path=path,
                url=url,
                text_content=text_content,
                metadata=metadata,
                topics=topics,
                remote_content=remote_content,
                reader=reader,
                include=include,
                exclude=exclude,
                upsert=upsert,
                skip_if_exists=skip_if_exists,
                auth=auth,
            )
        )

    def content_hash_exists(self, content_hash: str, workspace_id: Optional[str] = None) -> KnowledgeRow:
        if self.contents_db is None:
            return None
        content = self.contents_db.get_knowledge_content_by_content_hash(content_hash, workspace_id)
        return content

    async def _load_from_path(
        self,
        content: Content,
        upsert: bool,
        skip_if_exists: bool,
        include: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
    ):
        log_info(f"Adding content from path, {content.id}, {content.name}, {content.path}, {content.description}")
        path = Path(content.path)  # type: ignore

        if path.is_file():
            if self._should_include_file(str(path), include, exclude):
                log_info(f"Adding file {path} due to include/exclude filters")

                # Handle LightRAG special case - read file and upload directly
                if self.vector_db.__class__.__name__ == "LightRag":
                    await self._process_lightrag_content(content, KnowledgeContentOrigin.PATH)
                    return

                content.content_hash = self._build_content_hash(content)
                if self.vector_db and self.vector_db.content_hash_exists(content.content_hash) and skip_if_exists:
                    log_info(f"Content {content.content_hash} already exists, skipping")
                    return


                if not content.skip_content_insert:
                    self._add_to_contents_db(content)
                else:
                    log_info(f"Skipping content insert for {content.id}, {content.name}, {content.path}, {content.description}")

                if content.reader:

                    print(f"Reading content with reader: {content.reader.__class__.__name__}")
                    # TODO: We will refactor this to eventually pass authorization to all readers
                    import inspect

                    read_signature = inspect.signature(content.reader.read)
                    if "password" in read_signature.parameters and content.auth and content.auth.password:
                        read_documents = content.reader.read(
                            path, name=content.name or path.name, password=content.auth.password
                        )
                    else:
                        read_documents = content.reader.read(path, name=content.name or path.name)

                else:
                    reader = ReaderFactory.get_reader_for_extension(path.suffix, provider=self.provider)
                    log_info(f"Using Reader: {reader.__class__.__name__}")
                    if reader:
                        # TODO: We will refactor this to eventually pass authorization to all readers
                        import inspect

                        read_signature = inspect.signature(reader.read)
                        if "password" in read_signature.parameters and content.auth and content.auth.password:
                            read_documents = reader.read(
                                path, name=content.name or path.name, password=content.auth.password
                            )
                        else:
                            read_documents = reader.read(path, name=content.name or path.name)

                if not content.file_type:
                    content.file_type = path.suffix

                if not content.size and content.file_data:
                    content.size = len(content.file_data.content)  # type: ignore
                if not content.size:
                    try:
                        content.size = path.stat().st_size
                    except (OSError, IOError) as e:
                        log_warning(f"Could not get file size for {path}: {e}")
                        content.size = 0

                for read_document in read_documents:
                    read_document.content_id = content.id
                    read_document.id = generate_prefix_ulid("chunk")

                await self._handle_vector_db_insert(content, read_documents, upsert)

        elif path.is_dir():
            for file_path in path.iterdir():
                # Apply include/exclude filtering
                if not self._should_include_file(str(file_path), include, exclude):
                    log_debug(f"Skipping file {file_path} due to include/exclude filters")
                    continue

                id = generate_prefix_ulid("con")
                file_content = Content(
                    id=id,
                    name=content.name,
                    path=str(file_path),
                    metadata=content.metadata,
                    description=content.description,
                    reader=content.reader,
                    collection_id=content.collection_id,
                    workspace_id=content.workspace_id,
                )
                await self._load_from_path(file_content, upsert, skip_if_exists, include, exclude)
        else:
            log_warning(f"Invalid path: {path}")

    async def _load_from_url(
        self,
        content: Content,
        upsert: bool,
        skip_if_exists: bool,
    ):
        """Load the content in the contextual URL

        1. Set content hash
        2. Validate the URL
        3. Read the content
        4. Prepare and insert the content in the vector database
        """
        log_info(f"Adding content from URL {content.url}")
        content.file_type = "url"

        if not content.url:
            raise ValueError("No url provided")

        if self.vector_db.__class__.__name__ == "LightRag":
            await self._process_lightrag_content(content, KnowledgeContentOrigin.URL)
            return

        # 1. Set content hash
        content.content_hash = self._build_content_hash(content)
        if self.vector_db and self.vector_db.content_hash_exists(content.content_hash) and skip_if_exists:
            log_info(f"Content {content.content_hash} already exists, skipping")
            return
        self._add_to_contents_db(content)

        # 2. Validate URL
        try:
            from urllib.parse import urlparse

            parsed_url = urlparse(content.url)
            if not all([parsed_url.scheme, parsed_url.netloc]):
                content.status = ContentStatus.FAILED
                content.status_message = f"Invalid URL format: {content.url}"
                self._update_content(content)
                log_warning(f"Invalid URL format: {content.url}")
        except Exception as e:
            content.status = ContentStatus.FAILED
            content.status_message = f"Invalid URL: {content.url} - {str(e)}"
            self._update_content(content)
            log_warning(f"Invalid URL: {content.url} - {str(e)}")

        # 3. Fetch and load content
        async with AsyncClient() as client:
            response = await async_fetch_with_retry(content.url, client=client)
        bytes_content = BytesIO(response.content)

        # 4. Select reader
        # If a reader was provided by the user, use it
        reader = content.reader
        name = content.name if content.name else content.url
        # Else select based on file extension
        if reader is None:
            url_path = Path(parsed_url.path)
            file_extension = url_path.suffix.lower()
            if file_extension == ".csv":
                name = basename(parsed_url.path) or "data.csv"
                reader = self.csv_reader
            elif file_extension == ".pdf":
                reader = self.pdf_reader
            elif file_extension == ".docx":
                reader = self.docx_reader
            elif file_extension == ".json":
                reader = self.json_reader
            elif file_extension == ".markdown" or file_extension == ".md":
                reader = self.markdown_reader
            else:
                reader = self.text_reader

        # 5. Read content
        try:
            read_documents = []
            if reader is not None:
                # TODO: We will refactor this to eventually pass authorization to all readers
                import inspect

                read_signature = inspect.signature(reader.read)
                if reader.__class__.__name__ == "YouTubeReader":
                    read_documents = reader.read(content.url, name=name)
                elif "password" in read_signature.parameters and content.auth and content.auth.password:
                    read_documents = reader.read(bytes_content, name=name, password=content.auth.password)
                else:
                    read_documents = reader.read(bytes_content, name=name)
        except Exception as e:
            log_error(f"Error reading URL: {content.url} - {str(e)}")
            content.status = ContentStatus.FAILED
            content.status_message = f"Error reading URL: {content.url} - {str(e)}"
            self._update_content(content)
            return

        # 6. Chunk documents if needed
        if reader and not reader.chunk:
            read_documents = await reader.chunk_documents_async(read_documents)

        # 7. Prepare and insert the content in the vector database
        file_size = 0
        if read_documents:
            for read_document in read_documents:
                if read_document.size:
                    file_size += read_document.size
                read_document.content_id = content.id
        await self._handle_vector_db_insert(content, read_documents, upsert)

    async def _load_from_content(
        self,
        content: Content,
        upsert: bool = True,
        skip_if_exists: bool = True,
    ):

        if content.name:
            name = content.name
        elif content.file_data and content.file_data.content:
            if isinstance(content.file_data.content, bytes):
                name = content.file_data.content[:10].decode("utf-8", errors="ignore")
            elif isinstance(content.file_data.content, str):
                name = (
                    content.file_data.content[:10]
                    if len(content.file_data.content) >= 10
                    else content.file_data.content
                )
            else:
                name = str(content.file_data.content)[:10]
        else:
            name = None

        if name is not None:
            content.name = name

        log_info(f"Adding content from {content.name}")

        if content.file_data and self.vector_db.__class__.__name__ == "LightRag":
            await self._process_lightrag_content(content, KnowledgeContentOrigin.CONTENT)
            return

        content.content_hash = self._build_content_hash(content)
        if self.vector_db and self.vector_db.content_hash_exists(content.content_hash) and skip_if_exists:
            log_info(f"Content {content.content_hash} already exists, skipping")

            return
        self._add_to_contents_db(content)

        read_documents = []

        if isinstance(content.file_data, str):
            try:
                content_bytes = content.file_data.encode("utf-8")
            except UnicodeEncodeError:
                content_bytes = content.file_data.encode("latin-1")
            content_io = io.BytesIO(content_bytes)

            if content.reader:
                log_info(f"Using reader: {content.reader.__class__.__name__} to read content")
                read_documents = content.reader.read(content_io, name=name)
            else:
                text_reader = self.text_reader
                if text_reader:
                    read_documents = text_reader.read(content_io, name=name)
                else:
                    content.status = ContentStatus.FAILED
                    content.status_message = "Text reader not available"
                    self._update_content(content)
                    return

        elif isinstance(content.file_data, FileData):
            if content.file_data.type:
                if isinstance(content.file_data.content, bytes):
                    content_io = io.BytesIO(content.file_data.content)
                elif isinstance(content.file_data.content, str):
                    if self._is_text_mime_type(content.file_data.type):
                        try:
                            content_bytes = content.file_data.content.encode("utf-8")
                        except UnicodeEncodeError:
                            log_debug(f"UTF-8 encoding failed for {content.file_data.type}, using latin-1")
                            content_bytes = content.file_data.content.encode("latin-1")
                    else:
                        try:
                            content_bytes = content.file_data.content.encode("latin-1")
                        except UnicodeEncodeError:
                            log_debug(f"latin-1 encoding also failed, using UTF-8 with error replacement")
                            content_bytes = content.file_data.content.encode("utf-8")
                    content_io = io.BytesIO(content_bytes)
                else:
                    content_io = content.file_data.content  # type: ignore

                # Respect an explicitly provided reader; otherwise select based on file type
                if content.reader:
                    log_info(f"Using reader: {content.reader.__class__.__name__} to read content")
                    reader = content.reader
                else:
                    reader = self._select_reader(content.file_data.type)
                name = content.name if content.name else f"content_{content.file_data.type}"
                read_documents = reader.read(content_io, name=name)

                for read_document in read_documents:
                    if content.metadata:
                        read_document.meta_data.update(content.metadata)
                    read_document.content_id = content.id

                if len(read_documents) == 0:
                    content.status = ContentStatus.FAILED
                    content.status_message = "Content could not be read"
                    self._update_content(content)

        else:
            content.status = ContentStatus.FAILED
            content.status_message = "No content provided"
            self._update_content(content)
            return

        await self._handle_vector_db_insert(content, read_documents, upsert)

    async def _load_from_topics(
        self,
        content: Content,
        upsert: bool,
        skip_if_exists: bool,
    ):
        log_info(f"Adding content from topics: {content.topics}")

        if content.topics is None:
            log_warning("No topics provided for content")
            return

        for topic in content.topics:
            id = generate_prefix_ulid("con")
            content = Content(
                id=id,
                name=topic,
                metadata=content.metadata,
                reader=content.reader,
                status=ContentStatus.PROCESSING if content.reader else ContentStatus.FAILED,
                file_data=FileData(
                    type="Topic",
                ),
                topics=[topic],
                collection_id=content.collection_id,
                workspace_id=content.workspace_id,
            )

            if self.vector_db.__class__.__name__ == "LightRag":
                await self._process_lightrag_content(content, KnowledgeContentOrigin.TOPIC)
                return

            content.content_hash = self._build_content_hash(content)
            if self.vector_db and self.vector_db.content_hash_exists(content.content_hash) and skip_if_exists:
                log_info(f"Content {content.content_hash} already exists, skipping")
                continue

            self._add_to_contents_db(content)
            if content.reader is None:
                log_error(f"No reader available for topic: {topic}")
                continue
            read_documents = content.reader.read(topic)
            if len(read_documents) > 0:
                for read_document in read_documents:
                    read_document.content_id = id
                    if read_document.content:
                        read_document.size = len(read_document.content.encode("utf-8"))
            else:
                content.status = ContentStatus.FAILED
                content.status_message = "No content found for topic"
                self._update_content(content)

            await self._handle_vector_db_insert(content, read_documents, upsert)

    async def _load_from_remote_content(
        self,
        content: Content,
        upsert: bool,
        skip_if_exists: bool,
    ):
        if content.remote_content is None:
            log_warning("No remote content provided for content")
            return

        remote_content = content.remote_content

        if isinstance(remote_content, S3Content):
            await self._load_from_s3(content, upsert, skip_if_exists)

        elif isinstance(remote_content, GCSContent):
            await self._load_from_gcs(content, upsert, skip_if_exists)

        else:
            log_warning(f"Unsupported remote content type: {type(remote_content)}")

    async def _load_from_s3(self, content: Content, upsert: bool, skip_if_exists: bool):
        """Load the contextual S3 content.

        1. Identify objects to read
        2. Setup Content object
        3. Hash content and add it to the contents database
        4. Select reader
        5. Fetch and load the content
        6. Read the content
        7. Prepare and insert the content in the vector database
        8. Remove temporary file if needed
        """
        from mielto.cloud.aws.s3.object import S3Object

        remote_content: S3Content = cast(S3Content, content.remote_content)

        # 1. Identify objects to read
        objects_to_read: List[S3Object] = []
        if remote_content.bucket is not None:
            if remote_content.key is not None:
                _object = S3Object(bucket_name=remote_content.bucket.name, name=remote_content.key)
                objects_to_read.append(_object)
            elif remote_content.object is not None:
                objects_to_read.append(remote_content.object)
            elif remote_content.prefix is not None:
                objects_to_read.extend(remote_content.bucket.get_objects(prefix=remote_content.prefix))
            else:
                objects_to_read.extend(remote_content.bucket.get_objects())

        for s3_object in objects_to_read:
            # 2. Setup Content object
            id = str(uuid4())
            content_name = content.name or ""
            content_name += "_" + (s3_object.name or "")
            content_entry = Content(
                id=id,
                name=content_name,
                description=content.description,
                status=ContentStatus.PROCESSING,
                metadata=content.metadata,
                file_type="s3",
                collection_id=content.collection_id,
                workspace_id=content.workspace_id,
            )

            # 3. Hash content and add it to the contents database
            content_hash = self._build_content_hash(content_entry)
            if self.vector_db and self.vector_db.content_hash_exists(content_hash) and skip_if_exists:
                log_info(f"Content {content_hash} already exists, skipping")
                continue
            self._add_to_contents_db(content_entry)

            # 4. Select reader
            reader = content.reader
            if reader is None:
                if s3_object.uri.endswith(".pdf"):
                    reader = self.pdf_reader
                elif s3_object.uri.endswith(".csv"):
                    reader = self.csv_reader
                elif s3_object.uri.endswith(".docx"):
                    reader = self.docx_reader
                elif s3_object.uri.endswith(".json"):
                    reader = self.json_reader
                elif s3_object.uri.endswith(".markdown"):
                    reader = self.markdown_reader
                else:
                    reader = self.text_reader
            reader = cast(Reader, reader)

            # 5. Fetch and load the content
            temporary_file = None
            obj_name = content_name or s3_object.name.split("/")[-1]
            readable_content: Optional[Union[BytesIO, Path]] = None
            if s3_object.uri.endswith(".pdf"):
                readable_content = BytesIO(s3_object.get_resource().get()["Body"].read())
            else:
                temporary_file = Path("storage").joinpath(obj_name)
                readable_content = temporary_file
                s3_object.download(readable_content)  # type: ignore

            # 6. Read the content
            read_documents = reader.read(readable_content, name=obj_name)

            # 7. Prepare and insert the content in the vector database
            for read_document in read_documents:
                read_document.content_id = content.id
            await self._handle_vector_db_insert(content_entry, read_documents, upsert)

            # 8. Remove temporary file if needed
            if temporary_file:
                temporary_file.unlink()

    async def _load_from_gcs(self, content: Content, upsert: bool, skip_if_exists: bool):
        """Load the contextual GCS content.

        1. Identify objects to read
        2. Setup Content object
        3. Hash content and add it to the contents database
        4. Select reader
        5. Fetch and load the content
        6. Read the content
        7. Prepare and insert the content in the vector database
        """
        remote_content: GCSContent = cast(GCSContent, content.remote_content)

        # 1. Identify objects to read
        objects_to_read = []
        if remote_content.blob_name is not None:
            objects_to_read.append(remote_content.bucket.blob(remote_content.blob_name))  # type: ignore
        elif remote_content.prefix is not None:
            objects_to_read.extend(remote_content.bucket.list_blobs(prefix=remote_content.prefix))  # type: ignore
        else:
            objects_to_read.extend(remote_content.bucket.list_blobs())  # type: ignore

        for gcs_object in objects_to_read:
            # 2. Setup Content object
            id = str(uuid4())
            name = (content.name or "content") + "_" + gcs_object.name
            content_entry = Content(
                id=id,
                name=name,
                description=content.description,
                status=ContentStatus.PROCESSING,
                metadata=content.metadata,
                file_type="gcs",
                collection_id=content.collection_id,
                workspace_id=content.workspace_id,
            )

            # 3. Hash content and add it to the contents database
            content_hash = self._build_content_hash(content_entry)
            if self.vector_db and self.vector_db.content_hash_exists(content_hash) and skip_if_exists:
                log_info(f"Content {content_hash} already exists, skipping")
                continue

            # 4. Add it to the contents database
            self._add_to_contents_db(content_entry)

            # 5. Select reader
            reader = content.reader
            if reader is None:
                if gcs_object.name.endswith(".pdf"):
                    reader = self.pdf_reader
                elif gcs_object.name.endswith(".csv"):
                    reader = self.csv_reader
                elif gcs_object.name.endswith(".docx"):
                    reader = self.docx_reader
                elif gcs_object.name.endswith(".json"):
                    reader = self.json_reader
                elif gcs_object.name.endswith(".markdown"):
                    reader = self.markdown_reader
                else:
                    reader = self.text_reader
            reader = cast(Reader, reader)

            # 5. Fetch and load the content
            readable_content = BytesIO(gcs_object.download_as_bytes())

            # 6. Read the content
            read_documents = reader.read(readable_content, name=name)

            # 7. Prepare and insert the content in the vector database
            for read_document in read_documents:
                read_document.content_id = content.id
            await self._handle_vector_db_insert(content_entry, read_documents, upsert)

    async def _handle_vector_db_insert(self, content, read_documents, upsert):
        if not self.vector_db:
            log_error("No vector database configured")
            content.status = ContentStatus.FAILED
            content.status_message = "No vector database configured"
            self._update_content(content)
            return

        if self.vector_db.upsert_available() and upsert:
            try:
                await self.vector_db.async_upsert(content.content_hash, read_documents, content.metadata, collection_id=content.collection_id, workspace_id=content.workspace_id)
            except Exception as e:
                log_error(f"Error upserting document: {e}")
                content.status = ContentStatus.FAILED
                content.status_message = "Could not upsert embedding"
                self._update_content(content)
                return
        else:
            try:
                await self.vector_db.async_insert(
                    content.content_hash, 
                    documents=read_documents, 
                    filters=content.metadata,
                    collection_id=content.collection_id,
                    workspace_id=content.workspace_id
                )
            except Exception as e:
                log_error(f"Error inserting document: {e}")
                content.status = ContentStatus.FAILED
                content.status_message = "Could not insert embedding"
                self._update_content(content)
                return

        content.status = ContentStatus.COMPLETED
        self._update_content(content)

    async def _load_content(
        self,
        content: Content,
        upsert: bool,
        skip_if_exists: bool,
        include: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
    ) -> None:
        log_info(f"Loading content: {content.id}")

        if content.metadata:
            self.add_filters(content.metadata)

        if content.path:
            await self._load_from_path(content, upsert, skip_if_exists, include, exclude)

        if content.url:
            await self._load_from_url(content, upsert, skip_if_exists)

        if content.file_data:
            await self._load_from_content(content, upsert, skip_if_exists)

        if content.topics:
            await self._load_from_topics(content, upsert, skip_if_exists)

        if content.remote_content:
            await self._load_from_remote_content(content, upsert, skip_if_exists)

    def _build_content_hash(self, content: Content) -> str:
        """
        Build the content hash from the content.
        """
        if content.path:
            return hashlib.sha256(str(content.path).encode()).hexdigest()
        elif content.url:
            hash = hashlib.sha256(content.url.encode()).hexdigest()
            return hash
        elif content.file_data and content.file_data.content:
            name = content.name or "content"
            return hashlib.sha256(name.encode()).hexdigest()
        elif content.topics and len(content.topics) > 0:
            topic = content.topics[0]
            reader = type(content.reader).__name__ if content.reader else "unknown"
            return hashlib.sha256(f"{topic}-{reader}".encode()).hexdigest()
        else:
            # Fallback for edge cases
            import random
            import string

            fallback = (
                content.name
                or content.id
                or ("unknown_content" + "".join(random.choices(string.ascii_lowercase + string.digits, k=6)))
            )
            return hashlib.sha256(fallback.encode()).hexdigest()

    def _get_content_size(self, content: Content) -> Optional[int]:
        if content.file_data:
            if content.file_data.size:
                return content.file_data.size
            else:
                return len(content.file_data.content)
        elif content.size:
            return content.size
        else:
            return None

    def _add_to_contents_db(self, content: Content) -> Optional[KnowledgeRow]:
        if self.contents_db:
            created_at = content.created_at if content.created_at else int(time.time())
            updated_at = content.updated_at if content.updated_at else int(time.time())

            file_type = (
                content.file_type
                if content.file_type
                else content.file_data.type
                if content.file_data and content.file_data.type
                else None
            )
            file_data = content.file_data.to_dict() if content.file_data else None

            content_row = KnowledgeRow(
                id=content.id,
                name=content.name if content.name else "",
                description=content.description if content.description else "",
                metadata=content.metadata,
                type=file_type,
                size=self._get_content_size(content),
                linked_to=self.name,
                access_count=0,
                status=content.status if content.status else ContentStatus.PROCESSING,
                sync_status=content.sync_status if content.sync_status else ContentStatus.PROCESSING,
                definition=content.definition,
                status_message="",
                content_hash=content.content_hash,
                file_data=file_data,
                created_at=created_at,
                updated_at=updated_at,
                config={ "auth": content.auth },
                collection_id=content.collection_id,
                workspace_id=content.workspace_id,
            )
            self.contents_db.upsert_knowledge_content(knowledge_row=content_row)
            return content_row

    def _update_content(self, content: Content) -> Optional[Dict[str, Any]]:
        if self.contents_db:
            if not content.id:
                log_warning("Content id is required to update Knowledge content")
                return None

            # TODO: we shouldn't check for content here, we should trust the upsert method to handle conflicts
            content_row = self.contents_db.get_knowledge_content(content.id)
            if content_row is None:
                log_warning(f"Content row not found for id: {content.id}, cannot update status")
                return None

            if content.name is not None:
                content_row.name = content.name
            if content.description is not None:
                content_row.description = content.description
            if content.metadata is not None:
                content_row.metadata = content.metadata
            if content.status is not None:
                content_row.status = content.status
            if content.sync_status is not None:
                content_row.sync_status = content.sync_status
            if content.definition is not None:
                content_row.definition = content.definition
            if content.content_hash is not None:
                content_row.content_hash = content.content_hash
            if content.file_data is not None:
                content_row.file_data = content.file_data
            if content.status_message is not None:
                content_row.status_message = content.status_message if content.status_message else ""
            if content.external_id is not None:
                content_row.external_id = content.external_id
            if content.auth is not None:
                content_row.config = { "auth": content.auth, **content_row.config }
            if content.collection_id is not None:
                content_row.collection_id = content.collection_id
            if content.workspace_id is not None:
                content_row.workspace_id = content.workspace_id

            content_row.updated_at = int(time.time())
            self.contents_db.upsert_knowledge_content(knowledge_row=content_row)

            if self.vector_db and content.metadata:
                self.vector_db.update_metadata(content_id=content.id, metadata=content.metadata)

            if content.metadata:
                self.add_filters(content.metadata)

            return content_row.to_dict()

        else:
            log_warning(f"Contents DB not found for knowledge base: {self.name}")
            return None

    async def _process_lightrag_content(self, content: Content, content_type: KnowledgeContentOrigin) -> None:
        self._add_to_contents_db(content)
        if content_type == KnowledgeContentOrigin.PATH:
            if content.file_data is None:
                log_warning("No file data provided")

            if content.path is None:
                log_error("No path provided for content")
                return

            path = Path(content.path)

            log_info(f"Uploading file to LightRAG from path: {path}")
            try:
                # Read the file content from path
                with open(path, "rb") as f:
                    file_content = f.read()

                # Get file type from extension or content.file_type
                file_type = content.file_type or path.suffix

                if self.vector_db and hasattr(self.vector_db, "insert_file_bytes"):
                    result = await self.vector_db.insert_file_bytes(
                        file_content=file_content,
                        filename=path.name,  # Use the original filename with extension
                        content_type=file_type,
                        send_metadata=True,  # Enable metadata so server knows the file type
                    )

                else:
                    log_error("Vector database does not support file insertion")
                    content.status = ContentStatus.FAILED
                    self._update_content(content)
                    return
                content.external_id = result
                content.status = ContentStatus.COMPLETED
                self._update_content(content)
                return

            except Exception as e:
                log_error(f"Error uploading file to LightRAG: {e}")
                content.status = ContentStatus.FAILED
                content.status_message = f"Could not upload to LightRAG: {str(e)}"
                self._update_content(content)
                return

        elif content_type == KnowledgeContentOrigin.URL:
            log_info(f"Uploading file to LightRAG from URL: {content.url}")
            try:
                reader = content.reader or self.website_reader
                if reader is None:
                    log_error("No URL reader available")
                    content.status = ContentStatus.FAILED
                    self._update_content(content)
                    return

                reader.chunk = False
                read_documents = reader.read(content.url, name=content.name)

                for read_document in read_documents:
                    read_document.content_id = content.id

                if not read_documents:
                    log_error("No documents read from URL")
                    content.status = ContentStatus.FAILED
                    self._update_content(content)
                    return

                if self.vector_db and hasattr(self.vector_db, "insert_text"):
                    result = await self.vector_db.insert_text(
                        file_source=content.url,
                        text=read_documents[0].content,
                    )
                else:
                    log_error("Vector database does not support text insertion")
                    content.status = ContentStatus.FAILED
                    self._update_content(content)
                    return

                content.external_id = result
                content.status = ContentStatus.COMPLETED
                self._update_content(content)
                return

            except Exception as e:
                log_error(f"Error uploading file to LightRAG: {e}")
                content.status = ContentStatus.FAILED
                content.status_message = f"Could not upload to LightRAG: {str(e)}"
                self._update_content(content)
                return

        elif content_type == KnowledgeContentOrigin.CONTENT:
            filename = (
                content.file_data.filename if content.file_data and content.file_data.filename else "uploaded_file"
            )
            log_info(f"Uploading file to LightRAG: {filename}")

            # Use the content from file_data
            if content.file_data and content.file_data.content:
                if self.vector_db and hasattr(self.vector_db, "insert_file_bytes"):
                    result = await self.vector_db.insert_file_bytes(
                        file_content=content.file_data.content,
                        filename=filename,
                        content_type=content.file_data.type,
                        send_metadata=True,  # Enable metadata so server knows the file type
                    )
                else:
                    log_error("Vector database does not support file insertion")
                    content.status = ContentStatus.FAILED
                    self._update_content(content)
                    return
                content.external_id = result
                content.status = ContentStatus.COMPLETED
                self._update_content(content)
            else:
                log_warning(f"No file data available for LightRAG upload: {content.name}")
            return

        elif content_type == KnowledgeContentOrigin.TOPIC:
            log_info(f"Uploading file to LightRAG: {content.name}")

            if content.reader is None:
                log_error("No reader available for topic content")
                content.status = ContentStatus.FAILED
                self._update_content(content)
                return

            if not content.topics:
                log_error("No topics available for content")
                content.status = ContentStatus.FAILED
                self._update_content(content)
                return

            read_documents = content.reader.read(content.topics)
            if len(read_documents) > 0:

                if self.vector_db and hasattr(self.vector_db, "insert_text"):
                    result = await self.vector_db.insert_text(
                        file_source=content.topics[0],
                        text=read_documents[0].content,
                    )
                else:
                    log_error("Vector database does not support text insertion")
                    content.status = ContentStatus.FAILED
                    self._update_content(content)
                    return
                content.external_id = result
                content.status = ContentStatus.COMPLETED
                self._update_content(content)
                return
            else:
                log_warning(f"No documents found for LightRAG upload: {content.name}")
                return

    def search(
        self, query: str, max_results: Optional[int] = None, filters: Optional[Dict[str, Any]] = None, collection_id: Optional[str] = None, workspace_id: Optional[str] = None
    ) -> List[Document]:
        """Returns relevant documents matching a query"""

        try:
            if self.vector_db is None:
                log_warning("No vector db provided")
                return []

            _max_results = max_results or self.max_results
            log_debug(f"Getting {_max_results} relevant documents for query: {query}")
            return self.vector_db.search(query=query, limit=_max_results, filters=filters, collection_id=collection_id, workspace_id=workspace_id)
        except Exception as e:
            log_error(f"Error searching for documents: {e}")
            return []

    async def async_search(
        self, query: str, max_results: Optional[int] = None, filters: Optional[Dict[str, Any]] = None, collection_id: Optional[str] = None, workspace_id: Optional[str] = None
    ) -> List[Document]:
        """Returns relevant documents matching a query"""

        try:
            if self.vector_db is None:
                log_warning("No vector db provided")
                return []

            _max_results = max_results or self.max_results
            log_debug(f"Getting {_max_results} relevant documents for query: {query}")
            try:
                return await self.vector_db.async_search(query=query, limit=_max_results, filters=filters, collection_id=collection_id, workspace_id=workspace_id)
            except NotImplementedError:
                log_info("Vector db does not support async search")
                return self.search(query=query, max_results=_max_results, filters=filters)
        except Exception as e:
            log_error(f"Error searching for documents: {e}")
            return []

    def validate_filters(self, filters: Optional[Dict[str, Any]]) -> Tuple[Dict[str, Any], List[str]]:
        if self.valid_metadata_filters is None:
            self.valid_metadata_filters = set()
        self.valid_metadata_filters.update(self._get_filters_from_db)

        if not filters:
            return {}, []

        valid_filters: Dict[str, Any] = {}
        invalid_keys = []

        # If no metadata filters tracked yet, all keys are considered invalid
        if self.valid_metadata_filters is None:
            invalid_keys = list(filters.keys())
            log_debug(f"No valid metadata filters tracked yet. All filter keys considered invalid: {invalid_keys}")
            return {}, invalid_keys

        for key, value in filters.items():
            # Handle both normal keys and prefixed keys like meta_data.key
            base_key = key.split(".")[-1] if "." in key else key
            if base_key in self.valid_metadata_filters or key in self.valid_metadata_filters:
                valid_filters[key] = value
            else:
                invalid_keys.append(key)
                log_debug(f"Invalid filter key: {key} - not present in knowledge base")

        return valid_filters, invalid_keys

    def add_filters(self, metadata: Dict[str, Any]) -> None:
        if self.valid_metadata_filters is None:
            self.valid_metadata_filters = set()

        if metadata is not None:
            for key in metadata.keys():
                self.valid_metadata_filters.add(key)

    @cached_property
    def _get_filters_from_db(self) -> Set[str]:
        if self.contents_db is None:
            return set()
        contents, _ = self.get_content()
        valid_filters: Set[str] = set()
        for content in contents:
            if content.metadata:
                valid_filters.update(content.metadata.keys())
        return valid_filters

    def remove_vector_by_id(self, id: str) -> bool:
        if self.vector_db is None:
            log_warning("No vector DB provided")
            return False
        return self.vector_db.delete_by_id(id)

    def remove_vectors_by_name(self, name: str) -> bool:
        if self.vector_db is None:
            log_warning("No vector DB provided")
            return False
        return self.vector_db.delete_by_name(name)

    def remove_vectors_by_metadata(self, metadata: Dict[str, Any]) -> bool:
        if self.vector_db is None:
            log_warning("No vector DB provided")
            return False
        return self.vector_db.delete_by_metadata(metadata)

    # --- API Only Methods ---

    def patch_content(self, content: Content) -> Optional[Dict[str, Any]]:
        return self._update_content(content)

    def get_content_by_id(self, content_id: str) -> Optional[Content]:
        if self.contents_db is None:
            raise ValueError("No contents db provided")
        content_row = self.contents_db.get_knowledge_content(content_id)
        if content_row is None:
            return None
        content = Content(
            id=content_row.id,
            name=content_row.name,
            description=content_row.description,
            metadata=content_row.metadata,
            collection_id=content_row.collection_id,
            workspace_id=content_row.workspace_id,
            file_type=content_row.type,
            size=content_row.size,
            status=ContentStatus(content_row.status) if content_row.status else None,
            status_message=content_row.status_message,
            created_at=content_row.created_at,
            updated_at=content_row.updated_at if content_row.updated_at else content_row.created_at,
            external_id=content_row.external_id,
        )
        return content

    def get_content(
        self,
        limit: Optional[int] = None,
        page: Optional[int] = None,
        sort_by: Optional[str] = None,
        sort_order: Optional[str] = None,
    ) -> Tuple[List[Content], int]:
        if self.contents_db is None:
            raise ValueError("No contents db provided")
        contents, count = self.contents_db.get_knowledge_contents(
            limit=limit, page=page, sort_by=sort_by, sort_order=sort_order
        )

        result = []
        for content_row in contents:
            # Create Content from database row
            content = Content(
                id=content_row.id,
                name=content_row.name,
                description=content_row.description,
                metadata=content_row.metadata,
                size=content_row.size,
                file_type=content_row.type,
                status=ContentStatus(content_row.status) if content_row.status else None,
                status_message=content_row.status_message,
                created_at=content_row.created_at,
                updated_at=content_row.updated_at if content_row.updated_at else content_row.created_at,
                external_id=content_row.external_id,
                collection_id=content_row.collection_id,
                workspace_id=content_row.workspace_id,
            )
            result.append(content)
        return result, count

    def get_content_status(self, content_id: str) -> Tuple[Optional[ContentStatus], Optional[str]]:
        if self.contents_db is None:
            raise ValueError("No contents db provided")
        content_row = self.contents_db.get_knowledge_content(content_id)
        if content_row is None:
            return None, "Content not found"

        # Convert string status to enum, defaulting to PROCESSING if unknown
        status_str = content_row.status
        try:
            status = ContentStatus(status_str.lower()) if status_str else ContentStatus.PROCESSING
        except ValueError:
            # Handle legacy or unknown statuses
            if status_str and "failed" in status_str.lower():
                status = ContentStatus.FAILED
            elif status_str and "completed" in status_str.lower():
                status = ContentStatus.COMPLETED
            else:
                status = ContentStatus.PROCESSING

        return status, content_row.status_message

    def remove_content_by_id(self, content_id: str):
        if self.vector_db is not None:
            if self.vector_db.__class__.__name__ == "LightRag":
                # For LightRAG, get the content first to find the external_id
                content = self.get_content_by_id(content_id)
                if content and content.external_id:
                    self.vector_db.delete_by_external_id(content.external_id)  # type: ignore
                else:
                    log_warning(f"No external_id found for content {content_id}, cannot delete from LightRAG")
            else:
                self.vector_db.delete_by_content_id(content_id)

        if self.contents_db is not None:
            self.contents_db.delete_knowledge_content(content_id)

    def remove_all_content(self):
        contents, _ = self.get_content()
        for content in contents:
            if content.id is not None:
                self.remove_content_by_id(content.id)

    # --- Reader Factory Integration ---

    def construct_readers(self):
        """Initialize readers dictionary for lazy loading."""
        # Initialize empty readers dict - readers will be created on-demand
        if self.readers is None:
            self.readers = {}

    def add_reader(self, reader: Reader):
        """Add a custom reader to the knowledge base."""
        if self.readers is None:
            self.readers = {}

        # Generate a key for the reader
        reader_key = self._generate_reader_key(reader)
        self.readers[reader_key] = reader
        return reader

    def get_readers(self) -> Dict[str, Reader]:
        """Get all currently loaded readers (only returns readers that have been used)."""
        if self.readers is None:
            self.readers = {}

        return self.readers

    def _generate_reader_key(self, reader: Reader) -> str:
        """Generate a key for a reader instance."""
        if reader.name:
            return f"{reader.name.lower().replace(' ', '_')}"
        else:
            return f"{reader.__class__.__name__.lower().replace(' ', '_')}"

    def _select_reader(self, extension: str) -> Reader:
        """Select the appropriate reader for a file extension."""
        log_info(f"Selecting reader for extension: {extension}")
        return ReaderFactory.get_reader_for_extension(extension)

    def get_filters(self) -> List[str]:
        return [
            "filter_tag_1",
            "filter_tag2",
        ]

    # --- Convenience Properties for Backward Compatibility ---

    def _is_text_mime_type(self, mime_type: str) -> bool:
        """
        Check if a MIME type represents text content that can be safely encoded as UTF-8.

        Args:
            mime_type: The MIME type to check

        Returns:
            bool: True if it's a text type, False if binary
        """
        if not mime_type:
            return False

        text_types = [
            "text/",
            "application/json",
            "application/xml",
            "application/javascript",
            "application/csv",
            "application/sql",
        ]

        return any(mime_type.startswith(t) for t in text_types)

    def _should_include_file(self, file_path: str, include: Optional[List[str]], exclude: Optional[List[str]]) -> bool:
        """
        Determine if a file should be included based on include/exclude patterns.

        Logic:
        1. If include is specified, file must match at least one include pattern
        2. If exclude is specified, file must not match any exclude pattern
        3. If neither specified, include all files

        Args:
            file_path: Path to the file to check
            include: Optional list of include patterns (glob-style)
            exclude: Optional list of exclude patterns (glob-style)

        Returns:
            bool: True if file should be included, False otherwise
        """
        import fnmatch

        # If include patterns specified, file must match at least one
        if include:
            if not any(fnmatch.fnmatch(file_path, pattern) for pattern in include):
                return False

        # If exclude patterns specified, file must not match any
        if exclude:
            if any(fnmatch.fnmatch(file_path, pattern) for pattern in exclude):
                return False

        return True

    def _get_reader(self, reader_type: str) -> Optional[Reader]:
        """Get a cached reader or create it if not cached, handling missing dependencies gracefully."""
        if self.readers is None:
            self.readers = {}

        if reader_type not in self.readers:
            try:
                reader = ReaderFactory.create_reader(reader_type)
                if reader:
                    self.readers[reader_type] = reader
                else:
                    return None

            except Exception as e:
                log_warning(f"Cannot create {reader_type} reader {e}")
                return None

        return self.readers.get(reader_type)

    @property
    def pdf_reader(self) -> Optional[Reader]:
        """PDF reader - lazy loaded via factory."""
        return self._get_reader("pdf")

    @property
    def csv_reader(self) -> Optional[Reader]:
        """CSV reader - lazy loaded via factory."""
        return self._get_reader("csv")

    @property
    def docx_reader(self) -> Optional[Reader]:
        """Docx reader - lazy loaded via factory."""
        return self._get_reader("docx")

    @property
    def json_reader(self) -> Optional[Reader]:
        """JSON reader - lazy loaded via factory."""
        return self._get_reader("json")

    @property
    def markdown_reader(self) -> Optional[Reader]:
        """Markdown reader - lazy loaded via factory."""
        return self._get_reader("markdown")

    @property
    def text_reader(self) -> Optional[Reader]:
        """Text reader - lazy loaded via factory."""
        return self._get_reader("text")

    @property
    def website_reader(self) -> Optional[Reader]:
        """Website reader - lazy loaded via factory."""
        return self._get_reader("website")

    @property
    def firecrawl_reader(self) -> Optional[Reader]:
        """Firecrawl reader - lazy loaded via factory."""
        return self._get_reader("firecrawl")

    @property
    def youtube_reader(self) -> Optional[Reader]:
        """YouTube reader - lazy loaded via factory."""
        return self._get_reader("youtube")
