import asyncio
import hashlib
import io
import time
from dataclasses import dataclass
from enum import Enum
from io import BytesIO
from os.path import basename
from pathlib import Path
from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast, overload

from httpx import AsyncClient

from agno.db.base import AsyncBaseDb, BaseDb
from agno.db.schemas.knowledge import KnowledgeRow
from agno.filters import FilterExpr
from agno.knowledge.content import Content, ContentAuth, ContentStatus, FileData
from agno.knowledge.document import Document
from agno.knowledge.reader import Reader, ReaderFactory
from agno.knowledge.remote_content.remote_content import GCSContent, RemoteContent, S3Content
from agno.utils.http import async_fetch_with_retry
from agno.utils.log import log_debug, log_error, log_info, log_warning
from agno.utils.string import generate_id

ContentDict = Dict[str, Union[str, Dict[str, str]]]


class KnowledgeContentOrigin(Enum):
    PATH = "path"
    URL = "url"
    TOPIC = "topic"
    CONTENT = "content"


@dataclass
class Knowledge:
    """Knowledge class"""

    name: Optional[str] = None
    description: Optional[str] = None
    vector_db: Optional[Any] = None
    contents_db: Optional[Union[BaseDb, AsyncBaseDb]] = None
    max_results: int = 10
    readers: Optional[Dict[str, Reader]] = None

    def __post_init__(self):
        from agno.vectordb import VectorDb

        self.vector_db = cast(VectorDb, self.vector_db)
        if self.vector_db and not self.vector_db.exists():
            self.vector_db.create()

        self.construct_readers()

    # --- Add Contents ---
    @overload
    async def add_contents_async(self, contents: List[ContentDict]) -> None: ...

    @overload
    async def add_contents_async(
        self,
        *,
        paths: Optional[List[str]] = None,
        urls: Optional[List[str]] = None,
        metadata: Optional[Dict[str, str]] = None,
        topics: Optional[List[str]] = None,
        text_contents: Optional[List[str]] = None,
        reader: Optional[Reader] = None,
        include: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
        upsert: bool = True,
        skip_if_exists: bool = False,
        remote_content: Optional[RemoteContent] = None,
    ) -> None: ...

    async def add_contents_async(self, *args, **kwargs) -> None:
        if args and isinstance(args[0], list):
            arguments = args[0]
            upsert = kwargs.get("upsert", True)
            skip_if_exists = kwargs.get("skip_if_exists", False)
            for argument in arguments:
                await self.add_content_async(
                    name=argument.get("name"),
                    description=argument.get("description"),
                    path=argument.get("path"),
                    url=argument.get("url"),
                    metadata=argument.get("metadata"),
                    topics=argument.get("topics"),
                    text_content=argument.get("text_content"),
                    reader=argument.get("reader"),
                    include=argument.get("include"),
                    exclude=argument.get("exclude"),
                    upsert=argument.get("upsert", upsert),
                    skip_if_exists=argument.get("skip_if_exists", skip_if_exists),
                    remote_content=argument.get("remote_content", None),
                )

        elif kwargs:
            name = kwargs.get("name", [])
            metadata = kwargs.get("metadata", {})
            description = kwargs.get("description", [])
            topics = kwargs.get("topics", [])
            reader = kwargs.get("reader", None)
            paths = kwargs.get("paths", [])
            urls = kwargs.get("urls", [])
            text_contents = kwargs.get("text_contents", [])
            include = kwargs.get("include")
            exclude = kwargs.get("exclude")
            upsert = kwargs.get("upsert", True)
            skip_if_exists = kwargs.get("skip_if_exists", False)
            remote_content = kwargs.get("remote_content", None)
            for path in paths:
                await self.add_content_async(
                    name=name,
                    description=description,
                    path=path,
                    metadata=metadata,
                    include=include,
                    exclude=exclude,
                    upsert=upsert,
                    skip_if_exists=skip_if_exists,
                    reader=reader,
                )
            for url in urls:
                await self.add_content_async(
                    name=name,
                    description=description,
                    url=url,
                    metadata=metadata,
                    include=include,
                    exclude=exclude,
                    upsert=upsert,
                    skip_if_exists=skip_if_exists,
                    reader=reader,
                )
            for i, text_content in enumerate(text_contents):
                content_name = f"{name}_{i}" if name else f"text_content_{i}"
                log_debug(f"Adding text content: {content_name}")
                await self.add_content_async(
                    name=content_name,
                    description=description,
                    text_content=text_content,
                    metadata=metadata,
                    include=include,
                    exclude=exclude,
                    upsert=upsert,
                    skip_if_exists=skip_if_exists,
                    reader=reader,
                )
            if topics:
                await self.add_content_async(
                    name=name,
                    description=description,
                    topics=topics,
                    metadata=metadata,
                    include=include,
                    exclude=exclude,
                    upsert=upsert,
                    skip_if_exists=skip_if_exists,
                    reader=reader,
                )

            if remote_content:
                await self.add_content_async(
                    name=name,
                    metadata=metadata,
                    description=description,
                    remote_content=remote_content,
                    upsert=upsert,
                    skip_if_exists=skip_if_exists,
                    reader=reader,
                )

        else:
            raise ValueError("Invalid usage of add_contents.")

    @overload
    def add_contents(self, contents: List[ContentDict]) -> None: ...

    @overload
    def add_contents(
        self,
        *,
        paths: Optional[List[str]] = None,
        urls: Optional[List[str]] = None,
        metadata: Optional[Dict[str, str]] = None,
        topics: Optional[List[str]] = None,
        text_contents: Optional[List[str]] = None,
        reader: Optional[Reader] = None,
        include: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
        upsert: bool = True,
        skip_if_exists: bool = False,
        remote_content: Optional[RemoteContent] = None,
    ) -> None: ...

    def add_contents(self, *args, **kwargs) -> None:
        """
        Synchronously add multiple content items to the knowledge base.

        This method wraps the asynchronous add_contents method

        Supports two usage patterns:
        1. Pass a list of content dictionaries as first argument
        2. Pass keyword arguments with paths, urls, metadata, etc.

        Args:
            contents: List of content dictionaries (when used as first overload)
            paths: Optional list of file paths to load content from
            urls: Optional list of URLs to load content from
            metadata: Optional metadata dictionary to apply to all content
            topics: Optional list of topics to add
            text_contents: Optional list of text content strings to add
            reader: Optional reader to use for processing content
            include: Optional list of file patterns to include
            exclude: Optional list of file patterns to exclude
            upsert: Whether to update existing content if it already exists
            skip_if_exists: Whether to skip adding content if it already exists
            remote_content: Optional remote content (S3, GCS, etc.) to add
        """
        asyncio.run(self.add_contents_async(*args, **kwargs))

    # --- Add Content ---

    @overload
    async def add_content_async(
        self,
        *,
        path: Optional[str] = None,
        url: Optional[str] = None,
        text_content: Optional[str] = None,
        metadata: Optional[Dict[str, str]] = None,
        include: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
        upsert: bool = True,
        skip_if_exists: bool = False,
        reader: Optional[Reader] = None,
        auth: Optional[ContentAuth] = None,
    ) -> None: ...

    @overload
    async def add_content_async(self, *args, **kwargs) -> None: ...

    async def add_content_async(
        self,
        name: Optional[str] = None,
        description: Optional[str] = None,
        path: Optional[str] = None,
        url: Optional[str] = None,
        text_content: Optional[str] = None,
        metadata: Optional[Dict[str, Any]] = None,
        topics: Optional[List[str]] = None,
        remote_content: Optional[RemoteContent] = None,
        reader: Optional[Reader] = None,
        include: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
        upsert: bool = True,
        skip_if_exists: bool = True,
        auth: Optional[ContentAuth] = None,
    ) -> None:
        # Validation: At least one of the parameters must be provided
        if all(argument is None for argument in [path, url, text_content, topics, remote_content]):
            log_warning(
                "At least one of 'path', 'url', 'text_content', 'topics', or 'remote_content' must be provided."
            )
            return

        if not skip_if_exists:
            log_debug("skip_if_exists is disabled, disabling upsert")
            upsert = False

        content = None
        file_data = None
        if text_content:
            file_data = FileData(content=text_content, type="Text")

        content = Content(
            name=name,
            description=description,
            path=path,
            url=url,
            file_data=file_data if file_data else None,
            metadata=metadata,
            topics=topics,
            remote_content=remote_content,
            reader=reader,
            auth=auth,
        )
        content.content_hash = self._build_content_hash(content)
        content.id = generate_id(content.content_hash)

        await self._load_content(content, upsert, skip_if_exists, include, exclude)

    @overload
    def add_content(
        self,
        *,
        path: Optional[str] = None,
        url: Optional[str] = None,
        text_content: Optional[str] = None,
        metadata: Optional[Dict[str, str]] = None,
        include: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
        upsert: bool = True,
        skip_if_exists: bool = False,
        reader: Optional[Reader] = None,
        auth: Optional[ContentAuth] = None,
    ) -> None: ...

    @overload
    def add_content(self, *args, **kwargs) -> None: ...

    def add_content(
        self,
        name: Optional[str] = None,
        description: Optional[str] = None,
        path: Optional[str] = None,
        url: Optional[str] = None,
        text_content: Optional[str] = None,
        metadata: Optional[Dict[str, Any]] = None,
        topics: Optional[List[str]] = None,
        remote_content: Optional[RemoteContent] = None,
        reader: Optional[Reader] = None,
        include: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
        upsert: bool = True,
        skip_if_exists: bool = False,
        auth: Optional[ContentAuth] = None,
    ) -> None:
        """
        Synchronously add content to the knowledge base.

        Args:
            name: Optional name for the content
            description: Optional description for the content
            path: Optional file path to load content from
            url: Optional URL to load content from
            text_content: Optional text content to add directly
            metadata: Optional metadata dictionary
            topics: Optional list of topics
            remote_content: Optional cloud storage configuration
            reader: Optional custom reader for processing the content
            include: Optional list of file patterns to include
            exclude: Optional list of file patterns to exclude
            upsert: Whether to update existing content if it already exists
            skip_if_exists: Whether to skip adding content if it already exists
        """
        asyncio.run(
            self.add_content_async(
                name=name,
                description=description,
                path=path,
                url=url,
                text_content=text_content,
                metadata=metadata,
                topics=topics,
                remote_content=remote_content,
                reader=reader,
                include=include,
                exclude=exclude,
                upsert=upsert,
                skip_if_exists=skip_if_exists,
                auth=auth,
            )
        )

    def _should_skip(self, content_hash: str, skip_if_exists: bool) -> bool:
        """
        Handle the skip_if_exists logic for content that already exists in the vector database.

        Args:
            content_hash: The content hash string to check for existence
            skip_if_exists: Whether to skip if content already exists

        Returns:
            bool: True if should skip processing, False if should continue
        """
        from agno.vectordb import VectorDb

        self.vector_db = cast(VectorDb, self.vector_db)
        if self.vector_db and self.vector_db.content_hash_exists(content_hash) and skip_if_exists:
            log_debug(f"Content already exists: {content_hash}, skipping...")
            return True

        return False

    async def _load_from_path(
        self,
        content: Content,
        upsert: bool,
        skip_if_exists: bool,
        include: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
    ):
        from agno.vectordb import VectorDb

        self.vector_db = cast(VectorDb, self.vector_db)

        log_info(f"Adding content from path, {content.id}, {content.name}, {content.path}, {content.description}")
        path = Path(content.path)  # type: ignore

        if path.is_file():
            if self._should_include_file(str(path), include, exclude):
                log_debug(f"Adding file {path} due to include/exclude filters")

                await self._add_to_contents_db(content)
                if self._should_skip(content.content_hash, skip_if_exists):  # type: ignore[arg-type]
                    content.status = ContentStatus.COMPLETED
                    await self._aupdate_content(content)
                    return

                # Handle LightRAG special case - read file and upload directly
                if self.vector_db.__class__.__name__ == "LightRag":
                    await self._process_lightrag_content(content, KnowledgeContentOrigin.PATH)
                    return

                if content.reader:
                    # TODO: We will refactor this to eventually pass authorization to all readers
                    import inspect

                    read_signature = inspect.signature(content.reader.read)
                    if "password" in read_signature.parameters and content.auth and content.auth.password:
                        read_documents = content.reader.read(
                            path, name=content.name or path.name, password=content.auth.password
                        )
                    else:
                        read_documents = content.reader.read(path, name=content.name or path.name)

                else:
                    reader = ReaderFactory.get_reader_for_extension(path.suffix)
                    log_debug(f"Using Reader: {reader.__class__.__name__}")
                    if reader:
                        # TODO: We will refactor this to eventually pass authorization to all readers
                        import inspect

                        read_signature = inspect.signature(reader.read)
                        if "password" in read_signature.parameters and content.auth and content.auth.password:
                            read_documents = reader.read(
                                path, name=content.name or path.name, password=content.auth.password
                            )
                        else:
                            read_documents = reader.read(path, name=content.name or path.name)

                if not content.file_type:
                    content.file_type = path.suffix

                if not content.size and content.file_data:
                    content.size = len(content.file_data.content)  # type: ignore
                if not content.size:
                    try:
                        content.size = path.stat().st_size
                    except (OSError, IOError) as e:
                        log_warning(f"Could not get file size for {path}: {e}")
                        content.size = 0

                for read_document in read_documents:
                    read_document.content_id = content.id

                await self._handle_vector_db_insert(content, read_documents, upsert)

        elif path.is_dir():
            for file_path in path.iterdir():
                # Apply include/exclude filtering
                if not self._should_include_file(str(file_path), include, exclude):
                    log_debug(f"Skipping file {file_path} due to include/exclude filters")
                    continue

                file_content = Content(
                    name=content.name,
                    path=str(file_path),
                    metadata=content.metadata,
                    description=content.description,
                    reader=content.reader,
                )
                file_content.content_hash = self._build_content_hash(file_content)
                file_content.id = generate_id(file_content.content_hash)

                await self._load_from_path(file_content, upsert, skip_if_exists, include, exclude)
        else:
            log_warning(f"Invalid path: {path}")

    async def _load_from_url(
        self,
        content: Content,
        upsert: bool,
        skip_if_exists: bool,
    ):
        """Load the content in the contextual URL

        1. Set content hash
        2. Validate the URL
        3. Read the content
        4. Prepare and insert the content in the vector database
        """
        from agno.vectordb import VectorDb

        self.vector_db = cast(VectorDb, self.vector_db)

        log_info(f"Adding content from URL {content.url}")
        content.file_type = "url"

        if not content.url:
            raise ValueError("No url provided")

        # 1. Add content to contents database
        await self._add_to_contents_db(content)
        if self._should_skip(content.content_hash, skip_if_exists):  # type: ignore[arg-type]
            content.status = ContentStatus.COMPLETED
            await self._aupdate_content(content)
            return

        if self.vector_db.__class__.__name__ == "LightRag":
            await self._process_lightrag_content(content, KnowledgeContentOrigin.URL)
            return

        # 2. Validate URL
        try:
            from urllib.parse import urlparse

            parsed_url = urlparse(content.url)
            if not all([parsed_url.scheme, parsed_url.netloc]):
                content.status = ContentStatus.FAILED
                content.status_message = f"Invalid URL format: {content.url}"
                await self._aupdate_content(content)
                log_warning(f"Invalid URL format: {content.url}")
        except Exception as e:
            content.status = ContentStatus.FAILED
            content.status_message = f"Invalid URL: {content.url} - {str(e)}"
            await self._aupdate_content(content)
            log_warning(f"Invalid URL: {content.url} - {str(e)}")

        # 3. Fetch and load content if file has an extension
        url_path = Path(parsed_url.path)
        file_extension = url_path.suffix.lower()

        bytes_content = None
        if file_extension:
            async with AsyncClient() as client:
                response = await async_fetch_with_retry(content.url, client=client)
            bytes_content = BytesIO(response.content)

        # 4. Select reader
        # If a reader was provided by the user, use it
        reader = content.reader
        name = content.name if content.name else content.url
        # Else select based on file extension
        if reader is None:
            if file_extension == ".csv":
                name = basename(parsed_url.path) or "data.csv"
                reader = self.csv_reader
            elif file_extension == ".pdf":
                reader = self.pdf_reader
            elif file_extension == ".docx":
                reader = self.docx_reader
            elif file_extension == ".pptx":
                reader = self.pptx_reader
            elif file_extension == ".json":
                reader = self.json_reader
            elif file_extension == ".markdown":
                reader = self.markdown_reader
            else:
                reader = self.text_reader

        # 5. Read content
        try:
            read_documents = []
            if reader is not None:
                # TODO: We will refactor this to eventually pass authorization to all readers
                import inspect

                read_signature = inspect.signature(reader.read)
                if reader.__class__.__name__ == "YouTubeReader":
                    read_documents = reader.read(content.url, name=name)
                elif "password" in read_signature.parameters and content.auth and content.auth.password:
                    if bytes_content:
                        read_documents = reader.read(bytes_content, name=name, password=content.auth.password)
                    else:
                        read_documents = reader.read(content.url, name=name, password=content.auth.password)
                else:
                    if bytes_content:
                        read_documents = reader.read(bytes_content, name=name)
                    else:
                        read_documents = reader.read(content.url, name=name)

        except Exception as e:
            log_error(f"Error reading URL: {content.url} - {str(e)}")
            content.status = ContentStatus.FAILED
            content.status_message = f"Error reading URL: {content.url} - {str(e)}"
            await self._aupdate_content(content)
            return

        # 6. Chunk documents if needed
        if reader and not reader.chunk:
            read_documents = await reader.chunk_documents_async(read_documents)
        # 7. Prepare and insert the content in the vector database
        file_size = 0
        if read_documents:
            for read_document in read_documents:
                if read_document.size:
                    file_size += read_document.size
                read_document.content_id = content.id
        await self._handle_vector_db_insert(content, read_documents, upsert)

    async def _load_from_content(
        self,
        content: Content,
        upsert: bool = True,
        skip_if_exists: bool = False,
    ):
        from agno.vectordb import VectorDb

        self.vector_db = cast(VectorDb, self.vector_db)

        if content.name:
            name = content.name
        elif content.file_data and content.file_data.content:
            if isinstance(content.file_data.content, bytes):
                name = content.file_data.content[:10].decode("utf-8", errors="ignore")
            elif isinstance(content.file_data.content, str):
                name = (
                    content.file_data.content[:10]
                    if len(content.file_data.content) >= 10
                    else content.file_data.content
                )
            else:
                name = str(content.file_data.content)[:10]
        else:
            name = None

        if name is not None:
            content.name = name

        log_info(f"Adding content from {content.name}")

        await self._add_to_contents_db(content)
        if self._should_skip(content.content_hash, skip_if_exists):  # type: ignore[arg-type]
            content.status = ContentStatus.COMPLETED
            await self._aupdate_content(content)
            return

        if content.file_data and self.vector_db.__class__.__name__ == "LightRag":
            await self._process_lightrag_content(content, KnowledgeContentOrigin.CONTENT)
            return

        read_documents = []

        if isinstance(content.file_data, str):
            content_bytes = content.file_data.encode("utf-8", errors="replace")
            content_io = io.BytesIO(content_bytes)

            if content.reader:
                log_debug(f"Using reader: {content.reader.__class__.__name__} to read content")
                read_documents = content.reader.read(content_io, name=name)
            else:
                text_reader = self.text_reader
                if text_reader:
                    read_documents = text_reader.read(content_io, name=name)
                else:
                    content.status = ContentStatus.FAILED
                    content.status_message = "Text reader not available"
                    await self._aupdate_content(content)
                    return

        elif isinstance(content.file_data, FileData):
            if content.file_data.type:
                if isinstance(content.file_data.content, bytes):
                    content_io = io.BytesIO(content.file_data.content)
                elif isinstance(content.file_data.content, str):
                    content_bytes = content.file_data.content.encode("utf-8", errors="replace")
                    content_io = io.BytesIO(content_bytes)
                else:
                    content_io = content.file_data.content  # type: ignore

                # Respect an explicitly provided reader; otherwise select based on file type
                if content.reader:
                    log_debug(f"Using reader: {content.reader.__class__.__name__} to read content")
                    reader = content.reader
                else:
                    reader = self._select_reader(content.file_data.type)
                name = content.name if content.name else f"content_{content.file_data.type}"
                read_documents = reader.read(content_io, name=name)
                for read_document in read_documents:
                    if content.metadata:
                        read_document.meta_data.update(content.metadata)
                    read_document.content_id = content.id

                if len(read_documents) == 0:
                    content.status = ContentStatus.FAILED
                    content.status_message = "Content could not be read"
                    await self._aupdate_content(content)
                    return

        else:
            content.status = ContentStatus.FAILED
            content.status_message = "No content provided"
            await self._aupdate_content(content)
            return

        await self._handle_vector_db_insert(content, read_documents, upsert)

    async def _load_from_topics(
        self,
        content: Content,
        upsert: bool,
        skip_if_exists: bool,
    ):
        from agno.vectordb import VectorDb

        self.vector_db = cast(VectorDb, self.vector_db)
        log_info(f"Adding content from topics: {content.topics}")

        if content.topics is None:
            log_warning("No topics provided for content")
            return

        for topic in content.topics:
            content = Content(
                name=topic,
                metadata=content.metadata,
                reader=content.reader,
                status=ContentStatus.PROCESSING if content.reader else ContentStatus.FAILED,
                file_data=FileData(
                    type="Topic",
                ),
                topics=[topic],
            )
            content.content_hash = self._build_content_hash(content)
            content.id = generate_id(content.content_hash)

            await self._add_to_contents_db(content)
            if self._should_skip(content.content_hash, skip_if_exists):
                content.status = ContentStatus.COMPLETED
                await self._aupdate_content(content)
                return

            if self.vector_db.__class__.__name__ == "LightRag":
                await self._process_lightrag_content(content, KnowledgeContentOrigin.TOPIC)
                return

            if self.vector_db and self.vector_db.content_hash_exists(content.content_hash) and skip_if_exists:
                log_info(f"Content {content.content_hash} already exists, skipping")
                continue

            await self._add_to_contents_db(content)
            if content.reader is None:
                log_error(f"No reader available for topic: {topic}")
                content.status = ContentStatus.FAILED
                content.status_message = "No reader available for topic"
                await self._aupdate_content(content)
                continue

            read_documents = content.reader.read(topic)
            if len(read_documents) > 0:
                for read_document in read_documents:
                    read_document.content_id = content.id
                    if read_document.content:
                        read_document.size = len(read_document.content.encode("utf-8"))
            else:
                content.status = ContentStatus.FAILED
                content.status_message = "No content found for topic"
                await self._aupdate_content(content)

            await self._handle_vector_db_insert(content, read_documents, upsert)

    async def _load_from_remote_content(
        self,
        content: Content,
        upsert: bool,
        skip_if_exists: bool,
    ):
        if content.remote_content is None:
            log_warning("No remote content provided for content")
            return

        remote_content = content.remote_content

        if isinstance(remote_content, S3Content):
            await self._load_from_s3(content, upsert, skip_if_exists)

        elif isinstance(remote_content, GCSContent):
            await self._load_from_gcs(content, upsert, skip_if_exists)

        else:
            log_warning(f"Unsupported remote content type: {type(remote_content)}")

    async def _load_from_s3(self, content: Content, upsert: bool, skip_if_exists: bool):
        """Load the contextual S3 content.

        1. Identify objects to read
        2. Setup Content object
        3. Hash content and add it to the contents database
        4. Select reader
        5. Fetch and load the content
        6. Read the content
        7. Prepare and insert the content in the vector database
        8. Remove temporary file if needed
        """
        from agno.cloud.aws.s3.object import S3Object

        remote_content: S3Content = cast(S3Content, content.remote_content)

        # 1. Identify objects to read
        objects_to_read: List[S3Object] = []
        if remote_content.bucket is not None:
            if remote_content.key is not None:
                _object = S3Object(bucket_name=remote_content.bucket.name, name=remote_content.key)
                objects_to_read.append(_object)
            elif remote_content.object is not None:
                objects_to_read.append(remote_content.object)
            elif remote_content.prefix is not None:
                objects_to_read.extend(remote_content.bucket.get_objects(prefix=remote_content.prefix))
            else:
                objects_to_read.extend(remote_content.bucket.get_objects())

        for s3_object in objects_to_read:
            # 2. Setup Content object
            content_name = content.name or ""
            content_name += "_" + (s3_object.name or "")
            content_entry = Content(
                name=content_name,
                description=content.description,
                status=ContentStatus.PROCESSING,
                metadata=content.metadata,
                file_type="s3",
            )

            # 3. Hash content and add it to the contents database
            content_entry.content_hash = self._build_content_hash(content_entry)
            content_entry.id = generate_id(content_entry.content_hash)
            await self._add_to_contents_db(content_entry)
            if self._should_skip(content_entry.content_hash, skip_if_exists):
                content_entry.status = ContentStatus.COMPLETED
                await self._aupdate_content(content_entry)
                return

            # 4. Select reader
            reader = content.reader
            if reader is None:
                if s3_object.uri.endswith(".pdf"):
                    reader = self.pdf_reader
                elif s3_object.uri.endswith(".csv"):
                    reader = self.csv_reader
                elif s3_object.uri.endswith(".docx"):
                    reader = self.docx_reader
                elif s3_object.uri.endswith(".pptx"):
                    reader = self.pptx_reader
                elif s3_object.uri.endswith(".json"):
                    reader = self.json_reader
                elif s3_object.uri.endswith(".markdown"):
                    reader = self.markdown_reader
                else:
                    reader = self.text_reader
            reader = cast(Reader, reader)

            # 5. Fetch and load the content
            temporary_file = None
            obj_name = content_name or s3_object.name.split("/")[-1]
            readable_content: Optional[Union[BytesIO, Path]] = None
            if s3_object.uri.endswith(".pdf"):
                readable_content = BytesIO(s3_object.get_resource().get()["Body"].read())
            else:
                temporary_file = Path("storage").joinpath(obj_name)
                readable_content = temporary_file
                s3_object.download(readable_content)  # type: ignore

            # 6. Read the content
            read_documents = reader.read(readable_content, name=obj_name)

            # 7. Prepare and insert the content in the vector database
            for read_document in read_documents:
                read_document.content_id = content.id
            await self._handle_vector_db_insert(content_entry, read_documents, upsert)

            # 8. Remove temporary file if needed
            if temporary_file:
                temporary_file.unlink()

    async def _load_from_gcs(self, content: Content, upsert: bool, skip_if_exists: bool):
        """Load the contextual GCS content.

        1. Identify objects to read
        2. Setup Content object
        3. Hash content and add it to the contents database
        4. Select reader
        5. Fetch and load the content
        6. Read the content
        7. Prepare and insert the content in the vector database
        """
        remote_content: GCSContent = cast(GCSContent, content.remote_content)

        # 1. Identify objects to read
        objects_to_read = []
        if remote_content.blob_name is not None:
            objects_to_read.append(remote_content.bucket.blob(remote_content.blob_name))  # type: ignore
        elif remote_content.prefix is not None:
            objects_to_read.extend(remote_content.bucket.list_blobs(prefix=remote_content.prefix))  # type: ignore
        else:
            objects_to_read.extend(remote_content.bucket.list_blobs())  # type: ignore

        for gcs_object in objects_to_read:
            # 2. Setup Content object
            name = (content.name or "content") + "_" + gcs_object.name
            content_entry = Content(
                name=name,
                description=content.description,
                status=ContentStatus.PROCESSING,
                metadata=content.metadata,
                file_type="gcs",
            )

            # 3. Hash content and add it to the contents database
            content_entry.content_hash = self._build_content_hash(content_entry)
            content_entry.id = generate_id(content_entry.content_hash)
            await self._add_to_contents_db(content_entry)
            if self._should_skip(content_entry.content_hash, skip_if_exists):
                content_entry.status = ContentStatus.COMPLETED
                await self._aupdate_content(content_entry)
                return

            # 4. Select reader
            reader = content.reader
            if reader is None:
                if gcs_object.name.endswith(".pdf"):
                    reader = self.pdf_reader
                elif gcs_object.name.endswith(".csv"):
                    reader = self.csv_reader
                elif gcs_object.name.endswith(".docx"):
                    reader = self.docx_reader
                elif gcs_object.name.endswith(".pptx"):
                    reader = self.pptx_reader
                elif gcs_object.name.endswith(".json"):
                    reader = self.json_reader
                elif gcs_object.name.endswith(".markdown"):
                    reader = self.markdown_reader
                else:
                    reader = self.text_reader
            reader = cast(Reader, reader)

            # 5. Fetch and load the content
            readable_content = BytesIO(gcs_object.download_as_bytes())

            # 6. Read the content
            read_documents = reader.read(readable_content, name=name)

            # 7. Prepare and insert the content in the vector database
            for read_document in read_documents:
                read_document.content_id = content.id
            await self._handle_vector_db_insert(content_entry, read_documents, upsert)

    async def _handle_vector_db_insert(self, content: Content, read_documents, upsert):
        from agno.vectordb import VectorDb

        self.vector_db = cast(VectorDb, self.vector_db)

        if not self.vector_db:
            log_error("No vector database configured")
            content.status = ContentStatus.FAILED
            content.status_message = "No vector database configured"
            await self._aupdate_content(content)
            return

        if self.vector_db.upsert_available() and upsert:
            try:
                await self.vector_db.async_upsert(content.content_hash, read_documents, content.metadata)  # type: ignore[arg-type]
            except Exception as e:
                log_error(f"Error upserting document: {e}")
                content.status = ContentStatus.FAILED
                content.status_message = "Could not upsert embedding"
                await self._aupdate_content(content)
                return
        else:
            try:
                await self.vector_db.async_insert(
                    content.content_hash,  # type: ignore[arg-type]
                    documents=read_documents,
                    filters=content.metadata,  # type: ignore[arg-type]
                )
            except Exception as e:
                log_error(f"Error inserting document: {e}")
                content.status = ContentStatus.FAILED
                content.status_message = "Could not insert embedding"
                await self._aupdate_content(content)
                return

        content.status = ContentStatus.COMPLETED
        await self._aupdate_content(content)

    async def _load_content(
        self,
        content: Content,
        upsert: bool,
        skip_if_exists: bool,
        include: Optional[List[str]] = None,
        exclude: Optional[List[str]] = None,
    ) -> None:
        if content.path:
            await self._load_from_path(content, upsert, skip_if_exists, include, exclude)

        if content.url:
            await self._load_from_url(content, upsert, skip_if_exists)

        if content.file_data:
            await self._load_from_content(content, upsert, skip_if_exists)

        if content.topics:
            await self._load_from_topics(content, upsert, skip_if_exists)

        if content.remote_content:
            await self._load_from_remote_content(content, upsert, skip_if_exists)

    def _build_content_hash(self, content: Content) -> str:
        """
        Build the content hash from the content.
        """
        if content.path:
            return hashlib.sha256(str(content.path).encode()).hexdigest()
        elif content.url:
            hash = hashlib.sha256(content.url.encode()).hexdigest()
            return hash
        elif content.file_data and content.file_data.content:
            name = content.name or "content"
            return hashlib.sha256(name.encode()).hexdigest()
        elif content.topics and len(content.topics) > 0:
            topic = content.topics[0]
            reader = type(content.reader).__name__ if content.reader else "unknown"
            return hashlib.sha256(f"{topic}-{reader}".encode()).hexdigest()
        else:
            # Fallback for edge cases
            import random
            import string

            fallback = (
                content.name
                or content.id
                or ("unknown_content" + "".join(random.choices(string.ascii_lowercase + string.digits, k=6)))
            )
            return hashlib.sha256(fallback.encode()).hexdigest()

    def _ensure_string_field(self, value: Any, field_name: str, default: str = "") -> str:
        """
        Safely ensure a field is a string, handling various edge cases.

        Args:
            value: The value to convert to string
            field_name: Name of the field for logging purposes
            default: Default string value if conversion fails

        Returns:
            str: A safe string value
        """
        # Handle None/falsy values
        if value is None or value == "":
            return default

        # Handle unexpected list types (the root cause of our Pydantic warning)
        if isinstance(value, list):
            if len(value) == 0:
                log_debug(f"Empty list found for {field_name}, using default: '{default}'")
                return default
            elif len(value) == 1:
                # Single item list, extract the item
                log_debug(f"Single-item list found for {field_name}, extracting: '{value[0]}'")
                return str(value[0]) if value[0] is not None else default
            else:
                # Multiple items, join them
                log_debug(f"Multi-item list found for {field_name}, joining: {value}")
                return " | ".join(str(item) for item in value if item is not None)

        # Handle other unexpected types
        if not isinstance(value, str):
            log_debug(f"Non-string type {type(value)} found for {field_name}, converting: '{value}'")
            try:
                return str(value)
            except Exception as e:
                log_warning(f"Failed to convert {field_name} to string: {e}, using default")
                return default

        # Already a string, return as-is
        return value

    async def _add_to_contents_db(self, content: Content):
        if self.contents_db:
            created_at = content.created_at if content.created_at else int(time.time())
            updated_at = content.updated_at if content.updated_at else int(time.time())

            file_type = (
                content.file_type
                if content.file_type
                else content.file_data.type
                if content.file_data and content.file_data.type
                else None
            )
            # Safely handle string fields with proper type checking
            safe_name = self._ensure_string_field(content.name, "content.name", default="")
            safe_description = self._ensure_string_field(content.description, "content.description", default="")
            safe_linked_to = self._ensure_string_field(self.name, "knowledge.name", default="")
            safe_status_message = self._ensure_string_field(
                content.status_message, "content.status_message", default=""
            )

            content_row = KnowledgeRow(
                id=content.id,
                name=safe_name,
                description=safe_description,
                metadata=content.metadata,
                type=file_type,
                size=content.size
                if content.size
                else len(content.file_data.content)
                if content.file_data and content.file_data.content
                else None,
                linked_to=safe_linked_to,
                access_count=0,
                status=content.status if content.status else ContentStatus.PROCESSING,
                status_message=safe_status_message,
                created_at=created_at,
                updated_at=updated_at,
            )
            if isinstance(self.contents_db, AsyncBaseDb):
                await self.contents_db.upsert_knowledge_content(knowledge_row=content_row)
            else:
                self.contents_db.upsert_knowledge_content(knowledge_row=content_row)

    def _update_content(self, content: Content) -> Optional[Dict[str, Any]]:
        from agno.vectordb import VectorDb

        self.vector_db = cast(VectorDb, self.vector_db)
        if self.contents_db:
            if isinstance(self.contents_db, AsyncBaseDb):
                raise ValueError(
                    "update_content() is not supported with an async DB. Please use aupdate_content() instead."
                )

            if not content.id:
                log_warning("Content id is required to update Knowledge content")
                return None

            # TODO: we shouldn't check for content here, we should trust the upsert method to handle conflicts
            content_row = self.contents_db.get_knowledge_content(content.id)
            if content_row is None:
                log_warning(f"Content row not found for id: {content.id}, cannot update status")
                return None

            # Apply safe string handling for updates as well
            if content.name is not None:
                content_row.name = self._ensure_string_field(content.name, "content.name", default="")
            if content.description is not None:
                content_row.description = self._ensure_string_field(
                    content.description, "content.description", default=""
                )
            if content.metadata is not None:
                content_row.metadata = content.metadata
            if content.status is not None:
                content_row.status = content.status
            if content.status_message is not None:
                content_row.status_message = self._ensure_string_field(
                    content.status_message, "content.status_message", default=""
                )
            if content.external_id is not None:
                content_row.external_id = self._ensure_string_field(
                    content.external_id, "content.external_id", default=""
                )
            content_row.updated_at = int(time.time())
            self.contents_db.upsert_knowledge_content(knowledge_row=content_row)

            if self.vector_db and content.metadata:
                self.vector_db.update_metadata(content_id=content.id, metadata=content.metadata)

            return content_row.to_dict()

        else:
            if self.name:
                log_warning(f"Contents DB not found for knowledge base: {self.name}")
            else:
                log_warning("Contents DB not found for knowledge base")
            return None

    async def _aupdate_content(self, content: Content) -> Optional[Dict[str, Any]]:
        if self.contents_db:
            if not content.id:
                log_warning("Content id is required to update Knowledge content")
                return None

            # TODO: we shouldn't check for content here, we should trust the upsert method to handle conflicts
            if isinstance(self.contents_db, AsyncBaseDb):
                content_row = await self.contents_db.get_knowledge_content(content.id)
            else:
                content_row = self.contents_db.get_knowledge_content(content.id)
            if content_row is None:
                log_warning(f"Content row not found for id: {content.id}, cannot update status")
                return None

            if content.name is not None:
                content_row.name = content.name
            if content.description is not None:
                content_row.description = content.description
            if content.metadata is not None:
                content_row.metadata = content.metadata
            if content.status is not None:
                content_row.status = content.status
            if content.status_message is not None:
                content_row.status_message = content.status_message if content.status_message else ""
            if content.external_id is not None:
                content_row.external_id = content.external_id

            content_row.updated_at = int(time.time())
            if isinstance(self.contents_db, AsyncBaseDb):
                await self.contents_db.upsert_knowledge_content(knowledge_row=content_row)
            else:
                self.contents_db.upsert_knowledge_content(knowledge_row=content_row)

            if self.vector_db and content.metadata:
                self.vector_db.update_metadata(content_id=content.id, metadata=content.metadata)

            return content_row.to_dict()

        else:
            if self.name:
                log_warning(f"Contents DB not found for knowledge base: {self.name}")
            else:
                log_warning("Contents DB not found for knowledge base")
            return None

    async def _process_lightrag_content(self, content: Content, content_type: KnowledgeContentOrigin) -> None:
        from agno.vectordb import VectorDb

        self.vector_db = cast(VectorDb, self.vector_db)

        await self._add_to_contents_db(content)
        if content_type == KnowledgeContentOrigin.PATH:
            if content.file_data is None:
                log_warning("No file data provided")

            if content.path is None:
                log_error("No path provided for content")
                return

            path = Path(content.path)

            log_info(f"Uploading file to LightRAG from path: {path}")
            try:
                # Read the file content from path
                with open(path, "rb") as f:
                    file_content = f.read()

                # Get file type from extension or content.file_type
                file_type = content.file_type or path.suffix

                if self.vector_db and hasattr(self.vector_db, "insert_file_bytes"):
                    result = await self.vector_db.insert_file_bytes(
                        file_content=file_content,
                        filename=path.name,  # Use the original filename with extension
                        content_type=file_type,
                        send_metadata=True,  # Enable metadata so server knows the file type
                    )

                else:
                    log_error("Vector database does not support file insertion")
                    content.status = ContentStatus.FAILED
                    await self._aupdate_content(content)
                    return
                content.external_id = result
                content.status = ContentStatus.COMPLETED
                await self._aupdate_content(content)
                return

            except Exception as e:
                log_error(f"Error uploading file to LightRAG: {e}")
                content.status = ContentStatus.FAILED
                content.status_message = f"Could not upload to LightRAG: {str(e)}"
                await self._aupdate_content(content)
                return

        elif content_type == KnowledgeContentOrigin.URL:
            log_info(f"Uploading file to LightRAG from URL: {content.url}")
            try:
                reader = content.reader or self.website_reader
                if reader is None:
                    log_error("No URL reader available")
                    content.status = ContentStatus.FAILED
                    await self._aupdate_content(content)
                    return

                reader.chunk = False
                read_documents = reader.read(content.url, name=content.name)

                for read_document in read_documents:
                    read_document.content_id = content.id

                if not read_documents:
                    log_error("No documents read from URL")
                    content.status = ContentStatus.FAILED
                    await self._aupdate_content(content)
                    return

                if self.vector_db and hasattr(self.vector_db, "insert_text"):
                    result = await self.vector_db.insert_text(
                        file_source=content.url,
                        text=read_documents[0].content,
                    )
                else:
                    log_error("Vector database does not support text insertion")
                    content.status = ContentStatus.FAILED
                    await self._aupdate_content(content)
                    return

                content.external_id = result
                content.status = ContentStatus.COMPLETED
                await self._aupdate_content(content)
                return

            except Exception as e:
                log_error(f"Error uploading file to LightRAG: {e}")
                content.status = ContentStatus.FAILED
                content.status_message = f"Could not upload to LightRAG: {str(e)}"
                await self._aupdate_content(content)
                return

        elif content_type == KnowledgeContentOrigin.CONTENT:
            filename = (
                content.file_data.filename if content.file_data and content.file_data.filename else "uploaded_file"
            )
            log_info(f"Uploading file to LightRAG: {filename}")

            # Use the content from file_data
            if content.file_data and content.file_data.content:
                if self.vector_db and hasattr(self.vector_db, "insert_file_bytes"):
                    result = await self.vector_db.insert_file_bytes(
                        file_content=content.file_data.content,
                        filename=filename,
                        content_type=content.file_data.type,
                        send_metadata=True,  # Enable metadata so server knows the file type
                    )
                else:
                    log_error("Vector database does not support file insertion")
                    content.status = ContentStatus.FAILED
                    await self._aupdate_content(content)
                    return
                content.external_id = result
                content.status = ContentStatus.COMPLETED
                await self._aupdate_content(content)
            else:
                log_warning(f"No file data available for LightRAG upload: {content.name}")
            return

        elif content_type == KnowledgeContentOrigin.TOPIC:
            log_info(f"Uploading file to LightRAG: {content.name}")

            if content.reader is None:
                log_error("No reader available for topic content")
                content.status = ContentStatus.FAILED
                await self._aupdate_content(content)
                return

            if not content.topics:
                log_error("No topics available for content")
                content.status = ContentStatus.FAILED
                await self._aupdate_content(content)
                return

            read_documents = content.reader.read(content.topics)
            if len(read_documents) > 0:
                if self.vector_db and hasattr(self.vector_db, "insert_text"):
                    result = await self.vector_db.insert_text(
                        file_source=content.topics[0],
                        text=read_documents[0].content,
                    )
                else:
                    log_error("Vector database does not support text insertion")
                    content.status = ContentStatus.FAILED
                    await self._aupdate_content(content)
                    return
                content.external_id = result
                content.status = ContentStatus.COMPLETED
                await self._aupdate_content(content)
                return
            else:
                log_warning(f"No documents found for LightRAG upload: {content.name}")
                return

    def search(
        self,
        query: str,
        max_results: Optional[int] = None,
        filters: Optional[Union[Dict[str, Any], List[FilterExpr]]] = None,
        search_type: Optional[str] = None,
    ) -> List[Document]:
        """Returns relevant documents matching a query"""
        from agno.vectordb import VectorDb
        from agno.vectordb.search import SearchType

        self.vector_db = cast(VectorDb, self.vector_db)

        if (
            hasattr(self.vector_db, "search_type")
            and isinstance(self.vector_db.search_type, SearchType)
            and search_type
        ):
            self.vector_db.search_type = SearchType(search_type)
        try:
            if self.vector_db is None:
                log_warning("No vector db provided")
                return []

            _max_results = max_results or self.max_results
            log_debug(f"Getting {_max_results} relevant documents for query: {query}")
            return self.vector_db.search(query=query, limit=_max_results, filters=filters)
        except Exception as e:
            log_error(f"Error searching for documents: {e}")
            return []

    async def async_search(
        self,
        query: str,
        max_results: Optional[int] = None,
        filters: Optional[Union[Dict[str, Any], List[FilterExpr]]] = None,
        search_type: Optional[str] = None,
    ) -> List[Document]:
        """Returns relevant documents matching a query"""
        from agno.vectordb import VectorDb
        from agno.vectordb.search import SearchType

        self.vector_db = cast(VectorDb, self.vector_db)
        if (
            hasattr(self.vector_db, "search_type")
            and isinstance(self.vector_db.search_type, SearchType)
            and search_type
        ):
            self.vector_db.search_type = SearchType(search_type)
        try:
            if self.vector_db is None:
                log_warning("No vector db provided")
                return []

            _max_results = max_results or self.max_results
            log_debug(f"Getting {_max_results} relevant documents for query: {query}")
            try:
                return await self.vector_db.async_search(query=query, limit=_max_results, filters=filters)
            except NotImplementedError:
                log_info("Vector db does not support async search")
                return self.search(query=query, max_results=_max_results, filters=filters)
        except Exception as e:
            log_error(f"Error searching for documents: {e}")
            return []

    def get_valid_filters(self) -> Set[str]:
        if self.contents_db is None:
            log_warning("No contents db provided. This is required for filtering.")
            return set()
        contents, _ = self.get_content()
        valid_filters: Set[str] = set()
        for content in contents:
            if content.metadata:
                valid_filters.update(content.metadata.keys())

        return valid_filters

    async def async_get_valid_filters(self) -> Set[str]:
        if self.contents_db is None:
            log_warning("No contents db provided. This is required for filtering.")
            return set()
        contents, _ = await self.aget_content()
        valid_filters: Set[str] = set()
        for content in contents:
            if content.metadata:
                valid_filters.update(content.metadata.keys())

        return valid_filters

    def _validate_filters(
        self, filters: Union[Dict[str, Any], List[FilterExpr]], valid_metadata_filters: Set[str]
    ) -> Tuple[Union[Dict[str, Any], List[FilterExpr]], List[str]]:
        if not filters:
            return {}, []

        valid_filters: Union[Dict[str, Any], List[FilterExpr]] = {}
        invalid_keys = []

        if isinstance(filters, dict):
            # If no metadata filters tracked yet, all keys are considered invalid
            if valid_metadata_filters is None or not valid_metadata_filters:
                invalid_keys = list(filters.keys())
                log_warning(
                    f"No valid metadata filters tracked yet. All filter keys considered invalid: {invalid_keys}"
                )
                return {}, invalid_keys

            for key, value in filters.items():
                # Handle both normal keys and prefixed keys like meta_data.key
                base_key = key.split(".")[-1] if "." in key else key
                if base_key in valid_metadata_filters or key in valid_metadata_filters:
                    valid_filters[key] = value  # type: ignore
                else:
                    invalid_keys.append(key)
                    log_warning(f"Invalid filter key: {key} - not present in knowledge base")

        elif isinstance(filters, List):
            # Validate that list contains FilterExpr instances
            for i, filter_item in enumerate(filters):
                if not isinstance(filter_item, FilterExpr):
                    log_warning(
                        f"Invalid filter at index {i}: expected FilterExpr instance, "
                        f"got {type(filter_item).__name__}. "
                        f"Use filter expressions like EQ('key', 'value'), IN('key', [values]), "
                        f"AND(...), OR(...), NOT(...) from agno.filters"
                    )
            # Filter expressions are already validated, return empty dict/list
            # The actual filtering happens in the vector_db layer
            return filters, []

        return valid_filters, invalid_keys

    def validate_filters(
        self, filters: Union[Dict[str, Any], List[FilterExpr]]
    ) -> Tuple[Union[Dict[str, Any], List[FilterExpr]], List[str]]:
        valid_filters_from_db = self.get_valid_filters()

        valid_filters, invalid_keys = self._validate_filters(filters, valid_filters_from_db)

        return valid_filters, invalid_keys

    async def async_validate_filters(
        self, filters: Union[Dict[str, Any], List[FilterExpr]]
    ) -> Tuple[Union[Dict[str, Any], List[FilterExpr]], List[str]]:
        """Return a tuple containing a dict with all valid filters and a list of invalid filter keys"""
        valid_filters_from_db = await self.async_get_valid_filters()

        valid_filters, invalid_keys = self._validate_filters(filters, valid_filters_from_db)

        return valid_filters, invalid_keys

    def remove_vector_by_id(self, id: str) -> bool:
        from agno.vectordb import VectorDb

        self.vector_db = cast(VectorDb, self.vector_db)
        if self.vector_db is None:
            log_warning("No vector DB provided")
            return False
        return self.vector_db.delete_by_id(id)

    def remove_vectors_by_name(self, name: str) -> bool:
        from agno.vectordb import VectorDb

        self.vector_db = cast(VectorDb, self.vector_db)
        if self.vector_db is None:
            log_warning("No vector DB provided")
            return False
        return self.vector_db.delete_by_name(name)

    def remove_vectors_by_metadata(self, metadata: Dict[str, Any]) -> bool:
        from agno.vectordb import VectorDb

        self.vector_db = cast(VectorDb, self.vector_db)
        if self.vector_db is None:
            log_warning("No vector DB provided")
            return False
        return self.vector_db.delete_by_metadata(metadata)

    # --- API Only Methods ---

    def patch_content(self, content: Content) -> Optional[Dict[str, Any]]:
        return self._update_content(content)

    async def apatch_content(self, content: Content) -> Optional[Dict[str, Any]]:
        return await self._aupdate_content(content)

    def get_content_by_id(self, content_id: str) -> Optional[Content]:
        if self.contents_db is None:
            raise ValueError("No contents db provided")

        if isinstance(self.contents_db, AsyncBaseDb):
            raise ValueError(
                "get_content_by_id() is not supported for async databases. Please use aget_content_by_id() instead."
            )

        content_row = self.contents_db.get_knowledge_content(content_id)

        if content_row is None:
            return None
        content = Content(
            id=content_row.id,
            name=content_row.name,
            description=content_row.description,
            metadata=content_row.metadata,
            file_type=content_row.type,
            size=content_row.size,
            status=ContentStatus(content_row.status) if content_row.status else None,
            status_message=content_row.status_message,
            created_at=content_row.created_at,
            updated_at=content_row.updated_at if content_row.updated_at else content_row.created_at,
            external_id=content_row.external_id,
        )
        return content

    async def aget_content_by_id(self, content_id: str) -> Optional[Content]:
        if self.contents_db is None:
            raise ValueError("No contents db provided")

        if isinstance(self.contents_db, AsyncBaseDb):
            content_row = await self.contents_db.get_knowledge_content(content_id)
        else:
            content_row = self.contents_db.get_knowledge_content(content_id)

        if content_row is None:
            return None
        content = Content(
            id=content_row.id,
            name=content_row.name,
            description=content_row.description,
            metadata=content_row.metadata,
            file_type=content_row.type,
            size=content_row.size,
            status=ContentStatus(content_row.status) if content_row.status else None,
            status_message=content_row.status_message,
            created_at=content_row.created_at,
            updated_at=content_row.updated_at if content_row.updated_at else content_row.created_at,
            external_id=content_row.external_id,
        )
        return content

    def get_content(
        self,
        limit: Optional[int] = None,
        page: Optional[int] = None,
        sort_by: Optional[str] = None,
        sort_order: Optional[str] = None,
    ) -> Tuple[List[Content], int]:
        if self.contents_db is None:
            raise ValueError("No contents db provided")

        if isinstance(self.contents_db, AsyncBaseDb):
            raise ValueError("get_content() is not supported for async databases. Please use aget_content() instead.")

        contents, count = self.contents_db.get_knowledge_contents(
            limit=limit, page=page, sort_by=sort_by, sort_order=sort_order
        )

        result = []
        for content_row in contents:
            # Create Content from database row
            content = Content(
                id=content_row.id,
                name=content_row.name,
                description=content_row.description,
                metadata=content_row.metadata,
                size=content_row.size,
                file_type=content_row.type,
                status=ContentStatus(content_row.status) if content_row.status else None,
                status_message=content_row.status_message,
                created_at=content_row.created_at,
                updated_at=content_row.updated_at if content_row.updated_at else content_row.created_at,
                external_id=content_row.external_id,
            )
            result.append(content)
        return result, count

    async def aget_content(
        self,
        limit: Optional[int] = None,
        page: Optional[int] = None,
        sort_by: Optional[str] = None,
        sort_order: Optional[str] = None,
    ) -> Tuple[List[Content], int]:
        if self.contents_db is None:
            raise ValueError("No contents db provided")

        if isinstance(self.contents_db, AsyncBaseDb):
            contents, count = await self.contents_db.get_knowledge_contents(
                limit=limit, page=page, sort_by=sort_by, sort_order=sort_order
            )
        else:
            contents, count = self.contents_db.get_knowledge_contents(
                limit=limit, page=page, sort_by=sort_by, sort_order=sort_order
            )

        result = []
        for content_row in contents:
            # Create Content from database row
            content = Content(
                id=content_row.id,
                name=content_row.name,
                description=content_row.description,
                metadata=content_row.metadata,
                size=content_row.size,
                file_type=content_row.type,
                status=ContentStatus(content_row.status) if content_row.status else None,
                status_message=content_row.status_message,
                created_at=content_row.created_at,
                updated_at=content_row.updated_at if content_row.updated_at else content_row.created_at,
                external_id=content_row.external_id,
            )
            result.append(content)
        return result, count

    def get_content_status(self, content_id: str) -> Tuple[Optional[ContentStatus], Optional[str]]:
        if self.contents_db is None:
            raise ValueError("No contents db provided")

        if isinstance(self.contents_db, AsyncBaseDb):
            raise ValueError(
                "get_content_status() is not supported for async databases. Please use aget_content_status() instead."
            )

        content_row = self.contents_db.get_knowledge_content(content_id)
        if content_row is None:
            return None, "Content not found"

        # Convert string status to enum, defaulting to PROCESSING if unknown
        status_str = content_row.status
        try:
            status = ContentStatus(status_str.lower()) if status_str else ContentStatus.PROCESSING
        except ValueError:
            # Handle legacy or unknown statuses
            if status_str and "failed" in status_str.lower():
                status = ContentStatus.FAILED
            elif status_str and "completed" in status_str.lower():
                status = ContentStatus.COMPLETED
            else:
                status = ContentStatus.PROCESSING

        return status, content_row.status_message

    async def aget_content_status(self, content_id: str) -> Tuple[Optional[ContentStatus], Optional[str]]:
        if self.contents_db is None:
            raise ValueError("No contents db provided")

        if isinstance(self.contents_db, AsyncBaseDb):
            content_row = await self.contents_db.get_knowledge_content(content_id)
        else:
            content_row = self.contents_db.get_knowledge_content(content_id)

        if content_row is None:
            return None, "Content not found"

        # Convert string status to enum, defaulting to PROCESSING if unknown
        status_str = content_row.status
        try:
            status = ContentStatus(status_str.lower()) if status_str else ContentStatus.PROCESSING
        except ValueError:
            # Handle legacy or unknown statuses
            if status_str and "failed" in status_str.lower():
                status = ContentStatus.FAILED
            elif status_str and "completed" in status_str.lower():
                status = ContentStatus.COMPLETED
            else:
                status = ContentStatus.PROCESSING

        return status, content_row.status_message

    def remove_content_by_id(self, content_id: str):
        from agno.vectordb import VectorDb

        self.vector_db = cast(VectorDb, self.vector_db)
        if self.vector_db is not None:
            if self.vector_db.__class__.__name__ == "LightRag":
                # For LightRAG, get the content first to find the external_id
                content = self.get_content_by_id(content_id)
                if content and content.external_id:
                    self.vector_db.delete_by_external_id(content.external_id)  # type: ignore
                else:
                    log_warning(f"No external_id found for content {content_id}, cannot delete from LightRAG")
            else:
                self.vector_db.delete_by_content_id(content_id)

        if self.contents_db is not None:
            self.contents_db.delete_knowledge_content(content_id)

    async def aremove_content_by_id(self, content_id: str):
        if self.vector_db is not None:
            if self.vector_db.__class__.__name__ == "LightRag":
                # For LightRAG, get the content first to find the external_id
                content = await self.aget_content_by_id(content_id)
                if content and content.external_id:
                    self.vector_db.delete_by_external_id(content.external_id)  # type: ignore
                else:
                    log_warning(f"No external_id found for content {content_id}, cannot delete from LightRAG")
            else:
                self.vector_db.delete_by_content_id(content_id)

        if self.contents_db is not None:
            if isinstance(self.contents_db, AsyncBaseDb):
                await self.contents_db.delete_knowledge_content(content_id)
            else:
                self.contents_db.delete_knowledge_content(content_id)

    def remove_all_content(self):
        contents, _ = self.get_content()
        for content in contents:
            if content.id is not None:
                self.remove_content_by_id(content.id)

    async def aremove_all_content(self):
        contents, _ = await self.aget_content()
        for content in contents:
            if content.id is not None:
                await self.aremove_content_by_id(content.id)

    # --- Reader Factory Integration ---

    def construct_readers(self):
        """Initialize readers dictionary for lazy loading."""
        # Initialize empty readers dict - readers will be created on-demand
        if self.readers is None:
            self.readers = {}

    def add_reader(self, reader: Reader):
        """Add a custom reader to the knowledge base."""
        if self.readers is None:
            self.readers = {}

        # Generate a key for the reader
        reader_key = self._generate_reader_key(reader)
        self.readers[reader_key] = reader
        return reader

    def get_readers(self) -> Dict[str, Reader]:
        """Get all currently loaded readers (only returns readers that have been used)."""
        if self.readers is None:
            self.readers = {}

        return self.readers

    def _generate_reader_key(self, reader: Reader) -> str:
        """Generate a key for a reader instance."""
        if reader.name:
            return f"{reader.name.lower().replace(' ', '_')}"
        else:
            return f"{reader.__class__.__name__.lower().replace(' ', '_')}"

    def _select_reader(self, extension: str) -> Reader:
        """Select the appropriate reader for a file extension."""
        log_info(f"Selecting reader for extension: {extension}")
        return ReaderFactory.get_reader_for_extension(extension)

    # --- Convenience Properties for Backward Compatibility ---

    def _is_text_mime_type(self, mime_type: str) -> bool:
        """
        Check if a MIME type represents text content that can be safely encoded as UTF-8.

        Args:
            mime_type: The MIME type to check

        Returns:
            bool: True if it's a text type, False if binary
        """
        if not mime_type:
            return False

        text_types = [
            "text/",
            "application/json",
            "application/xml",
            "application/javascript",
            "application/csv",
            "application/sql",
        ]

        return any(mime_type.startswith(t) for t in text_types)

    def _should_include_file(self, file_path: str, include: Optional[List[str]], exclude: Optional[List[str]]) -> bool:
        """
        Determine if a file should be included based on include/exclude patterns.

        Logic:
        1. If include is specified, file must match at least one include pattern
        2. If exclude is specified, file must not match any exclude pattern
        3. If neither specified, include all files

        Args:
            file_path: Path to the file to check
            include: Optional list of include patterns (glob-style)
            exclude: Optional list of exclude patterns (glob-style)

        Returns:
            bool: True if file should be included, False otherwise
        """
        import fnmatch

        # If include patterns specified, file must match at least one
        if include:
            if not any(fnmatch.fnmatch(file_path, pattern) for pattern in include):
                return False

        # If exclude patterns specified, file must not match any
        if exclude:
            if any(fnmatch.fnmatch(file_path, pattern) for pattern in exclude):
                return False

        return True

    def _get_reader(self, reader_type: str) -> Optional[Reader]:
        """Get a cached reader or create it if not cached, handling missing dependencies gracefully."""
        if self.readers is None:
            self.readers = {}

        if reader_type not in self.readers:
            try:
                reader = ReaderFactory.create_reader(reader_type)
                if reader:
                    self.readers[reader_type] = reader
                else:
                    return None

            except Exception as e:
                log_warning(f"Cannot create {reader_type} reader {e}")
                return None

        return self.readers.get(reader_type)

    @property
    def pdf_reader(self) -> Optional[Reader]:
        """PDF reader - lazy loaded via factory."""
        return self._get_reader("pdf")

    @property
    def csv_reader(self) -> Optional[Reader]:
        """CSV reader - lazy loaded via factory."""
        return self._get_reader("csv")

    @property
    def docx_reader(self) -> Optional[Reader]:
        """Docx reader - lazy loaded via factory."""
        return self._get_reader("docx")

    @property
    def pptx_reader(self) -> Optional[Reader]:
        """PPTX reader - lazy loaded via factory."""
        return self._get_reader("pptx")

    @property
    def json_reader(self) -> Optional[Reader]:
        """JSON reader - lazy loaded via factory."""
        return self._get_reader("json")

    @property
    def markdown_reader(self) -> Optional[Reader]:
        """Markdown reader - lazy loaded via factory."""
        return self._get_reader("markdown")

    @property
    def text_reader(self) -> Optional[Reader]:
        """Text reader - lazy loaded via factory."""
        return self._get_reader("text")

    @property
    def website_reader(self) -> Optional[Reader]:
        """Website reader - lazy loaded via factory."""
        return self._get_reader("website")

    @property
    def firecrawl_reader(self) -> Optional[Reader]:
        """Firecrawl reader - lazy loaded via factory."""
        return self._get_reader("firecrawl")

    @property
    def youtube_reader(self) -> Optional[Reader]:
        """YouTube reader - lazy loaded via factory."""
        return self._get_reader("youtube")
