"""Tests for the loader module.

This module contains comprehensive test cases for document and content loading utilities,
including web crawling, file processing, and content chunking functionality.
"""

import os
import tempfile
from collections.abc import Generator
from typing import NoReturn
from unittest.mock import Mock, PropertyMock, mock_open, patch

import requests

from arklex.utils.loader import (
    CrawledObject,
    DocObject,
    Loader,
    SourceType,
    encode_image,
)


class TestEncodeImage:
    """Test cases for encode_image function."""

    def test_encode_image_success(self) -> None:
        """Test successful image encoding."""
        with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp_file:
            tmp_file.write(b"fake image data")
            tmp_file_path = tmp_file.name

        try:
            with patch("builtins.open", create=True) as mock_open:
                mock_open.return_value.__enter__.return_value.read.return_value = (
                    b"fake image data"
                )
                with patch("base64.b64encode") as mock_b64encode:
                    mock_b64encode.return_value.decode.return_value = (
                        "ZmFrZSBpbWFnZSBkYXRh"
                    )
                    result = encode_image(tmp_file_path)
                    assert result == "ZmFrZSBpbWFnZSBkYXRh"
        finally:
            os.unlink(tmp_file_path)

    def test_encode_image_file_not_found(self) -> None:
        """Test image encoding with non-existent file."""
        result = encode_image("nonexistent.png")
        assert result is None

    def test_encode_image_general_exception(self) -> None:
        """Test image encoding with general exception."""
        with patch("builtins.open", side_effect=Exception("Permission denied")):
            result = encode_image("test.png")
            assert result is None

    def test_encode_image_nonexistent(self) -> None:
        """Test image encoding with None path."""
        result = encode_image(None)
        assert result is None

    def test_encode_image_with_directory(self) -> None:
        """Test image encoding with directory path."""
        with tempfile.TemporaryDirectory() as temp_dir:
            result = encode_image(temp_dir)
            assert result is None

    def test_encode_image_with_non_image_file(self) -> None:
        """Test image encoding with non-image file."""
        with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp_file:
            tmp_file.write(b"text content")
            tmp_file_path = tmp_file.name

        try:
            with patch("builtins.open", create=True) as mock_open:
                mock_open.return_value.__enter__.return_value.read.return_value = (
                    b"text content"
                )
                with patch("base64.b64encode") as mock_b64encode:
                    mock_b64encode.return_value.decode.return_value = "dGV4dCBjb250ZW50"
                    result = encode_image(tmp_file_path)
                    assert result == "dGV4dCBjb250ZW50"
        finally:
            os.unlink(tmp_file_path)


class TestSourceType:
    """Test cases for SourceType enum."""

    def test_source_type_values(self) -> None:
        """Test SourceType enum values."""
        assert SourceType.WEB.value == 1
        assert SourceType.FILE.value == 2
        assert SourceType.TEXT.value == 3

    def test_source_type_unknown(self) -> None:
        """Test SourceType with unknown value."""
        # Test that we can access the enum values
        assert hasattr(SourceType, "WEB")
        assert hasattr(SourceType, "FILE")
        assert hasattr(SourceType, "TEXT")


class TestDocObject:
    """Test cases for DocObject class."""

    def test_doc_object_initialization(self) -> None:
        """Test DocObject initialization."""
        doc = DocObject("test_id", "test_source")
        assert doc.id == "test_id"
        assert doc.source == "test_source"

    def test_docobject_missing_fields(self) -> None:
        """Test DocObject with missing fields."""
        doc = DocObject("", "")
        assert doc.id == ""
        assert doc.source == ""


class TestCrawledObject:
    """Test cases for CrawledObject class."""

    def test_crawled_object_initialization(self) -> None:
        """Test CrawledObject initialization."""
        crawled = CrawledObject(
            "test_id", "test_source", "test_content", {"key": "value"}
        )
        assert crawled.id == "test_id"
        assert crawled.source == "test_source"
        assert crawled.content == "test_content"
        assert crawled.metadata == {"key": "value"}

    def test_crawled_object_default_values(self) -> None:
        """Test CrawledObject with default values."""
        crawled = CrawledObject("test_id", "test_source", "test_content")
        assert crawled.metadata == {}
        assert crawled.is_chunk is False
        assert crawled.is_error is False
        assert crawled.error_message is None
        assert crawled.source_type == SourceType.WEB

    def test_crawled_object_to_dict(self) -> None:
        """Test CrawledObject to_dict method."""
        crawled = CrawledObject(
            "test_id", "test_source", "test_content", {"key": "value"}
        )
        result = crawled.to_dict()
        assert result["id"] == "test_id"
        assert result["source"] == "test_source"
        assert result["content"] == "test_content"
        assert result["metadata"] == {"key": "value"}

    def test_crawled_object_from_dict(self) -> None:
        """Test CrawledObject from_dict method."""
        data = {
            "id": "test_id",
            "source": "test_source",
            "content": "test_content",
            "metadata": {"key": "value"},
            "is_chunk": False,
            "is_error": False,
            "error_message": None,
            "source_type": SourceType.WEB,
        }
        crawled = CrawledObject.from_dict(data)
        assert crawled.id == "test_id"
        assert crawled.source == "test_source"
        assert crawled.content == "test_content"
        assert crawled.metadata == {"key": "value"}

    def test_crawled_object_from_dict_with_all_fields(self) -> None:
        """Test CrawledObject.from_dict with all fields populated."""
        data = {
            "id": "test_id",
            "source": "test_source",
            "content": "test_content",
            "metadata": {"key": "value"},
            "is_chunk": True,
            "is_error": True,
            "error_message": "test error",
            "source_type": SourceType.FILE,
        }

        result = CrawledObject.from_dict(data)
        assert result.id == "test_id"
        assert result.source == "test_source"
        assert result.content == "test_content"
        assert result.metadata == {"key": "value"}
        assert result.is_chunk is True
        assert result.is_error is True
        assert result.error_message == "test error"
        assert result.source_type == SourceType.FILE


class TestLoader:
    """Test cases for Loader class."""

    def test_loader_initialization(self) -> None:
        """Test Loader initialization."""
        loader = Loader()
        assert loader is not None

    def test_to_crawled_url_objs(self) -> None:
        """Test converting URL list to CrawledObject list."""
        loader = Loader()
        urls = ["http://example.com", "http://test.com"]
        result = loader.to_crawled_url_objs(urls)
        assert len(result) == 2
        assert result[0].source == "http://example.com"
        assert result[1].source == "http://test.com"

    def test_crawl_urls_selenium_success(self) -> None:
        """Test successful Selenium crawling."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        with patch("selenium.webdriver.Chrome") as mock_driver:
            mock_driver_instance = Mock()
            mock_driver.return_value = mock_driver_instance
            mock_driver_instance.page_source = (
                "<html><title>Test</title><body>Content</body></html>"
            )
            mock_driver_instance.quit.return_value = None

            with patch("time.sleep"):
                result = loader.crawl_urls(url_objects)
                assert len(result) == 1
                assert result[0].source == "http://example.com"

    def test_crawl_urls_selenium_failure_requests_success(self) -> None:
        """Test Selenium failure with requests fallback success."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        # Mock Selenium failure
        with (
            patch(
                "selenium.webdriver.Chrome", side_effect=Exception("Selenium failed")
            ),
            patch("requests.get") as mock_get,
        ):
            mock_response = Mock()
            mock_response.text = '<html><body><a href="http://example.com/page1">Link 1</a></body></html>'
            mock_get.return_value = mock_response

            result = loader.crawl_urls(url_objects)
            assert len(result) == 1
            # The implementation includes both link text and URL
            assert result[0].content == "Link 1 http://example.com/page1"

        # Mock both Selenium and requests failure
        with (
            patch(
                "selenium.webdriver.Chrome", side_effect=Exception("Selenium failed")
            ),
            patch("requests.get", side_effect=Exception("Requests failed")),
        ):
            result = loader.crawl_urls(url_objects)
            assert len(result) == 1
            # When both fail, the implementation creates mock content instead of error
            assert not result[0].is_error
            assert "Welcome to example.com" in result[0].content

    def test_crawl_urls_all_failure_mock_content(self) -> None:
        """Test all crawling methods failure with mock content fallback."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        # Mock both Selenium and requests failure
        with (
            patch(
                "selenium.webdriver.Chrome", side_effect=Exception("Selenium failed")
            ),
            patch("requests.get", side_effect=Exception("Requests failed")),
        ):
            result = loader.crawl_urls(url_objects)
            assert len(result) == 1
            assert result[0].source == "http://example.com"
            assert "mock_content" in result[0].metadata

    def test_crawl_with_selenium_timeout(self) -> None:
        """Test _crawl_with_selenium with timeout."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        # Mock Selenium to raise timeout exception
        with (
            patch("selenium.webdriver.Chrome", side_effect=Exception("timeout")),
            patch("time.sleep"),
        ):
            result = loader._crawl_with_selenium(url_objects)
            assert len(result) == 1
            assert result[0].is_error

    def test_crawl_with_selenium_retry_success(self) -> None:
        """Test Selenium crawling with retry success."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        with patch("selenium.webdriver.Chrome") as mock_driver:
            mock_driver_instance = Mock()
            mock_driver.return_value = mock_driver_instance
            mock_driver_instance.page_source = (
                "<html><title>Test</title><body>Content</body></html>"
            )
            mock_driver_instance.quit.return_value = None

            # First call fails, second succeeds
            mock_driver_instance.page_source = (
                "<html><title>Test</title><body>Content</body></html>"
            )

            with patch("time.sleep"):
                result = loader._crawl_with_selenium(url_objects)
                assert len(result) == 1

    def test_crawl_with_selenium_expected_error_filtering(self) -> None:
        """Test expected error filtering logic in selenium crawling."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]
        # Patch webdriver.Chrome to raise an expected error
        mock_driver = Mock()
        mock_driver.get.side_effect = Exception("timeout")
        with (
            patch("selenium.webdriver.Chrome", return_value=mock_driver),
            patch(
                "webdriver_manager.chrome.ChromeDriverManager.install",
                return_value="/tmp/chromedriver",
            ),
            patch("selenium.webdriver.chrome.service.Service"),
        ):
            result = loader._crawl_with_selenium(url_objects)
            assert len(result) == 1
            assert result[0].is_error
            assert "timeout" in result[0].error_message

    def test_crawl_with_requests_http_error(self) -> None:
        """Test requests crawling with HTTP error."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        with patch("requests.get") as mock_get:
            mock_get.side_effect = requests.HTTPError("404 Not Found")
            result = loader._crawl_with_requests(url_objects)
            assert len(result) == 1
            assert result[0].is_error

    def test_create_mock_content_from_urls_various_urls(self) -> None:
        """Test creating mock content from various URL types."""
        loader = Loader()
        url_objects = [
            DocObject("1", "http://example.com/company"),
            DocObject("2", "http://example.com/contact"),
            DocObject("3", "http://example.com/privacy"),
            DocObject("4", "http://example.com/terms"),
            DocObject("5", "http://example.com/resources"),
            DocObject("6", "http://example.com/solutions"),
            DocObject("7", "http://example.com/clouffee"),
            DocObject("8", "http://example.com/headquarters"),
            DocObject("9", "http://example.com/award"),
            DocObject("10", "http://example.com/cleaning"),
            DocObject("11", "http://example.com/delivery"),
            DocObject("12", "http://example.com/production"),
            DocObject("13", "http://example.com/"),
        ]

        result = loader._create_mock_content_from_urls(url_objects)
        assert len(result) == 13
        assert all("mock_content" in doc.metadata for doc in result)

    def test_get_all_urls_timeout(self) -> None:
        """Test URL discovery timeout handling in get_all_urls."""
        loader = Loader()
        # Patch time.time to simulate timeout
        with patch("time.time", side_effect=[0, 100, 200, 300, 400, 1000]):
            result = loader.get_all_urls("http://example.com", 1)
            assert isinstance(result, list)

    def test_get_all_urls_exception_handling(self) -> None:
        """Test get_all_urls with exception handling."""
        loader = Loader()

        with patch.object(
            loader, "get_outsource_urls", side_effect=Exception("Network error")
        ):
            result = loader.get_all_urls("http://example.com", 10)
            assert len(result) == 1  # Should include the base URL

    def test_check_url_edge_cases(self) -> None:
        """Test URL validation logic in _check_url for various edge cases."""
        loader = Loader()
        # Valid URL
        assert loader._check_url("http://example.com/page", "http://example.com")
        # Invalid: not base
        assert not loader._check_url("http://other.com/page", "http://example.com")
        # Invalid: file extension
        assert not loader._check_url(
            "http://example.com/file.pdf", "http://example.com"
        )
        # Invalid: same as base
        assert not loader._check_url("http://example.com", "http://example.com")
        # Invalid: empty
        assert not loader._check_url("", "http://example.com")

    def test_get_candidates_websites_edge_cases(self) -> None:
        """Test get_candidates_websites with edge cases."""
        loader = Loader()

        # Test with empty list
        result = loader.get_candidates_websites([], 5)
        assert len(result) == 0

        # Test with fewer URLs than top_k
        urls = [
            CrawledObject("1", "http://example1.com", "content1"),
            CrawledObject("2", "http://example2.com", "content2"),
        ]
        result = loader.get_candidates_websites(urls, 5)
        assert len(result) == 2

    def test_crawl_file_with_mistral_api(self) -> None:
        """Test crawl_file with Mistral API."""
        loader = Loader()
        doc_obj = DocObject("1", "test.txt")

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".txt", delete=False
        ) as tmp_file:
            tmp_file.write("Test content")
            tmp_file_path = tmp_file.name

        try:
            with patch("mistralai.Mistral") as mock_mistral:
                mock_instance = mock_mistral.return_value
                mock_instance.generate.return_value = "API response"
                result = loader.crawl_file(doc_obj)
                assert isinstance(result, CrawledObject)
        finally:
            os.remove(tmp_file_path)

    def test_chunk_with_empty_docs(self) -> None:
        """Test chunk method with empty documents."""
        result = Loader.chunk([])
        assert len(result) == 0

    def test_chunk_with_exception(self) -> None:
        """Test chunk method with exception."""
        docs = [CrawledObject("1", "test.txt", "content")]

        with patch(
            "langchain.text_splitter.RecursiveCharacterTextSplitter",
            side_effect=Exception("Splitter error"),
        ):
            result = Loader.chunk(docs)
            assert len(result) == 1
            # The result is a Document object, not CrawledObject, so we check differently
            assert hasattr(result[0], "page_content")

    def test_loader_crawl_urls_empty(self) -> None:
        """Test crawling empty URL list."""
        loader = Loader()
        result = loader.crawl_urls([])
        assert len(result) == 0

    def test_loader_chunk_empty_docs(self) -> None:
        """Test chunking empty documents."""
        result = Loader.chunk([])
        assert len(result) == 0

    def test_source_type_unknown(self) -> None:
        """Test SourceType with unknown value."""
        # Test that we can access the enum values
        assert hasattr(SourceType, "WEB")
        assert hasattr(SourceType, "FILE")
        assert hasattr(SourceType, "TEXT")

    def test_crawl_with_selenium_chromedriver_installation_failure(self) -> None:
        """Test ChromeDriver installation failure handling."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        # Patch ChromeDriverManager().install to raise an exception
        with patch(
            "webdriver_manager.chrome.ChromeDriverManager.install",
            side_effect=Exception("install fail"),
        ):
            result = loader._crawl_with_selenium(url_objects)
            assert len(result) == 1
            assert result[0].is_error
            assert "ChromeDriver installation failed" in result[0].error_message

    def test_crawl_with_selenium_driver_quit_exception(self) -> None:
        """Test Selenium crawling with driver quit exception."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        mock_driver = Mock()
        mock_driver.get.side_effect = Exception("page load fail")
        mock_driver.quit.side_effect = Exception("quit fail")
        with (
            patch("selenium.webdriver.Chrome", return_value=mock_driver),
            patch(
                "webdriver_manager.chrome.ChromeDriverManager.install",
                return_value="/tmp/chromedriver",
            ),
            patch("selenium.webdriver.chrome.service.Service"),
        ):
            result = loader._crawl_with_selenium(url_objects)
            assert len(result) == 1
            assert result[0].is_error

    def test_crawl_with_selenium_webdriver_exception(self) -> None:
        """Test Selenium crawling with webdriver exception."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        with (
            patch(
                "selenium.webdriver.Chrome",
                side_effect=Exception("chrome not reachable"),
            ),
            patch("time.sleep"),
        ):
            result = loader._crawl_with_selenium(url_objects)
            assert len(result) == 1
            assert result[0].is_error

    def test_crawl_with_requests_timeout(self) -> None:
        """Test requests timeout handling in _crawl_with_requests."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]
        with patch("requests.get", side_effect=requests.Timeout("timeout")):
            result = loader._crawl_with_requests(url_objects)
            assert len(result) == 1
            assert result[0].is_error
            assert "timeout" in result[0].error_message

    def test_crawl_file_permission_error(self) -> None:
        """Test crawl_file with permission error."""
        loader = Loader()
        doc_obj = DocObject("1", "/root/protected.txt")

        with patch("builtins.open", side_effect=PermissionError("Permission denied")):
            result = loader.crawl_file(doc_obj)
            assert result.is_error
            assert "Error loading" in result.error_message

    def test_crawl_file_unicode_error(self) -> None:
        """Test crawl_file with Unicode decode error."""
        loader = Loader()
        doc_obj = DocObject("1", "test.txt")

        with patch(
            "builtins.open",
            side_effect=UnicodeDecodeError("utf-8", b"", 0, 1, "invalid utf-8"),
        ):
            result = loader.crawl_file(doc_obj)
            assert result.is_error
            assert "Error loading" in result.error_message

    def test_crawl_file_general_exception(self) -> None:
        """Test crawl_file with general exception."""
        loader = Loader()
        doc_obj = DocObject("1", "test.txt")

        with patch("builtins.open", side_effect=Exception("General error")):
            result = loader.crawl_file(doc_obj)
            assert result.is_error
            assert "Error loading" in result.error_message

    def test_crawl_file_with_mistral_api_error(self) -> None:
        """Test crawl_file with Mistral API error."""
        loader = Loader()
        doc_obj = DocObject("1", "test.txt")

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".txt", delete=False
        ) as tmp_file:
            tmp_file.write("Test content")
            tmp_file_path = tmp_file.name

        try:
            with patch("mistralai.Mistral", side_effect=Exception("API error")):
                result = loader.crawl_file(doc_obj)
                assert result.is_error
        finally:
            os.remove(tmp_file_path)

    def test_save_method(self) -> None:
        """Test save method."""
        docs = [CrawledObject("1", "test.txt", "content")]

        with tempfile.NamedTemporaryFile(
            mode="w", suffix=".json", delete=False
        ) as tmp_file:
            tmp_file_path = tmp_file.name

        try:
            Loader.save(tmp_file_path, docs)
            assert os.path.exists(tmp_file_path)
        finally:
            os.unlink(tmp_file_path)

    def test_chunk_method_with_exception(self) -> None:
        """Test chunk method with exception."""
        docs = [CrawledObject("1", "test.txt", "content")]

        with patch(
            "langchain.text_splitter.RecursiveCharacterTextSplitter",
            side_effect=Exception("Splitter error"),
        ):
            result = Loader.chunk(docs)
            assert len(result) == 1
            # The result is a Document object, not CrawledObject, so we check differently
            assert hasattr(result[0], "page_content")

    def test_get_outsource_urls_processing_exception(self) -> None:
        """Test URL processing exceptions in get_outsource_urls."""
        loader = Loader()

        # Patch requests.get to return a response with a link that raises exception
        class DummyLink:
            def get(self, _: str) -> NoReturn:
                raise Exception("fail href")

        class DummySoup:
            def find_all(self, _: str) -> list[DummyLink]:
                return [DummyLink()]

        class DummyResponse:
            status_code = 200
            text = "<html></html>"

        with (
            patch("requests.get", return_value=DummyResponse()),
            patch("bs4.BeautifulSoup", return_value=DummySoup()),
        ):
            # Should log error and continue
            result = loader.get_outsource_urls(
                "http://example.com", "http://example.com"
            )
            assert isinstance(result, list)

    def test_get_candidates_websites_graph(self) -> None:
        """Test graph operations in get_candidates_websites."""
        loader = Loader()
        # Create two crawled objects with content referencing each other
        c1 = CrawledObject(
            "id1",
            "http://a.com",
            "http://b.com",
            {},
            False,
            False,
            None,
            SourceType.WEB,
        )
        c2 = CrawledObject(
            "id2",
            "http://b.com",
            "http://a.com",
            {},
            False,
            False,
            None,
            SourceType.WEB,
        )
        result = loader.get_candidates_websites([c1, c2], top_k=2)
        assert isinstance(result, list)
        assert all(isinstance(x, CrawledObject) for x in result)

    def test_crawl_file_unsupported_file_type(self) -> None:
        """Test crawl_file with unsupported file type."""
        loader = Loader()
        # Create a DocObject with unsupported file type
        local_obj = DocObject("1", "test.unsupported")

        # Create a mock Path object with the required properties
        mock_path = Mock()
        mock_path.suffix = ".unsupported"
        mock_path.name = "test.unsupported"

        with patch("arklex.utils.loader.Path", return_value=mock_path):
            result = loader.crawl_file(local_obj)
            # Should return CrawledObject with error for unsupported file type
            assert isinstance(result, CrawledObject)
            assert result.is_error is True
            assert "Unsupported file type" in result.error_message

    def test_crawl_file_with_missing_file_type(self) -> None:
        """Test crawl_file with missing file type (covers lines 833, 835, 839, 841)."""
        loader = Loader()

        # Create a DocObject with no file type
        local_obj = DocObject("1", "test")

        # Create a mock Path object instead of trying to patch real Path properties
        mock_path = Mock()
        mock_path.suffix = ""
        mock_path.name = "test"

        with patch("arklex.utils.loader.Path", return_value=mock_path):
            result = loader.crawl_file(local_obj)
            # Should return CrawledObject with error for missing file type
            assert isinstance(result, CrawledObject)
            assert result.is_error is True
            assert "No file type detected" in result.error_message

    def test_crawl_file_with_unsupported_file_type(self) -> None:
        loader = Loader()

        with patch("arklex.utils.loader.Path") as mock_path:
            # Mock the Path class itself to have _flavour attribute
            mock_path._flavour = Mock()
            mock_path_instance = Mock()
            mock_path_instance.suffix = ".xyz"  # Unsupported file type
            mock_path_instance.name = "testfile.xyz"
            # Add the _flavour attribute that Path objects have
            mock_path_instance._flavour = Mock()
            mock_path.return_value = mock_path_instance

            with patch("arklex.utils.loader.MISTRAL_API_KEY", None):
                result = loader.crawl_file(DocObject("test_id", "testfile.xyz"))

                assert result.is_error is True
                assert "Unsupported file type" in result.error_message

    def test_crawl_file_with_mistral_api_key_not_set(self) -> None:
        loader = Loader()

        with patch("arklex.utils.loader.Path") as mock_path:
            # Mock the Path class itself to have _flavour attribute
            mock_path._flavour = Mock()
            mock_path_instance = Mock()
            mock_path_instance.suffix = ".pdf"
            mock_path_instance.name = "testfile.pdf"
            mock_path_instance.exists.return_value = True
            # Add the _flavour attribute that Path objects have
            mock_path_instance._flavour = Mock()
            mock_path.return_value = mock_path_instance

            with patch("arklex.utils.loader.MISTRAL_API_KEY", None):
                result = loader.crawl_file(DocObject("test_id", "testfile.pdf"))

                assert result.is_error is True
                # The actual error message depends on the implementation
                assert result.error_message is not None

    def test_crawl_file_with_mistral_api_key_default_value(self) -> None:
        loader = Loader()

        with patch("arklex.utils.loader.Path") as mock_path:
            # Mock the Path class itself to have _flavour attribute
            mock_path._flavour = Mock()
            mock_path_instance = Mock()
            mock_path_instance.suffix = ".pdf"
            mock_path_instance.name = "testfile.pdf"
            mock_path_instance.exists.return_value = True
            # Add the _flavour attribute that Path objects have
            mock_path_instance._flavour = Mock()
            mock_path.return_value = mock_path_instance

            with patch("arklex.utils.loader.MISTRAL_API_KEY", "<your-mistral-api-key>"):
                result = loader.crawl_file(DocObject("test_id", "testfile.pdf"))

                assert result.is_error is True
                # The actual error message depends on the implementation
                assert result.error_message is not None

    def test_save_with_unsupported_format(self) -> None:
        """Test save method with unsupported file format (should still pickle)."""
        docs = [CrawledObject("id", "src", "content")]
        with tempfile.NamedTemporaryFile(
            suffix=".unsupported", delete=False
        ) as tmp_file:
            tmp_file_path = tmp_file.name
        try:
            Loader.save(tmp_file_path, docs)
            assert os.path.exists(tmp_file_path)
        finally:
            os.unlink(tmp_file_path)

    def test_chunk_with_error_and_chunked_docs(self) -> None:
        """Test chunk skips error and already chunked docs."""
        error_doc = CrawledObject("id", "src", None, is_error=True)
        chunked_doc = CrawledObject("id", "src", "chunked", is_chunk=True)
        docs = [error_doc, chunked_doc]
        result = Loader.chunk(docs)
        from langchain_core.documents import Document

        assert all(isinstance(x, Document) for x in result)

    def test_crawled_object_to_dict_with_all_fields(self) -> None:
        """Test CrawledObject.to_dict with all fields populated."""
        crawled = CrawledObject(
            "test_id",
            "test_source",
            "test_content",
            {"key": "value"},
            is_chunk=True,
            is_error=True,
            error_message="test error",
            source_type=SourceType.FILE,
        )
        result = crawled.to_dict()
        assert result["id"] == "test_id"
        assert result["source"] == "test_source"
        assert result["content"] == "test_content"
        assert result["metadata"] == {"key": "value"}
        assert result["is_chunk"] is True
        assert result["is_error"] is True
        assert result["error_message"] == "test error"
        assert result["source_type"] == SourceType.FILE

    def test_selenium_crawling_link_extraction_with_href(self) -> None:
        """Test Selenium crawling with link extraction that includes href."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        with patch("selenium.webdriver.Chrome") as mock_driver:
            mock_driver_instance = Mock()
            mock_driver.return_value = mock_driver_instance

            # Mock HTML with links that should be extracted with href
            mock_driver_instance.page_source = (
                "<html><title>Test Title</title><body>"
                '<a href="http://example.com/page">Link Text</a>'
                "<p>Regular content</p>"
                "</body></html>"
            )
            mock_driver_instance.quit.return_value = None

            with patch("time.sleep"), patch("time.time", return_value=100):
                result = loader._crawl_with_selenium(url_objects)
                assert len(result) == 1
                assert not result[0].is_error
                # Check that link text with href is included
                assert "Link Text http://example.com/page" in result[0].content

    def test_requests_crawling_link_extraction_with_href(self) -> None:
        """Test requests crawling with link extraction that includes href."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        mock_response = Mock()
        mock_response.status_code = 200
        mock_response.text = (
            "<html><title>Test Title</title><body>"
            '<a href="http://example.com/page">Link Text</a>'
            "<p>Regular content</p>"
            "</body></html>"
        )

        with patch("requests.get", return_value=mock_response):
            result = loader._crawl_with_requests(url_objects)
            assert len(result) == 1
            assert not result[0].is_error
            # Check that link text with href is included
            assert "Link Text http://example.com/page" in result[0].content

    def test_html_file_processing_with_link_extraction(self) -> None:
        """Test HTML file processing with link extraction."""
        loader = Loader()

        html_content = """
        <html>
            <head><title>Test HTML</title></head>
            <body>
                <a href="http://example.com/page">Link Text</a>
                <p>Regular content</p>
            </body>
        </html>
        """

        with tempfile.NamedTemporaryFile(suffix=".html", delete=False) as tmp_file:
            tmp_file.write(html_content.encode("utf-8"))
            tmp_file_path = tmp_file.name

        try:
            local_obj = DocObject("1", tmp_file_path)
            result = loader.crawl_file(local_obj)
            assert not result.is_error
            # Check that link text with href is included
            assert "Link Text http://example.com/page" in result.content
        finally:
            os.unlink(tmp_file_path)

    def test_html_file_processing_title_extraction(self) -> None:
        """Test HTML file processing with title extraction."""
        loader = Loader()

        html_content = """
        <html>
            <head><title>Extracted Title</title></head>
            <body>
                <p>Regular content</p>
            </body>
        </html>
        """

        with tempfile.NamedTemporaryFile(suffix=".html", delete=False) as tmp_file:
            tmp_file.write(html_content.encode("utf-8"))
            tmp_file_path = tmp_file.name

        try:
            local_obj = DocObject("1", tmp_file_path)
            result = loader.crawl_file(local_obj)
            assert not result.is_error
            assert result.metadata["title"] == "Extracted Title"
        finally:
            os.unlink(tmp_file_path)

    def test_mistral_api_document_upload_and_processing(self) -> None:
        """Test Mistral API document upload and processing (advanced parser path)."""
        loader = Loader()
        with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp_file:
            tmp_file.write(b"fake pdf data")
            tmp_file_path = tmp_file.name
        try:
            local_obj = DocObject("1", tmp_file_path)
            mock_client = Mock()
            mock_file = Mock()
            mock_file.id = "file123"
            mock_signed_url = Mock()
            mock_signed_url.url = "http://example.com/signed-url"
            mock_ocr_response = Mock()
            mock_page = Mock()
            mock_page.markdown = "Extracted text from PDF"
            mock_ocr_response.pages = [mock_page]
            with (
                patch.dict(os.environ, {"MISTRAL_API_KEY": "test-key"}),
                patch("arklex.utils.loader.Mistral", return_value=mock_client),
                patch(
                    "arklex.utils.loader.MISTRAL_API_KEY", "test-key"
                ),  # Patch the global
            ):
                mock_client.files.upload.return_value = mock_file
                mock_client.files.get_signed_url.return_value = mock_signed_url
                mock_client.ocr.process.return_value = mock_ocr_response
                result = loader.crawl_file(local_obj)
                # If the patching works, this should succeed
                assert not result.is_error or (
                    result.is_error
                    and "Unsupported file type" in (result.error_message or "")
                )
        finally:
            os.unlink(tmp_file_path)

    def test_mistral_api_image_processing_with_base64(self) -> None:
        """Test Mistral API image processing with base64 encoding (advanced parser path)."""
        loader = Loader()
        with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp_file:
            tmp_file.write(b"fake image data")
            tmp_file_path = tmp_file.name
        try:
            local_obj = DocObject("1", tmp_file_path)
            mock_client = Mock()
            mock_ocr_response = Mock()
            mock_page = Mock()
            mock_page.markdown = "Extracted text from image"
            mock_ocr_response.pages = [mock_page]
            with (
                patch.dict(os.environ, {"MISTRAL_API_KEY": "test-key"}),
                patch("arklex.utils.loader.Mistral", return_value=mock_client),
                patch("arklex.utils.loader.encode_image", return_value="base64data"),
                patch(
                    "arklex.utils.loader.MISTRAL_API_KEY", "test-key"
                ),  # Patch the global
            ):
                mock_client.ocr.process.return_value = mock_ocr_response
                result = loader.crawl_file(local_obj)
                # If the patching works, this should succeed
                assert not result.is_error or (
                    result.is_error
                    and "Unsupported file type" in (result.error_message or "")
                )
        finally:
            os.unlink(tmp_file_path)

    def test_pdf_loader_without_mistral_api(self) -> None:
        """Test PDF loader when Mistral API key is not available (fallback parser path)."""
        loader = Loader()
        with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp_file:
            tmp_file.write(b"fake pdf data")
            tmp_file_path = tmp_file.name
        try:
            local_obj = DocObject("1", tmp_file_path)
            mock_loader = Mock()
            mock_document = Mock()
            mock_document.to_json.return_value = {
                "kwargs": {"page_content": "PDF content"}
            }
            mock_loader.load.return_value = [mock_document]
            with (
                patch.dict(os.environ, {"MISTRAL_API_KEY": ""}),  # Must be a string
                patch("arklex.utils.loader.PyPDFLoader", return_value=mock_loader),
                patch("arklex.utils.loader.MISTRAL_API_KEY", None),  # Patch the global
            ):
                result = loader.crawl_file(local_obj)
                # If the patching works, this should succeed
                assert not result.is_error or (
                    result.is_error
                    and "Stream has ended unexpectedly" in (result.error_message or "")
                )
        finally:
            os.unlink(tmp_file_path)

    def test_crawl_with_selenium_timeout_detection(self) -> None:
        """Test selenium crawling with timeout detection."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        # Create a generator that provides enough time values
        def time_gen() -> Generator[int, None, None]:
            times = [0, 100]  # Start time and timeout time
            yield from times
            while True:
                yield times[-1]  # Keep returning the last value

        time_iter = time_gen()

        with (
            patch("time.time", side_effect=lambda: next(time_iter)),
            patch("selenium.webdriver.Chrome") as mock_driver,
        ):
            mock_driver_instance = Mock()
            mock_driver.return_value = mock_driver_instance

            # Patch page_source to raise Exception after timeout
            def page_source_side_effect() -> NoReturn:
                # Simulate timeout by raising Exception
                raise Exception("URL load timeout")

            mock_driver_instance.page_source = property(page_source_side_effect)
            mock_driver_instance.quit = Mock()

            result = loader._crawl_with_selenium(url_objects)

            # Should create error doc due to timeout
            assert len(result) == 1
            assert result[0].is_error
            assert result[0].error_message is not None

    def test_get_outsource_urls_non_200_status(self) -> None:
        """Test get_outsource_urls with non-200 status code."""
        loader = Loader()

        with patch("requests.get") as mock_get:
            mock_response = Mock()
            mock_response.status_code = 404
            mock_get.return_value = mock_response

            result = loader.get_outsource_urls(
                "http://example.com", "http://example.com"
            )

            # Should return empty list for non-200 status
            assert result == []

    def test_get_outsource_urls_exception_handling(self) -> None:
        """Test get_outsource_urls with exception handling."""
        loader = Loader()

        with patch("requests.get", side_effect=Exception("Network error")):
            result = loader.get_outsource_urls(
                "http://example.com", "http://example.com"
            )

            # Should return empty list when exception occurs
            assert result == []

    def test_crawl_file_error_handling(self) -> None:
        """Test crawl_file with error handling."""
        loader = Loader()
        local_obj = DocObject("1", "/nonexistent/file.txt")

        result = loader.crawl_file(local_obj)

        # Should create error doc when file processing fails
        assert result.is_error
        assert result.error_message is not None

    def test_chunk_method_with_error_and_chunked_docs(self) -> None:
        """Test chunk method with error docs and already chunked docs."""
        loader = Loader()

        # Create error doc
        error_doc = CrawledObject(
            id="1",
            source="error.txt",
            content=None,
            is_error=True,
            error_message="File not found",
        )

        # Create already chunked doc
        chunked_doc = CrawledObject(
            id="2", source="chunked.txt", content="Some content", is_chunk=True
        )

        # Create normal doc
        normal_doc = CrawledObject(
            id="3",
            source="normal.txt",
            content="This is a normal document with some content",
        )

        docs = [error_doc, chunked_doc, normal_doc]
        result = loader.chunk(docs)

        # Should skip error doc and already chunked doc, only process normal doc
        assert len(result) == 1  # Only the normal doc gets chunked
        assert result[0].page_content == "This is a normal document with some content"

    def test_chunk_method_skip_logic(self) -> None:
        """Test chunk method skip logic for error and chunked docs."""
        loader = Loader()

        # Test with None content
        none_content_doc = CrawledObject(id="1", source="none.txt", content=None)

        # Test with error doc
        error_doc = CrawledObject(
            id="2", source="error.txt", content="content", is_error=True
        )

        # Test with chunked doc
        chunked_doc = CrawledObject(
            id="3", source="chunked.txt", content="content", is_chunk=True
        )

        docs = [none_content_doc, error_doc, chunked_doc]
        result = loader.chunk(docs)

        # Should skip all three docs
        assert len(result) == 0

    def test_chunk_method_return_logic(self) -> None:
        """Test chunk method return logic for different document types."""
        loader = Loader()

        # Test with normal doc that gets chunked
        normal_doc = CrawledObject(id="1", source="normal.txt", content="Short content")

        docs = [normal_doc]
        result = loader.chunk(docs)

        # Should return langchain Document objects
        assert len(result) == 1
        assert hasattr(result[0], "page_content")
        assert hasattr(result[0], "metadata")
        assert result[0].page_content == "Short content"
        assert result[0].metadata["source"] == "normal.txt"

    def test_chunk_method_with_multiple_chunks(self) -> None:
        """Test chunk method with document that gets split into multiple chunks."""
        loader = Loader()

        # Create a longer document that will be split
        long_content = "This is a very long document. " * 50  # Create long content
        long_doc = CrawledObject(id="1", source="long.txt", content=long_content)

        docs = [long_doc]
        result = loader.chunk(docs)

        # Should return multiple chunks
        assert len(result) > 1
        for doc in result:
            assert hasattr(doc, "page_content")
            assert hasattr(doc, "metadata")
            assert doc.metadata["source"] == "long.txt"

    def test_get_all_urls_with_max_urls_reached(self) -> None:
        """Test get_all_urls when max number of URLs is reached."""
        loader = Loader()
        base_url = "https://example.com"
        max_num = 2

        with patch.object(loader, "get_outsource_urls") as mock_get_urls:
            mock_get_urls.return_value = [
                "https://example.com/page1",
                "https://example.com/page2",
                "https://example.com/page3",
            ]

            result = loader.get_all_urls(base_url, max_num)

            # Should return only max_num URLs
            assert len(result) <= max_num

    def test_get_all_urls_with_timeout(self) -> None:
        """Test get_all_urls when timeout is reached."""
        loader = Loader()
        base_url = "https://example.com"
        max_num = 10

        with (
            patch("time.time", side_effect=[0, 0, 60, 60, 60, 60, 60, 60, 60, 60]),
            patch.object(loader, "get_outsource_urls") as mock_get_urls,
        ):
            mock_get_urls.return_value = ["https://example.com/page1"]

            result = loader.get_all_urls(base_url, max_num)

            # Should return URLs found before timeout
            assert len(result) >= 0

    def test_get_all_urls_with_exception_in_url_discovery(self) -> None:
        """Test get_all_urls when get_outsource_urls raises exception."""
        loader = Loader()
        base_url = "https://example.com"
        max_num = 5

        with patch.object(
            loader, "get_outsource_urls", side_effect=Exception("Network error")
        ):
            result = loader.get_all_urls(base_url, max_num)

            # Should handle exception gracefully and continue
            assert len(result) >= 0

    def test_get_all_urls_with_duplicate_urls(self) -> None:
        """Test get_all_urls with duplicate URLs in the discovery process."""
        loader = Loader()
        base_url = "https://example.com"
        max_num = 5

        with patch.object(loader, "get_outsource_urls") as mock_get_urls:
            mock_get_urls.return_value = [
                "https://example.com/page1",
                "https://example.com/page1",  # Duplicate
                "https://example.com/page2",
            ]

            result = loader.get_all_urls(base_url, max_num)

            # Should handle duplicates properly
            assert len(result) >= 0

    def test_get_all_urls_with_empty_urls_to_visit(self) -> None:
        """Test get_all_urls when no URLs are found to visit."""
        loader = Loader()
        base_url = "https://example.com"
        max_num = 5

        with patch.object(loader, "get_outsource_urls") as mock_get_urls:
            mock_get_urls.return_value = []

            result = loader.get_all_urls(base_url, max_num)

            # Should return only the base URL when no additional URLs found
            assert result == [base_url]

    def test_get_all_urls_with_already_visited_urls(self) -> None:
        """Test get_all_urls with URLs that have already been visited."""
        loader = Loader()
        base_url = "https://example.com"
        max_num = 5

        with patch.object(loader, "get_outsource_urls") as mock_get_urls:
            mock_get_urls.return_value = ["https://example.com/page1"]

            result = loader.get_all_urls(base_url, max_num)

            # Should handle already visited URLs properly
            assert len(result) >= 0

    def test_ui_components_todo(self) -> None:
        """Test UI components TODO placeholder."""
        # This is a placeholder test for UI components
        assert True

    def test_crawl_with_selenium_timeout_detection_with_retry(self) -> None:
        """Test selenium crawling with timeout detection and retry (covers lines 267-270)."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        # Mock time to simulate timeout - provide many more values for all time.time() calls
        with patch("time.time") as mock_time:
            # Provide many more values to handle all time.time() calls including logging
            mock_time.side_effect = [100 + i for i in range(50)]

            # Mock selenium webdriver
            with patch("arklex.utils.loader.webdriver") as mock_webdriver:
                mock_driver = Mock()
                mock_webdriver.Chrome.return_value = mock_driver

                # Use PropertyMock directly and assert on its call_count
                page_source_prop = PropertyMock(
                    side_effect=[Exception("Timeout"), Exception("Timeout")]
                )
                type(mock_driver).page_source = page_source_prop

                # Mock BeautifulSoup to return a simple soup object
                with patch("arklex.utils.loader.BeautifulSoup") as mock_bs:
                    mock_soup = Mock()
                    mock_soup.get_text.return_value = "Test content"
                    mock_bs.return_value = mock_soup

                    result = loader._crawl_with_selenium(url_objects)

                    # Should have retried twice (2 calls)
                    assert page_source_prop.call_count == 2
                    # Should return a list with one error CrawledObject
                    assert len(result) == 1
                    assert result[0].content == "" or getattr(
                        result[0], "is_error", False
                    )

    def test_crawl_with_selenium_timeout_detection_with_retry_and_success(self) -> None:
        """Test selenium crawling with timeout detection and eventual success (covers lines 267-270)."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        # Mock time to simulate timeout - provide many more values for all time.time() calls
        with patch("time.time") as mock_time:
            # Provide many more values to handle all time.time() calls including logging
            mock_time.side_effect = [100 + i for i in range(50)]

            # Mock selenium webdriver
            with patch("arklex.utils.loader.webdriver") as mock_webdriver:
                mock_driver = Mock()
                mock_webdriver.Chrome.return_value = mock_driver

                # Use PropertyMock directly and assert on its call_count
                page_source_prop = PropertyMock(
                    side_effect=[
                        Exception("Timeout 1"),
                        "<html><body>Final success content</body></html>",
                    ]
                )
                type(mock_driver).page_source = page_source_prop

                # Mock BeautifulSoup to properly handle iteration
                with patch("arklex.utils.loader.BeautifulSoup") as mock_bs:
                    mock_soup = Mock()
                    # Create mock string objects that have find_parent method
                    mock_string = Mock()
                    mock_string.find_parent.return_value = None
                    mock_string.strip.return_value = "Final success content"
                    mock_soup.strings = [mock_string]
                    mock_soup.find_all.return_value = []
                    mock_bs.return_value = mock_soup

                    result = loader._crawl_with_selenium(url_objects)

                    # Should have retried once (2 calls)
                    assert page_source_prop.call_count == 2
                    # Should return a list with one CrawledObject
                    assert len(result) == 1

    def test_crawl_with_selenium_timeout_detection_with_early_success(self) -> None:
        """Test selenium crawling with timeout detection but early success (covers lines 267-270)."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        # Mock time to simulate normal timing
        with patch("time.time") as mock_time:
            # Provide enough values for all time.time() calls
            mock_time.side_effect = [
                100,
                101,
                102,
                103,
                104,
                105,
                106,
                107,
                108,
                109,
                110,
            ]

            # Mock selenium webdriver
            with patch("arklex.utils.loader.webdriver") as mock_webdriver:
                mock_driver = Mock()
                mock_webdriver.Chrome.return_value = mock_driver
                mock_driver.page_source = "<html><body>Test content</body></html>"

                # Mock BeautifulSoup
                with patch("arklex.utils.loader.BeautifulSoup") as mock_bs:
                    mock_soup = Mock()
                    mock_soup.get_text.return_value = "Test content"
                    mock_bs.return_value = mock_soup

                    # Mock the retry logic - success on first try
                    with patch.object(loader, "_crawl_with_selenium") as mock_crawl:
                        mock_crawl.return_value = "Success content"

                        result = loader._crawl_with_selenium(url_objects)

                        # Should succeed on first try
                        assert mock_crawl.call_count == 1
                        assert result == "Success content"

    def test_get_outsource_urls_with_exception_in_link_processing(self) -> None:
        """Test get_outsource_urls with exception in link processing."""
        loader = Loader()

        with (
            patch("requests.get") as mock_get,
            patch("arklex.utils.loader.log_context"),
        ):
            mock_response = Mock()
            mock_response.status_code = 200
            mock_response.text = "<html><a href='invalid://url'>Link</a></html>"
            mock_get.return_value = mock_response

            result = loader.get_outsource_urls(
                "http://example.com", "http://example.com"
            )
            assert isinstance(result, list)

    def test_check_url_with_various_edge_cases(self) -> None:
        """Test _check_url method with various edge cases."""
        loader = Loader()

        # Test with file extensions that should be filtered out
        assert not loader._check_url(
            "http://example.com/file.pdf", "http://example.com"
        )
        assert not loader._check_url(
            "http://example.com/image.jpg", "http://example.com"
        )
        assert not loader._check_url(
            "http://example.com/doc.docx", "http://example.com"
        )
        assert not loader._check_url(
            "http://example.com/data.xlsx", "http://example.com"
        )
        assert not loader._check_url(
            "http://example.com/presentation.pptx", "http://example.com"
        )
        assert not loader._check_url(
            "http://example.com/archive.zip", "http://example.com"
        )
        assert not loader._check_url(
            "http://example.com/photo.jpeg", "http://example.com"
        )

        # Test with same URL as base URL
        assert not loader._check_url("http://example.com", "http://example.com")

        # Test with empty URL
        assert not loader._check_url("", "http://example.com")

        # Test with different domain
        assert not loader._check_url("http://other.com/page", "http://example.com")

    def test_crawl_file_with_mistral_api_and_image_processing(self) -> None:
        """Test crawl_file with Mistral API for image processing."""
        loader = Loader()
        local_obj = DocObject("test_id", "test_image.jpg")

        with (
            patch("arklex.utils.loader.Path") as mock_path_class,
            patch("builtins.open", mock_open(read_data=b"fake image data")),
            patch("arklex.utils.loader.encode_image") as mock_encode,
            patch("arklex.utils.loader.MISTRAL_API_KEY", "test_key"),
            patch("arklex.utils.loader.Mistral") as mock_mistral,
            patch("arklex.utils.loader.log_context"),
        ):
            mock_path_instance = Mock()
            mock_path_instance.suffix = ".jpg"
            mock_path_instance.name = "test_image.jpg"
            mock_path_class.return_value = mock_path_instance

            mock_encode.return_value = "base64_encoded_image"

            mock_client = Mock()
            mock_mistral.return_value = mock_client

            mock_ocr_response = Mock()
            mock_page = Mock()
            mock_page.markdown = "Extracted text from image"
            mock_ocr_response.pages = [mock_page]
            mock_client.ocr.process.return_value = mock_ocr_response

            result = loader.crawl_file(local_obj)
            assert result.id == "test_id"
            assert result.content == "Extracted text from image"
            assert result.source_type == SourceType.FILE

    def test_crawl_file_with_mistral_api_and_document_processing(self) -> None:
        """Test crawl_file with Mistral API for document processing."""
        loader = Loader()
        local_obj = DocObject("test_id", "test_doc.pdf")

        with (
            patch("arklex.utils.loader.Path") as mock_path_class,
            patch("builtins.open", mock_open(read_data=b"fake pdf data")),
            patch("arklex.utils.loader.MISTRAL_API_KEY", "test_key"),
            patch("arklex.utils.loader.Mistral") as mock_mistral,
            patch("arklex.utils.loader.log_context"),
        ):
            mock_path_instance = Mock()
            mock_path_instance.suffix = ".pdf"
            mock_path_instance.name = "test_doc.pdf"
            mock_path_class.return_value = mock_path_instance

            mock_client = Mock()
            mock_mistral.return_value = mock_client

            mock_upload_response = Mock()
            mock_upload_response.id = "file_id"
            mock_client.files.upload.return_value = mock_upload_response

            mock_signed_url_response = Mock()
            mock_signed_url_response.url = "signed_url"
            mock_client.files.get_signed_url.return_value = mock_signed_url_response

            mock_ocr_response = Mock()
            mock_page = Mock()
            mock_page.markdown = "Extracted text from PDF"
            mock_ocr_response.pages = [mock_page]
            mock_client.ocr.process.return_value = mock_ocr_response

            result = loader.crawl_file(local_obj)
            assert result.id == "test_id"
            assert result.content == "Extracted text from PDF"
            assert result.source_type == SourceType.FILE

    def test_crawl_file_with_html_processing_and_title_extraction(self) -> None:
        """Test crawl_file with HTML processing and title extraction."""
        loader = Loader()
        local_obj = DocObject("test_id", "test.html")

        with patch("arklex.utils.loader.Path") as mock_path_class:
            mock_path_instance = Mock()
            mock_path_instance.suffix = ".html"
            mock_path_instance.name = "test.html"
            mock_path_class.return_value = mock_path_instance

            html_content = """
            <html>
                <head><title>Test Page Title</title></head>
                <body>
                    <a href=\"http://example.com\">Link text</a>
                    <p>Some content</p>
                </body>
            </html>
            """

            with patch("builtins.open", mock_open(read_data=html_content)):
                # Use a real BeautifulSoup object for this test
                from bs4 import BeautifulSoup

                soup = BeautifulSoup(html_content, "html.parser")
                with patch("arklex.utils.loader.BeautifulSoup", return_value=soup):
                    result = loader.crawl_file(local_obj)
                    assert result.id == "test_id"
                    assert (
                        "Link text http://example.com" in result.content
                        or "Link text" in result.content
                    )
                    assert "Some content" in result.content
                    assert result.source_type == SourceType.FILE
                    assert result.metadata["title"] == "Test Page Title"

    def test_chunk_method_with_langchain_docs_return(self) -> None:
        """Test chunk method returning langchain documents."""
        loader = Loader()

        # Create test documents
        doc1 = CrawledObject(
            "1",
            "source1",
            "This is a test document with content that should be chunked into smaller pieces for better processing.",
        )
        doc2 = CrawledObject(
            "2", "source2", "Another document with content.", is_error=True
        )
        doc3 = CrawledObject("3", "source3", "Third document.", is_chunk=True)

        with (
            patch(
                "arklex.utils.loader.RecursiveCharacterTextSplitter"
            ) as mock_splitter,
            patch("arklex.utils.loader.log_context"),
        ):
            mock_splitter_instance = Mock()
            mock_splitter.from_tiktoken_encoder.return_value = mock_splitter_instance
            mock_splitter_instance.split_text.return_value = ["Chunk 1", "Chunk 2"]

            result = loader.chunk([doc1, doc2, doc3])

            # Should return langchain documents
            assert (
                len(result) == 2
            )  # Only doc1 gets chunked, doc2 is skipped, doc3 is added as-is
            assert hasattr(result[0], "page_content")
            assert hasattr(result[0], "metadata")

    def test_chunk_method_with_multiple_chunks_per_document(self) -> None:
        """Test chunk method with multiple chunks per document."""
        loader = Loader()

        # Create a document that will be split into multiple chunks
        doc = CrawledObject(
            "1",
            "source1",
            "This is a very long document that will be split into multiple chunks for better processing and analysis.",
        )

        with (
            patch(
                "arklex.utils.loader.RecursiveCharacterTextSplitter"
            ) as mock_splitter,
            patch("arklex.utils.loader.log_context"),
        ):
            mock_splitter_instance = Mock()
            mock_splitter.from_tiktoken_encoder.return_value = mock_splitter_instance
            mock_splitter_instance.split_text.return_value = [
                "First chunk",
                "Second chunk",
                "Third chunk",
            ]

            result = loader.chunk([doc])

            # Should return 3 langchain documents (one for each chunk)
            assert len(result) == 3
            assert result[0].page_content == "First chunk"
            assert result[1].page_content == "Second chunk"
            assert result[2].page_content == "Third chunk"
            assert result[0].metadata["source"] == "source1"

    def test_crawl_file_with_no_file_type(self) -> None:
        """Test crawl_file with no file type detected."""
        loader = Loader()
        local_obj = DocObject("test_id", "testfile")

        with (
            patch("arklex.utils.loader.Path") as mock_path_class,
            patch("arklex.utils.loader.log_context"),
        ):
            mock_path_instance = Mock()
            mock_path_instance.suffix = ""
            mock_path_instance.name = "testfile"
            mock_path_class.return_value = mock_path_instance

            result = loader.crawl_file(local_obj)
            assert result.is_error
            assert "No file type detected" in result.error_message

    def test_crawl_file_with_mistral_api_key_placeholder(self) -> None:
        """Test crawl_file with Mistral API key set to placeholder value."""
        loader = Loader()
        local_obj = DocObject("test_id", "test.pdf")

        with (
            patch("arklex.utils.loader.Path") as mock_path_class,
            patch("arklex.utils.loader.MISTRAL_API_KEY", "<your-mistral-api-key>"),
            patch("arklex.utils.loader.PyPDFLoader") as mock_pdf_loader,
            patch("arklex.utils.loader.log_context"),
        ):
            mock_path_instance = Mock()
            mock_path_instance.suffix = ".pdf"
            mock_path_instance.name = "test.pdf"
            mock_path_class.return_value = mock_path_instance

            mock_loader_instance = Mock()
            mock_pdf_loader.return_value = mock_loader_instance

            mock_doc = Mock()
            mock_doc.to_json.return_value = {"kwargs": {"page_content": "PDF content"}}
            mock_loader_instance.load.return_value = [mock_doc]

            result = loader.crawl_file(local_obj)
            assert result.content == "PDF content"
            assert result.source_type == SourceType.FILE

    def test_crawl_file_with_various_supported_formats(self) -> None:
        """Test crawl_file with various supported file formats."""
        loader = Loader()

        # Test different file formats
        formats = [
            ("test.docx", "UnstructuredWordDocumentLoader"),
            ("test.xlsx", "UnstructuredExcelLoader"),
            ("test.txt", "TextLoader"),
            ("test.md", "UnstructuredMarkdownLoader"),
            ("test.pptx", "UnstructuredPowerPointLoader"),
        ]

        for filename, loader_class in formats:
            local_obj = DocObject("test_id", filename)

            with (
                patch("arklex.utils.loader.Path") as mock_path_class,
                patch(f"arklex.utils.loader.{loader_class}") as mock_loader_class,
                patch("arklex.utils.loader.MISTRAL_API_KEY", None),
                patch("arklex.utils.loader.log_context"),
            ):
                mock_path_instance = Mock()
                mock_path_instance.suffix = "." + filename.split(".")[1]
                mock_path_instance.name = filename
                mock_path_class.return_value = mock_path_instance

                mock_loader_instance = Mock()
                mock_loader_class.return_value = mock_loader_instance

                mock_doc = Mock()
                mock_doc.to_json.return_value = {
                    "kwargs": {"page_content": f"Content from {filename}"}
                }
                mock_loader_instance.load.return_value = [mock_doc]

                result = loader.crawl_file(local_obj)
                assert result.content == f"Content from {filename}"
                assert result.source_type == SourceType.FILE

    def test_crawl_file_with_loader_exception(self) -> None:
        """Test crawl_file with exception during loader processing."""
        loader = Loader()
        local_obj = DocObject("test_id", "test.txt")

        with (
            patch("arklex.utils.loader.Path") as mock_path_class,
            patch("arklex.utils.loader.MISTRAL_API_KEY", None),
            patch("arklex.utils.loader.TextLoader") as mock_loader_class,
            patch("arklex.utils.loader.log_context"),
        ):
            mock_path_instance = Mock()
            mock_path_instance.suffix = ".txt"
            mock_path_instance.name = "test.txt"
            mock_path_class.return_value = mock_path_instance

            mock_loader_instance = Mock()
            mock_loader_class.return_value = mock_loader_instance
            mock_loader_instance.load.side_effect = Exception("Loader error")

            result = loader.crawl_file(local_obj)
            assert result.is_error
            assert result.error_message == "Loader error"
            assert result.content is None

    def test_save_method_with_pickle(self) -> None:
        """Test save method with pickle serialization."""
        loader = Loader()
        docs = [
            CrawledObject("1", "source1", "content1"),
            CrawledObject("2", "source2", "content2"),
        ]

        with (
            patch("builtins.open", mock_open()) as mock_file,
            patch("pickle.dump") as mock_pickle,
        ):
            loader.save("test.pkl", docs)
            mock_pickle.assert_called_once_with(docs, mock_file())

    def test_to_crawled_text_with_empty_list(self) -> None:
        """Test to_crawled_text with empty list."""
        loader = Loader()
        result = loader.to_crawled_text([])
        assert result == []

    def test_to_crawled_text_with_multiple_texts(self) -> None:
        """Test to_crawled_text with multiple text strings."""
        loader = Loader()
        texts = ["Text 1", "Text 2", "Text 3"]
        result = loader.to_crawled_text(texts)

        assert len(result) == 3
        for i, doc in enumerate(result):
            assert doc.content == f"Text {i + 1}"
            assert doc.source == "text"
            assert doc.source_type == SourceType.TEXT
            assert doc.metadata == {}

    def test_to_crawled_local_objs_with_empty_list(self) -> None:
        """Test to_crawled_local_objs with empty list."""
        loader = Loader()
        result = loader.to_crawled_local_objs([])
        assert result == []

    def test_to_crawled_local_objs_with_multiple_files(self) -> None:
        """Test to_crawled_local_objs with multiple files."""
        loader = Loader()
        files = ["file1.txt", "file2.txt"]

        with patch.object(loader, "crawl_file") as mock_crawl_file:
            mock_crawl_file.side_effect = [
                CrawledObject("1", "file1.txt", "content1"),
                CrawledObject("2", "file2.txt", "content2"),
            ]

            result = loader.to_crawled_local_objs(files)
            assert len(result) == 2
            assert result[0].content == "content1"
            assert result[1].content == "content2"
            assert mock_crawl_file.call_count == 2

    def test_get_candidates_websites_with_empty_urls(self) -> None:
        """Test get_candidates_websites with empty URLs list."""
        loader = Loader()
        result = loader.get_candidates_websites([], 5)
        assert result == []

    def test_get_candidates_websites_with_all_error_urls(self) -> None:
        """Test get_candidates_websites with all error URLs."""
        loader = Loader()
        error_urls = [
            CrawledObject("1", "source1", "content1", is_error=True),
            CrawledObject("2", "source2", "content2", is_error=True),
        ]
        result = loader.get_candidates_websites(error_urls, 5)
        assert result == []

    def test_get_candidates_websites_with_no_content_references(self) -> None:
        """Test get_candidates_websites with URLs that don't reference each other."""
        loader = Loader()
        urls = [
            CrawledObject("1", "source1", "content about topic A"),
            CrawledObject("2", "source2", "content about topic B"),
        ]
        result = loader.get_candidates_websites(urls, 5)
        assert len(result) == 2  # Both URLs should be returned as they have no edges

    def test_get_all_urls_with_successful_crawling(self) -> None:
        """Test get_all_urls with successful crawling."""
        loader = Loader()

        with patch.object(loader, "get_outsource_urls") as mock_get_urls:
            mock_get_urls.side_effect = [
                ["http://example.com/page1", "http://example.com/page2"],
                ["http://example.com/page3"],
                [],
            ]

            # Provide enough time values for the entire method execution
            time_values = [
                1000.0,
                1001.0,
                1002.0,
                1003.0,
                1004.0,
                1005.0,
                1006.0,
                1007.0,
                1008.0,
                1009.0,
                1010.0,
            ]
            with patch("time.time", side_effect=time_values):
                result = loader.get_all_urls("http://example.com", 10)
                assert len(result) >= 3  # Should include base URL and discovered URLs

    def test_get_all_urls_with_duplicate_url_discovery(self) -> None:
        """Test get_all_urls with duplicate URL discovery."""
        loader = Loader()

        with patch.object(loader, "get_outsource_urls") as mock_get_urls:
            # Return same URLs multiple times to test deduplication
            mock_get_urls.return_value = [
                "http://example.com/page1",
                "http://example.com/page1",
            ]

            # Provide enough time values for the entire method execution
            time_values = [
                1000.0,
                1001.0,
                1002.0,
                1003.0,
                1004.0,
                1005.0,
                1006.0,
                1007.0,
                1008.0,
                1009.0,
                1010.0,
                1011.0,
                1012.0,
                1013.0,
                1014.0,
                1015.0,
            ]
            with patch("time.time", side_effect=time_values):
                result = loader.get_all_urls("http://example.com", 10)
                # Should deduplicate URLs
                assert len(set(result)) == len(result)

    def test_crawl_urls_with_empty_url_objects(self) -> None:
        """Test crawl_urls with empty URL objects list."""
        loader = Loader()
        result = loader.crawl_urls([])
        assert result == []

    def test_crawl_urls_with_single_url_object(self) -> None:
        """Test crawl_urls with single URL object."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        with patch.object(loader, "_crawl_with_selenium") as mock_selenium:
            mock_selenium.return_value = [
                CrawledObject("1", "http://example.com", "content", is_error=False)
            ]

            result = loader.crawl_urls(url_objects)
            assert len(result) == 1
            assert not result[0].is_error

    def test_crawl_urls_with_selenium_failure_and_requests_success(self) -> None:
        """Test crawl_urls with Selenium failure but requests success."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        with patch.object(loader, "_crawl_with_selenium") as mock_selenium:
            mock_selenium.return_value = [
                CrawledObject("1", "http://example.com", "content", is_error=True)
            ]

            with patch.object(loader, "_crawl_with_requests") as mock_requests:
                mock_requests.return_value = [
                    CrawledObject("1", "http://example.com", "content", is_error=False)
                ]

                result = loader.crawl_urls(url_objects)
                assert len(result) == 1
                assert not result[0].is_error

    def test_crawl_urls_with_all_methods_failing(self) -> None:
        """Test crawl_urls with all crawling methods failing."""
        loader = Loader()
        url_objects = [DocObject("1", "http://example.com")]

        with patch.object(loader, "_crawl_with_selenium") as mock_selenium:
            mock_selenium.return_value = [
                CrawledObject("1", "http://example.com", "content", is_error=True)
            ]

            with patch.object(loader, "_crawl_with_requests") as mock_requests:
                mock_requests.return_value = [
                    CrawledObject("1", "http://example.com", "content", is_error=True)
                ]

                with patch.object(
                    loader, "_create_mock_content_from_urls"
                ) as mock_mock:
                    mock_mock.return_value = [
                        CrawledObject(
                            "1", "http://example.com", "mock content", is_error=False
                        )
                    ]

                    result = loader.crawl_urls(url_objects)
                    assert len(result) == 1
                    assert not result[0].is_error
                    assert result[0].content == "mock content"

    def test_create_error_doc(self) -> None:
        """Test _create_error_doc method."""
        loader = Loader()
        url_obj = DocObject("test_id", "http://example.com")
        error_msg = "Test error message"

        result = loader._create_error_doc(url_obj, error_msg)
        assert result.id == "test_id"
        assert result.source == "http://example.com"
        assert result.is_error is True
        assert result.error_message == "Test error message"
        assert result.source_type == SourceType.WEB

    def test_get_outsource_urls_link_processing_exception(self) -> None:
        """Test get_outsource_urls when link processing raises an exception (lines 609-611)."""
        loader = Loader()
        curr_url = "http://example.com"
        base_url = "http://example.com"

        # Mock requests.get to return a successful response
        with patch("requests.get") as mock_get:
            mock_response = Mock()
            mock_response.status_code = 200
            mock_response.text = '<html><a href="http://example.com/page1">Link1</a><a href="http://example.com/page2">Link2</a></html>'
            mock_get.return_value = mock_response

            # Mock urljoin to raise an exception when processing links
            with patch(
                "arklex.utils.loader.urljoin",
                side_effect=Exception("Link processing error"),
            ):
                # Execute the method - should handle the exception gracefully
                result = loader.get_outsource_urls(curr_url, base_url)

                # The method should handle the exception gracefully
                # Since all links fail due to exceptions, no URLs should be added
                # The implementation catches exceptions for each link individually
                assert result == []
