"""
Comprehensive coverage tests for DownloadService.

Covers all major methods including:
- __init__, close, context manager, _setup_directories
- _normalize_url, _get_url_hash
- is_already_downloaded
- get_text_content
- queue_research_downloads, _is_downloadable
- download_resource
- _download_pdf
- _extract_text_from_pdf
- download_as_text and its sub-methods
- _try_library_text_extraction
- _try_existing_text
- _try_legacy_text_file
- _try_existing_pdf_extraction
- _try_api_text_extraction
- _fallback_pdf_extraction
- _get_downloader
- _download_generic, _download_arxiv, _download_pubmed, etc.
- _save_text_with_db
- _create_text_document_record
- _record_failed_text_extraction
"""

from unittest.mock import MagicMock, patch

import pytest

from local_deep_research.research_library.services.download_service import (
    DownloadService,
)

MODULE = "local_deep_research.research_library.services.download_service"


@pytest.fixture
def svc():
    """Create a DownloadService with mocked __init__."""
    with patch.object(DownloadService, "__init__", lambda self, *a, **kw: None):
        service = DownloadService.__new__(DownloadService)
        service.username = "test_user"
        service.password = "test_pass"
        service._closed = False
        service.downloaders = []
        service.retry_manager = MagicMock()
        service.settings = MagicMock()
        service.library_root = "/tmp/test_library"
        service._pubmed_delay = 1.0
        service._last_pubmed_request = 0.0
        return service


# ============================================================
# _normalize_url
# ============================================================


class TestNormalizeUrl:
    def test_strips_http(self, svc):
        assert (
            svc._normalize_url("http://example.com/page") == "example.com/page"
        )

    def test_strips_https(self, svc):
        assert (
            svc._normalize_url("https://example.com/page") == "example.com/page"
        )

    def test_strips_www(self, svc):
        assert (
            svc._normalize_url("https://www.example.com/page")
            == "example.com/page"
        )

    def test_trailing_slashes(self, svc):
        assert (
            svc._normalize_url("https://example.com/page///")
            == "example.com/page"
        )

    def test_sorts_query_params(self, svc):
        result = svc._normalize_url("https://example.com/s?z=1&a=2")
        assert result == "example.com/s?a=2&z=1"

    def test_lowercases(self, svc):
        # Note: regex ^https?:// is case-sensitive, so uppercase protocol is not stripped
        assert (
            svc._normalize_url("https://EXAMPLE.COM/Page") == "example.com/page"
        )

    def test_no_query(self, svc):
        assert (
            svc._normalize_url("https://example.com/path") == "example.com/path"
        )

    def test_empty_query(self, svc):
        result = svc._normalize_url("https://example.com/page?")
        assert result == "example.com/page?"

    def test_no_protocol(self, svc):
        assert svc._normalize_url("example.com/Page") == "example.com/page"


# ============================================================
# _get_url_hash
# ============================================================


class TestGetUrlHash:
    def test_returns_hash(self, svc):
        with patch(f"{MODULE}.get_url_hash", return_value="hash123") as m:
            result = svc._get_url_hash("https://www.example.com/page")
            m.assert_called_once_with("example.com/page")
            assert result == "hash123"

    def test_equivalent_urls_same_hash(self, svc):
        h1 = svc._get_url_hash("https://www.example.com/page")
        h2 = svc._get_url_hash("http://Example.COM/page")
        assert h1 == h2


# ============================================================
# close / context manager
# ============================================================


class TestClose:
    def test_closes_downloaders(self, svc):
        d1, d2 = MagicMock(), MagicMock()
        svc.downloaders = [d1, d2]
        svc.close()
        d1.close.assert_called_once()
        d2.close.assert_called_once()

    def test_clears_refs(self, svc):
        svc.close()
        assert svc.downloaders == []
        assert svc.retry_manager is None
        assert svc.settings is None

    def test_idempotent(self, svc):
        d1 = MagicMock()
        svc.downloaders = [d1]
        svc.close()
        svc.close()
        d1.close.assert_called_once()

    def test_handles_no_close_attr(self, svc):
        svc.downloaders = [object(), MagicMock()]
        svc.close()  # no error

    def test_handles_close_exception(self, svc):
        d = MagicMock()
        d.close.side_effect = RuntimeError("boom")
        svc.downloaders = [d]
        svc.close()  # swallowed


class TestContextManager:
    def test_enter_returns_self(self, svc):
        assert svc.__enter__() is svc

    def test_exit_calls_close(self, svc):
        with patch.object(svc, "close") as m:
            ret = svc.__exit__(None, None, None)
            m.assert_called_once()
            assert ret is False


# ============================================================
# _setup_directories
# ============================================================


class TestSetupDirectories:
    def test_creates_dirs(self, svc, tmp_path):
        svc.library_root = str(tmp_path / "lib")
        svc._setup_directories()
        assert (tmp_path / "lib").is_dir()
        assert (tmp_path / "lib" / "pdfs").is_dir()


# ============================================================
# is_already_downloaded
# ============================================================


class TestIsAlreadyDownloaded:
    def test_found_and_file_exists(self, svc, tmp_path):
        f = tmp_path / "test.pdf"
        f.write_bytes(b"pdf")
        tracker = MagicMock()
        tracker.file_path = str(f)

        mock_session = MagicMock()
        mock_session.query.return_value.filter_by.return_value.first.return_value = tracker

        with (
            patch(f"{MODULE}.get_user_db_session") as mock_ctx,
            patch(f"{MODULE}.get_absolute_path_from_settings", return_value=f),
        ):
            mock_ctx.return_value.__enter__ = MagicMock(
                return_value=mock_session
            )
            mock_ctx.return_value.__exit__ = MagicMock(return_value=False)
            result = svc.is_already_downloaded("https://example.com/paper.pdf")
            assert result == (True, str(f))

    def test_found_but_file_deleted(self, svc, tmp_path):
        tracker = MagicMock()
        tracker.file_path = "/nonexistent/path.pdf"
        abs_path = tmp_path / "gone.pdf"  # doesn't exist

        mock_session = MagicMock()
        mock_session.query.return_value.filter_by.return_value.first.return_value = tracker

        with (
            patch(f"{MODULE}.get_user_db_session") as mock_ctx,
            patch(
                f"{MODULE}.get_absolute_path_from_settings",
                return_value=abs_path,
            ),
        ):
            mock_ctx.return_value.__enter__ = MagicMock(
                return_value=mock_session
            )
            mock_ctx.return_value.__exit__ = MagicMock(return_value=False)
            result = svc.is_already_downloaded("https://example.com/paper.pdf")
            assert result == (False, None)
            # Should mark as not downloaded
            assert tracker.is_downloaded is False

    def test_found_but_path_blocked(self, svc):
        tracker = MagicMock()
        tracker.file_path = "/some/path"

        mock_session = MagicMock()
        mock_session.query.return_value.filter_by.return_value.first.return_value = tracker

        with (
            patch(f"{MODULE}.get_user_db_session") as mock_ctx,
            patch(
                f"{MODULE}.get_absolute_path_from_settings", return_value=None
            ),
        ):
            mock_ctx.return_value.__enter__ = MagicMock(
                return_value=mock_session
            )
            mock_ctx.return_value.__exit__ = MagicMock(return_value=False)
            result = svc.is_already_downloaded("https://example.com/paper.pdf")
            assert result == (False, None)

    def test_not_found(self, svc):
        mock_session = MagicMock()
        mock_session.query.return_value.filter_by.return_value.first.return_value = None

        with patch(f"{MODULE}.get_user_db_session") as mock_ctx:
            mock_ctx.return_value.__enter__ = MagicMock(
                return_value=mock_session
            )
            mock_ctx.return_value.__exit__ = MagicMock(return_value=False)
            result = svc.is_already_downloaded("https://example.com/paper.pdf")
            assert result == (False, None)


# ============================================================
# get_text_content
# ============================================================


class TestGetTextContent:
    def test_resource_not_found(self, svc):
        mock_session = MagicMock()
        mock_session.query.return_value.get.return_value = None

        with patch(f"{MODULE}.get_user_db_session") as mock_ctx:
            mock_ctx.return_value.__enter__ = MagicMock(
                return_value=mock_session
            )
            mock_ctx.return_value.__exit__ = MagicMock(return_value=False)
            assert svc.get_text_content(999) is None

    def test_downloader_returns_text(self, svc):
        resource = MagicMock()
        resource.url = "https://arxiv.org/abs/1234"
        resource.title = "Test Paper Title for Coverage"

        mock_session = MagicMock()
        mock_session.query.return_value.get.return_value = resource

        downloader = MagicMock()
        downloader.can_handle.return_value = True
        downloader.download_text.return_value = "Extracted text content"
        svc.downloaders = [downloader]

        with patch(f"{MODULE}.get_user_db_session") as mock_ctx:
            mock_ctx.return_value.__enter__ = MagicMock(
                return_value=mock_session
            )
            mock_ctx.return_value.__exit__ = MagicMock(return_value=False)
            result = svc.get_text_content(1)
            assert result == "Extracted text content"

    def test_downloader_returns_none(self, svc):
        resource = MagicMock()
        resource.url = "https://example.com/paper"

        mock_session = MagicMock()
        mock_session.query.return_value.get.return_value = resource

        downloader = MagicMock()
        downloader.can_handle.return_value = True
        downloader.download_text.return_value = None
        svc.downloaders = [downloader]

        with patch(f"{MODULE}.get_user_db_session") as mock_ctx:
            mock_ctx.return_value.__enter__ = MagicMock(
                return_value=mock_session
            )
            mock_ctx.return_value.__exit__ = MagicMock(return_value=False)
            assert svc.get_text_content(1) is None

    def test_downloader_raises_exception(self, svc):
        resource = MagicMock()
        resource.url = "https://example.com/paper"

        mock_session = MagicMock()
        mock_session.query.return_value.get.return_value = resource

        downloader = MagicMock()
        downloader.can_handle.return_value = True
        downloader.download_text.side_effect = RuntimeError("fail")
        svc.downloaders = [downloader]

        with patch(f"{MODULE}.get_user_db_session") as mock_ctx:
            mock_ctx.return_value.__enter__ = MagicMock(
                return_value=mock_session
            )
            mock_ctx.return_value.__exit__ = MagicMock(return_value=False)
            assert svc.get_text_content(1) is None

    def test_no_downloader_handles_url(self, svc):
        resource = MagicMock()
        resource.url = "https://weird.com/stuff"

        mock_session = MagicMock()
        mock_session.query.return_value.get.return_value = resource

        downloader = MagicMock()
        downloader.can_handle.return_value = False
        svc.downloaders = [downloader]

        with patch(f"{MODULE}.get_user_db_session") as mock_ctx:
            mock_ctx.return_value.__enter__ = MagicMock(
                return_value=mock_session
            )
            mock_ctx.return_value.__exit__ = MagicMock(return_value=False)
            assert svc.get_text_content(1) is None


# ============================================================
# _is_downloadable
# ============================================================


class TestIsDownloadable:
    def test_delegates_to_utility(self, svc):
        resource = MagicMock()
        resource.url = "https://example.com/paper.pdf"
        with patch(f"{MODULE}.is_downloadable_url", return_value=True) as m:
            assert svc._is_downloadable(resource) is True
            m.assert_called_once_with("https://example.com/paper.pdf")

    def test_not_downloadable(self, svc):
        resource = MagicMock()
        resource.url = "ftp://example.com"
        with patch(f"{MODULE}.is_downloadable_url", return_value=False):
            assert svc._is_downloadable(resource) is False


# ============================================================
# queue_research_downloads
# ============================================================


class TestQueueResearchDownloads:
    def _make_session_ctx(self, session):
        ctx = MagicMock()
        ctx.__enter__ = MagicMock(return_value=session)
        ctx.__exit__ = MagicMock(return_value=False)
        return ctx

    def test_no_resources(self, svc):
        mock_session = MagicMock()
        mock_session.query.return_value.filter_by.return_value.all.return_value = []

        with patch(
            f"{MODULE}.get_user_db_session",
            return_value=self._make_session_ctx(mock_session),
        ):
            # Pass collection_id to skip the local import of get_default_library_id
            result = svc.queue_research_downloads(
                "res-1", collection_id="lib-1"
            )
            assert result == 0

    def test_no_resources_default_collection(self, svc):
        """When no collection_id, it fetches default library id."""
        mock_session = MagicMock()
        mock_session.query.return_value.filter_by.return_value.all.return_value = []

        with (
            patch(
                f"{MODULE}.get_user_db_session",
                return_value=self._make_session_ctx(mock_session),
            ),
            patch(
                "local_deep_research.database.library_init.get_default_library_id",
                return_value="lib-1",
            ),
        ):
            result = svc.queue_research_downloads("res-1")
            assert result == 0

    def test_resource_already_has_document_id(self, svc):
        resource = MagicMock()
        resource.document_id = "doc-existing"
        resource.url = "https://example.com/paper.pdf"

        mock_session = MagicMock()
        mock_session.query.return_value.filter_by.return_value.all.return_value = [
            resource
        ]

        with (
            patch(
                f"{MODULE}.get_user_db_session",
                return_value=self._make_session_ctx(mock_session),
            ),
            patch(f"{MODULE}.is_downloadable_url", return_value=True),
        ):
            result = svc.queue_research_downloads(
                "res-1", collection_id="lib-1"
            )
            assert result == 0

    def test_queues_new_resource(self, svc):
        resource = MagicMock()
        resource.id = 1
        resource.document_id = None
        resource.url = "https://example.com/paper.pdf"

        mock_session = MagicMock()
        mock_session.query.return_value.filter_by.return_value.all.return_value = [
            resource
        ]
        # For the 3 filter_by calls (existing_queue, existing_doc, any_queue): all return None
        mock_session.query.return_value.filter_by.return_value.first.return_value = None

        with (
            patch(
                f"{MODULE}.get_user_db_session",
                return_value=self._make_session_ctx(mock_session),
            ),
            patch(f"{MODULE}.is_downloadable_url", return_value=True),
        ):
            result = svc.queue_research_downloads(
                "res-1", collection_id="lib-1"
            )
            assert result == 1
            mock_session.add.assert_called_once()

    def test_resets_existing_queue_entry(self, svc):
        resource = MagicMock()
        resource.id = 1
        resource.document_id = None
        resource.url = "https://example.com/paper.pdf"

        any_queue = MagicMock()

        mock_session = MagicMock()
        mock_session.query.return_value.filter_by.return_value.all.return_value = [
            resource
        ]
        # First two first() calls return None (existing_queue, existing_doc)
        # Third returns any_queue
        mock_session.query.return_value.filter_by.return_value.first.side_effect = [
            None,  # existing_queue
            None,  # existing_doc
            any_queue,  # any_queue
        ]

        with (
            patch(
                f"{MODULE}.get_user_db_session",
                return_value=self._make_session_ctx(mock_session),
            ),
            patch(f"{MODULE}.is_downloadable_url", return_value=True),
        ):
            result = svc.queue_research_downloads(
                "res-1", collection_id="col-5"
            )
            assert result == 1
            assert any_queue.collection_id == "col-5"

    def test_not_downloadable_skipped(self, svc):
        resource = MagicMock()
        resource.url = "not-a-url"
        resource.document_id = None

        mock_session = MagicMock()
        mock_session.query.return_value.filter_by.return_value.all.return_value = [
            resource
        ]

        with (
            patch(
                f"{MODULE}.get_user_db_session",
                return_value=self._make_session_ctx(mock_session),
            ),
            patch(f"{MODULE}.is_downloadable_url", return_value=False),
        ):
            result = svc.queue_research_downloads(
                "res-1", collection_id="lib-1"
            )
            assert result == 0


# ============================================================
# _get_downloader
# ============================================================


class TestGetDownloader:
    def test_returns_matching_downloader(self, svc):
        d1 = MagicMock()
        d1.can_handle.return_value = False
        d2 = MagicMock()
        d2.can_handle.return_value = True
        svc.downloaders = [d1, d2]
        assert svc._get_downloader("https://arxiv.org/abs/1234") is d2

    def test_returns_none_if_no_match(self, svc):
        d1 = MagicMock()
        d1.can_handle.return_value = False
        svc.downloaders = [d1]
        assert svc._get_downloader("https://weird.com") is None


# ============================================================
# _extract_text_from_pdf
# ============================================================


class TestExtractTextFromPdf:
    def test_pdfplumber_success(self, svc):
        mock_page = MagicMock()
        mock_page.extract_text.return_value = "Page 1 text"
        mock_pdf = MagicMock()
        mock_pdf.pages = [mock_page]
        mock_pdf.__enter__ = MagicMock(return_value=mock_pdf)
        mock_pdf.__exit__ = MagicMock(return_value=False)

        with patch(f"{MODULE}.pdfplumber") as mock_plumber:
            mock_plumber.open.return_value = mock_pdf
            result = svc._extract_text_from_pdf(b"fake pdf content")
            assert result == "Page 1 text"

    def test_pdfplumber_empty_falls_to_pypdf(self, svc):
        # pdfplumber returns empty
        mock_pdf_plumber = MagicMock()
        mock_pdf_plumber.pages = []
        mock_pdf_plumber.__enter__ = MagicMock(return_value=mock_pdf_plumber)
        mock_pdf_plumber.__exit__ = MagicMock(return_value=False)

        # pypdf succeeds
        mock_pypdf_page = MagicMock()
        mock_pypdf_page.extract_text.return_value = "PyPDF text"
        mock_reader = MagicMock()
        mock_reader.pages = [mock_pypdf_page]

        with (
            patch(f"{MODULE}.pdfplumber") as mock_plumber,
            patch(f"{MODULE}.PdfReader", return_value=mock_reader),
        ):
            mock_plumber.open.return_value = mock_pdf_plumber
            result = svc._extract_text_from_pdf(b"fake pdf")
            assert result == "PyPDF text"

    def test_both_extractors_fail(self, svc):
        mock_pdf_plumber = MagicMock()
        mock_pdf_plumber.pages = []
        mock_pdf_plumber.__enter__ = MagicMock(return_value=mock_pdf_plumber)
        mock_pdf_plumber.__exit__ = MagicMock(return_value=False)

        mock_reader = MagicMock()
        mock_reader.pages = []

        with (
            patch(f"{MODULE}.pdfplumber") as mock_plumber,
            patch(f"{MODULE}.PdfReader", return_value=mock_reader),
        ):
            mock_plumber.open.return_value = mock_pdf_plumber
            assert svc._extract_text_from_pdf(b"bad pdf") is None

    def test_exception_returns_none(self, svc):
        with patch(f"{MODULE}.pdfplumber") as mock_plumber:
            mock_plumber.open.side_effect = Exception("corrupt")
            assert svc._extract_text_from_pdf(b"corrupt") is None

    def test_pdfplumber_pages_with_none_text(self, svc):
        """Pages that return None text are skipped."""
        page1 = MagicMock()
        page1.extract_text.return_value = None
        page2 = MagicMock()
        page2.extract_text.return_value = "Text from page 2"
        mock_pdf = MagicMock()
        mock_pdf.pages = [page1, page2]
        mock_pdf.__enter__ = MagicMock(return_value=mock_pdf)
        mock_pdf.__exit__ = MagicMock(return_value=False)

        with patch(f"{MODULE}.pdfplumber") as mock_plumber:
            mock_plumber.open.return_value = mock_pdf
            result = svc._extract_text_from_pdf(b"pdf")
            assert result == "Text from page 2"


# ============================================================
# _download_generic
# ============================================================


class TestDownloadGeneric:
    def test_success_pdf_content_type(self, svc):
        mock_resp = MagicMock()
        mock_resp.headers = {"Content-Type": "application/pdf"}
        mock_resp.content = b"%PDF-1.4 fake content"
        mock_resp.raise_for_status = MagicMock()

        with patch(f"{MODULE}.safe_get", return_value=mock_resp):
            result = svc._download_generic("https://example.com/paper.pdf")
            assert result == b"%PDF-1.4 fake content"

    def test_success_pdf_magic_bytes(self, svc):
        mock_resp = MagicMock()
        mock_resp.headers = {"Content-Type": "application/octet-stream"}
        mock_resp.content = b"%PDF-1.4 content"
        mock_resp.raise_for_status = MagicMock()

        with patch(f"{MODULE}.safe_get", return_value=mock_resp):
            result = svc._download_generic("https://example.com/file")
            assert result == b"%PDF-1.4 content"

    def test_not_pdf_returns_none(self, svc):
        mock_resp = MagicMock()
        mock_resp.headers = {"Content-Type": "text/html"}
        mock_resp.content = b"<html>not pdf</html>"
        mock_resp.raise_for_status = MagicMock()

        with patch(f"{MODULE}.safe_get", return_value=mock_resp):
            assert svc._download_generic("https://example.com") is None

    def test_exception_returns_none(self, svc):
        with patch(f"{MODULE}.safe_get", side_effect=Exception("timeout")):
            assert svc._download_generic("https://example.com") is None


# ============================================================
# _download_arxiv
# ============================================================


class TestDownloadArxiv:
    def test_converts_abs_to_pdf(self, svc):
        with patch.object(
            svc, "_download_generic", return_value=b"pdf"
        ) as mock_dl:
            result = svc._download_arxiv("https://arxiv.org/abs/2301.12345")
            assert result == b"pdf"
            called_url = mock_dl.call_args[0][0]
            assert "pdf" in called_url
            assert called_url.endswith(".pdf")

    def test_already_pdf_url(self, svc):
        with patch.object(svc, "_download_generic", return_value=b"pdf"):
            result = svc._download_arxiv("https://arxiv.org/pdf/2301.12345.pdf")
            assert result == b"pdf"

    def test_exception_returns_none(self, svc):
        with patch.object(
            svc, "_download_generic", side_effect=Exception("err")
        ):
            assert svc._download_arxiv("https://arxiv.org/abs/123") is None


# ============================================================
# _download_semantic_scholar
# ============================================================


class TestDownloadSemanticScholar:
    def test_returns_none(self, svc):
        assert (
            svc._download_semantic_scholar(
                "https://semanticscholar.org/paper/123"
            )
            is None
        )


# ============================================================
# _download_biorxiv
# ============================================================


class TestDownloadBiorxiv:
    def test_converts_url_to_pdf(self, svc):
        with patch.object(svc, "_download_generic", return_value=b"pdf"):
            result = svc._download_biorxiv("https://biorxiv.org/content/123v1")
            assert result == b"pdf"

    def test_exception_returns_none(self, svc):
        with patch.object(
            svc, "_download_generic", side_effect=Exception("fail")
        ):
            assert svc._download_biorxiv("https://biorxiv.org/123") is None


# ============================================================
# _try_europe_pmc
# ============================================================


class TestTryEuropePmc:
    def test_success(self, svc):
        api_resp = MagicMock()
        api_resp.status_code = 200
        api_resp.json.return_value = {
            "resultList": {
                "result": [
                    {"isOpenAccess": "Y", "hasPDF": "Y", "pmcid": "PMC123456"}
                ]
            }
        }

        pdf_resp = MagicMock()
        pdf_resp.status_code = 200
        pdf_resp.headers = {"content-type": "application/pdf"}
        pdf_resp.content = b"%PDF content"

        with patch(f"{MODULE}.safe_get", side_effect=[api_resp, pdf_resp]):
            result = svc._try_europe_pmc("12345")
            assert result == b"%PDF content"

    def test_not_open_access(self, svc):
        api_resp = MagicMock()
        api_resp.status_code = 200
        api_resp.json.return_value = {
            "resultList": {
                "result": [
                    {
                        "isOpenAccess": "N",
                        "hasPDF": "Y",
                    }
                ]
            }
        }

        with patch(f"{MODULE}.safe_get", return_value=api_resp):
            assert svc._try_europe_pmc("12345") is None

    def test_no_results(self, svc):
        api_resp = MagicMock()
        api_resp.status_code = 200
        api_resp.json.return_value = {"resultList": {"result": []}}

        with patch(f"{MODULE}.safe_get", return_value=api_resp):
            assert svc._try_europe_pmc("12345") is None

    def test_api_failure(self, svc):
        api_resp = MagicMock()
        api_resp.status_code = 500

        with patch(f"{MODULE}.safe_get", return_value=api_resp):
            assert svc._try_europe_pmc("12345") is None

    def test_exception(self, svc):
        with patch(f"{MODULE}.safe_get", side_effect=Exception("network")):
            assert svc._try_europe_pmc("12345") is None

    def test_no_pmcid(self, svc):
        api_resp = MagicMock()
        api_resp.status_code = 200
        api_resp.json.return_value = {
            "resultList": {
                "result": [{"isOpenAccess": "Y", "hasPDF": "Y", "pmcid": None}]
            }
        }

        with patch(f"{MODULE}.safe_get", return_value=api_resp):
            assert svc._try_europe_pmc("12345") is None


# ============================================================
# _try_existing_text
# ============================================================


class TestTryExistingText:
    def test_no_document(self, svc):
        session = MagicMock()
        session.query.return_value.filter_by.return_value.first.return_value = (
            None
        )
        assert svc._try_existing_text(session, 1) is None

    def test_has_text_content(self, svc):
        doc = MagicMock()
        doc.text_content = "Some text"
        doc.extraction_method = "pdf_extraction"
        session = MagicMock()
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )
        assert svc._try_existing_text(session, 1) == (True, None)

    def test_failed_extraction(self, svc):
        doc = MagicMock()
        doc.text_content = "Some text"
        doc.extraction_method = "failed"
        session = MagicMock()
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )
        assert svc._try_existing_text(session, 1) is None

    def test_no_text_content(self, svc):
        doc = MagicMock()
        doc.text_content = None
        doc.extraction_method = "pdf_extraction"
        session = MagicMock()
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )
        assert svc._try_existing_text(session, 1) is None

    def test_no_extraction_method(self, svc):
        doc = MagicMock()
        doc.text_content = "text"
        doc.extraction_method = None
        session = MagicMock()
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )
        assert svc._try_existing_text(session, 1) is None


# ============================================================
# _try_legacy_text_file
# ============================================================


class TestTryLegacyTextFile:
    def test_no_txt_dir(self, svc, tmp_path):
        svc.library_root = str(tmp_path)
        session = MagicMock()
        resource = MagicMock()
        assert svc._try_legacy_text_file(session, resource, 1) is None

    def test_no_matching_files(self, svc, tmp_path):
        svc.library_root = str(tmp_path)
        (tmp_path / "txt").mkdir()
        session = MagicMock()
        resource = MagicMock()
        assert svc._try_legacy_text_file(session, resource, 1) is None

    def test_found_legacy_file(self, svc, tmp_path):
        svc.library_root = str(tmp_path)
        txt_dir = tmp_path / "txt"
        txt_dir.mkdir()
        (txt_dir / "paper_42.txt").write_text("Legacy text")

        session = MagicMock()
        resource = MagicMock()
        with patch.object(svc, "_create_text_document_record") as m:
            result = svc._try_legacy_text_file(session, resource, 42)
            assert result == (True, None)
            m.assert_called_once()


# ============================================================
# _try_existing_pdf_extraction
# ============================================================


class TestTryExistingPdfExtraction:
    def test_no_pdf_document(self, svc):
        session = MagicMock()
        session.query.return_value.filter_by.return_value.first.return_value = (
            None
        )
        resource = MagicMock()
        assert svc._try_existing_pdf_extraction(session, resource, 1) is None

    def test_not_completed(self, svc):
        doc = MagicMock()
        doc.status = "pending"
        session = MagicMock()
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )
        resource = MagicMock()
        assert svc._try_existing_pdf_extraction(session, resource, 1) is None

    def test_no_file_path(self, svc):
        doc = MagicMock()
        doc.status = "completed"
        doc.file_path = None
        session = MagicMock()
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )
        resource = MagicMock()
        assert svc._try_existing_pdf_extraction(session, resource, 1) is None

    def test_sentinel_file_path(self, svc):
        doc = MagicMock()
        doc.status = "completed"
        doc.file_path = "text_only_not_stored"
        session = MagicMock()
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )
        resource = MagicMock()
        with patch(f"{MODULE}.FILE_PATH_SENTINELS", ("text_only_not_stored",)):
            assert (
                svc._try_existing_pdf_extraction(session, resource, 1) is None
            )

    def test_path_traversal_blocked(self, svc):
        doc = MagicMock()
        doc.status = "completed"
        doc.file_path = "../../etc/passwd"
        session = MagicMock()
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )
        resource = MagicMock()

        with (
            patch(f"{MODULE}.FILE_PATH_SENTINELS", ()),
            patch(f"{MODULE}.PathValidator") as pv,
        ):
            pv.validate_safe_path.side_effect = ValueError("blocked")
            assert (
                svc._try_existing_pdf_extraction(session, resource, 1) is None
            )

    def test_file_not_found(self, svc, tmp_path):
        doc = MagicMock()
        doc.status = "completed"
        doc.file_path = str(tmp_path / "missing.pdf")
        session = MagicMock()
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )
        resource = MagicMock()

        with (
            patch(f"{MODULE}.FILE_PATH_SENTINELS", ()),
            patch(f"{MODULE}.PathValidator") as pv,
        ):
            pv.validate_safe_path.return_value = str(tmp_path / "missing.pdf")
            assert (
                svc._try_existing_pdf_extraction(session, resource, 1) is None
            )

    def test_successful_extraction(self, svc, tmp_path):
        pdf_file = tmp_path / "paper.pdf"
        pdf_file.write_bytes(b"fake pdf")

        doc = MagicMock()
        doc.status = "completed"
        doc.file_path = str(pdf_file)
        doc.id = "doc-1"
        session = MagicMock()
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )
        resource = MagicMock()

        with (
            patch(f"{MODULE}.FILE_PATH_SENTINELS", ()),
            patch(f"{MODULE}.PathValidator") as pv,
            patch.object(
                svc, "_extract_text_from_pdf", return_value="extracted text"
            ),
            patch.object(svc, "_save_text_with_db"),
        ):
            pv.validate_safe_path.return_value = str(pdf_file)
            result = svc._try_existing_pdf_extraction(session, resource, 1)
            assert result == (True, None)

    def test_extraction_returns_none(self, svc, tmp_path):
        pdf_file = tmp_path / "paper.pdf"
        pdf_file.write_bytes(b"bad pdf")

        doc = MagicMock()
        doc.status = "completed"
        doc.file_path = str(pdf_file)
        session = MagicMock()
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )
        resource = MagicMock()

        with (
            patch(f"{MODULE}.FILE_PATH_SENTINELS", ()),
            patch(f"{MODULE}.PathValidator") as pv,
            patch.object(svc, "_extract_text_from_pdf", return_value=None),
        ):
            pv.validate_safe_path.return_value = str(pdf_file)
            assert (
                svc._try_existing_pdf_extraction(session, resource, 1) is None
            )

    def test_read_exception_falls_through(self, svc, tmp_path):
        pdf_file = tmp_path / "paper.pdf"
        pdf_file.write_bytes(b"pdf")

        doc = MagicMock()
        doc.status = "completed"
        doc.file_path = str(pdf_file)
        session = MagicMock()
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )
        resource = MagicMock()

        with (
            patch(f"{MODULE}.FILE_PATH_SENTINELS", ()),
            patch(f"{MODULE}.PathValidator") as pv,
            patch("builtins.open", side_effect=IOError("disk error")),
        ):
            pv.validate_safe_path.return_value = str(pdf_file)
            assert (
                svc._try_existing_pdf_extraction(session, resource, 1) is None
            )


# ============================================================
# _try_api_text_extraction
# ============================================================


class TestTryApiTextExtraction:
    def test_no_downloader(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.url = "https://weird.com"
        with patch.object(svc, "_get_downloader", return_value=None):
            assert svc._try_api_text_extraction(session, resource) is None

    def test_download_fails(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.url = "https://example.com"

        downloader = MagicMock()
        result_obj = MagicMock()
        result_obj.is_success = False
        result_obj.content = None
        downloader.download_with_result.return_value = result_obj

        with patch.object(svc, "_get_downloader", return_value=downloader):
            assert svc._try_api_text_extraction(session, resource) is None

    def test_success_with_bytes(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.url = "https://example.com"
        resource.id = 1
        resource.title = "Test Paper with Long Title for Slicing"

        downloader = MagicMock()
        result_obj = MagicMock()
        result_obj.is_success = True
        result_obj.content = b"Text content from API"
        downloader.download_with_result.return_value = result_obj

        with (
            patch.object(svc, "_get_downloader", return_value=downloader),
            patch.object(svc, "_save_text_with_db"),
        ):
            result = svc._try_api_text_extraction(session, resource)
            assert result == (True, None)

    def test_success_with_string(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.url = "https://example.com"
        resource.title = "Test Paper with Long Title for Slicing"

        downloader = MagicMock()
        result_obj = MagicMock()
        result_obj.is_success = True
        result_obj.content = "Already a string"
        downloader.download_with_result.return_value = result_obj

        with (
            patch.object(svc, "_get_downloader", return_value=downloader),
            patch.object(svc, "_save_text_with_db"),
        ):
            result = svc._try_api_text_extraction(session, resource)
            assert result == (True, None)

    def test_arxiv_source_detected(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.url = "https://arxiv.org/abs/1234"
        resource.title = "Test Paper with Long Title for Slicing"

        from local_deep_research.research_library.downloaders import (
            ArxivDownloader,
        )

        downloader = MagicMock(spec=ArxivDownloader)
        result_obj = MagicMock()
        result_obj.is_success = True
        result_obj.content = "text"
        downloader.download_with_result.return_value = result_obj

        with (
            patch.object(svc, "_get_downloader", return_value=downloader),
            patch.object(svc, "_save_text_with_db") as save_mock,
        ):
            svc._try_api_text_extraction(session, resource)
            # Check extraction_source was arxiv_api
            save_mock.assert_called_once()
            kwargs = save_mock.call_args
            assert kwargs[1].get("extraction_source") == "arxiv_api" or (
                len(kwargs[0]) > 4 and kwargs[0][4] == "arxiv_api"
            )

    def test_save_raises_returns_error(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.url = "https://example.com"
        resource.id = 1
        resource.title = "Test Paper with Long Title for Slicing"

        downloader = MagicMock()
        result_obj = MagicMock()
        result_obj.is_success = True
        result_obj.content = "text"
        downloader.download_with_result.return_value = result_obj

        with (
            patch.object(svc, "_get_downloader", return_value=downloader),
            patch.object(
                svc, "_save_text_with_db", side_effect=RuntimeError("db error")
            ),
            patch(f"{MODULE}.sanitize_for_log", return_value="safe error"),
        ):
            result = svc._try_api_text_extraction(session, resource)
            assert result[0] is False
            assert "safe error" in result[1]


# ============================================================
# _fallback_pdf_extraction
# ============================================================


class TestFallbackPdfExtraction:
    def test_no_downloader(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.url = "https://example.com"
        resource.title = "Test Paper with Long Title for Slicing"

        with (
            patch.object(svc, "_get_downloader", return_value=None),
            patch.object(svc, "_record_failed_text_extraction"),
        ):
            success, msg = svc._fallback_pdf_extraction(session, resource)
            assert success is False
            assert "No compatible downloader" in msg

    def test_download_fails(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.url = "https://example.com"
        resource.title = "Test Paper with Long Title for Slicing"

        downloader = MagicMock()
        result_obj = MagicMock()
        result_obj.is_success = False
        result_obj.content = None
        result_obj.skip_reason = "Rate limited"
        downloader.download_with_result.return_value = result_obj

        with (
            patch.object(svc, "_get_downloader", return_value=downloader),
            patch.object(svc, "_record_failed_text_extraction"),
        ):
            success, msg = svc._fallback_pdf_extraction(session, resource)
            assert success is False

    def test_text_extraction_fails(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.url = "https://example.com"
        resource.title = "Test Paper with Long Title for Slicing"

        downloader = MagicMock()
        result_obj = MagicMock()
        result_obj.is_success = True
        result_obj.content = b"pdf bytes"
        downloader.download_with_result.return_value = result_obj

        with (
            patch.object(svc, "_get_downloader", return_value=downloader),
            patch.object(svc, "_extract_text_from_pdf", return_value=None),
            patch.object(svc, "_record_failed_text_extraction"),
        ):
            success, msg = svc._fallback_pdf_extraction(session, resource)
            assert success is False

    def test_success(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.url = "https://example.com"
        resource.title = "Test Paper with Long Title for Slicing"

        downloader = MagicMock()
        result_obj = MagicMock()
        result_obj.is_success = True
        result_obj.content = b"pdf bytes"
        downloader.download_with_result.return_value = result_obj

        with (
            patch.object(svc, "_get_downloader", return_value=downloader),
            patch.object(
                svc, "_extract_text_from_pdf", return_value="Extracted!"
            ),
            patch.object(svc, "_save_text_with_db"),
        ):
            success, msg = svc._fallback_pdf_extraction(session, resource)
            assert success is True
            assert msg is None

    def test_save_exception(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.url = "https://example.com"
        resource.id = 1
        resource.title = "Test Paper with Long Title for Slicing"

        downloader = MagicMock()
        result_obj = MagicMock()
        result_obj.is_success = True
        result_obj.content = b"pdf bytes"
        downloader.download_with_result.return_value = result_obj

        with (
            patch.object(svc, "_get_downloader", return_value=downloader),
            patch.object(svc, "_extract_text_from_pdf", return_value="text"),
            patch.object(
                svc, "_save_text_with_db", side_effect=RuntimeError("db")
            ),
            patch(f"{MODULE}.sanitize_for_log", return_value="safe"),
        ):
            success, msg = svc._fallback_pdf_extraction(session, resource)
            assert success is False

    def test_download_fails_no_skip_reason(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.url = "https://example.com"
        resource.title = "Test Paper with Long Title for Slicing"

        downloader = MagicMock()
        result_obj = MagicMock()
        result_obj.is_success = False
        result_obj.content = None
        result_obj.skip_reason = None
        downloader.download_with_result.return_value = result_obj

        with (
            patch.object(svc, "_get_downloader", return_value=downloader),
            patch.object(svc, "_record_failed_text_extraction"),
        ):
            success, msg = svc._fallback_pdf_extraction(session, resource)
            assert success is False
            assert "Failed to download PDF" in msg


# ============================================================
# _save_text_with_db
# ============================================================


class TestSaveTextWithDb:
    def test_update_existing_doc_with_pdf_document_id(self, svc):
        session = MagicMock()
        doc = MagicMock()
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )
        resource = MagicMock()
        resource.url = "https://example.com"

        svc._save_text_with_db(
            resource,
            "Test text content",
            session,
            extraction_method="native_api",
            extraction_source="arxiv_api",
            pdf_document_id="doc-123",
        )
        assert doc.text_content == "Test text content"
        assert doc.extraction_quality == "high"

    def test_update_existing_doc_pdf_extraction(self, svc):
        session = MagicMock()
        doc = MagicMock()
        resource = MagicMock()

        with patch(f"{MODULE}.get_document_for_resource", return_value=doc):
            svc._save_text_with_db(
                resource,
                "Text",
                session,
                extraction_method="pdf_extraction",
                extraction_source="pdfplumber",
            )
            assert doc.extraction_quality == "medium"

    def test_update_existing_doc_low_quality(self, svc):
        session = MagicMock()
        doc = MagicMock()
        resource = MagicMock()

        with patch(f"{MODULE}.get_document_for_resource", return_value=doc):
            svc._save_text_with_db(
                resource,
                "Text",
                session,
                extraction_method="other_method",
                extraction_source="unknown",
            )
            assert doc.extraction_quality == "low"

    def test_create_new_doc_with_library_collection(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.id = 1
        resource.research_id = "res-1"
        resource.url = "https://example.com"
        resource.title = "Title"

        library_col = MagicMock()
        library_col.id = "lib-col-1"

        # No existing doc
        with (
            patch(f"{MODULE}.get_document_for_resource", return_value=None),
            patch(f"{MODULE}.get_source_type_id", return_value="src-1"),
            patch(f"{MODULE}.uuid.uuid4", return_value="new-uuid"),
        ):
            # session.query(Collection).filter_by(name="Library").first() -> library_col
            session.query.return_value.filter_by.return_value.first.return_value = library_col
            svc._save_text_with_db(
                resource,
                "New text",
                session,
                extraction_method="native_api",
                extraction_source="arxiv_api",
            )
            assert session.add.call_count == 2  # doc + doc_collection

    def test_create_new_doc_no_library_collection(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.id = 1
        resource.research_id = "res-1"
        resource.url = "https://example.com"
        resource.title = "Title"

        with (
            patch(f"{MODULE}.get_document_for_resource", return_value=None),
            patch(f"{MODULE}.get_source_type_id", return_value="src-1"),
            patch(f"{MODULE}.uuid.uuid4", return_value="new-uuid"),
        ):
            session.query.return_value.filter_by.return_value.first.return_value = None
            svc._save_text_with_db(
                resource,
                "New text",
                session,
                extraction_method="pdf_extraction",
                extraction_source="pdfplumber",
            )
            # Only doc added, no collection link
            assert session.add.call_count == 1

    def test_get_source_type_raises(self, svc):
        session = MagicMock()
        resource = MagicMock()

        with (
            patch(f"{MODULE}.get_document_for_resource", return_value=None),
            patch(
                f"{MODULE}.get_source_type_id", side_effect=RuntimeError("db")
            ),
        ):
            with pytest.raises(RuntimeError):
                svc._save_text_with_db(
                    resource,
                    "text",
                    session,
                    extraction_method="native_api",
                    extraction_source="api",
                )


# ============================================================
# _create_text_document_record
# ============================================================


class TestCreateTextDocumentRecord:
    def test_updates_existing_doc(self, svc, tmp_path):
        f = tmp_path / "text.txt"
        f.write_text("Some legacy text content")

        session = MagicMock()
        resource = MagicMock()
        doc = MagicMock()

        with patch(f"{MODULE}.get_document_for_resource", return_value=doc):
            svc._create_text_document_record(
                session,
                resource,
                f,
                extraction_method="unknown",
                extraction_source="legacy_file",
            )
            assert doc.text_content == "Some legacy text content"
            assert doc.extraction_quality == "low"

    def test_no_doc_found(self, svc, tmp_path):
        f = tmp_path / "text.txt"
        f.write_text("text")

        session = MagicMock()
        resource = MagicMock()
        resource.id = 1

        with patch(f"{MODULE}.get_document_for_resource", return_value=None):
            # Should not raise, just log warning
            svc._create_text_document_record(
                session,
                resource,
                f,
                extraction_method="unknown",
                extraction_source="legacy_file",
            )

    def test_read_exception(self, svc, tmp_path):
        f = tmp_path / "text.txt"
        # Don't create the file - read will fail
        session = MagicMock()
        resource = MagicMock()
        # Should not raise
        svc._create_text_document_record(
            session,
            resource,
            f,
            extraction_method="unknown",
            extraction_source="legacy",
        )


# ============================================================
# _record_failed_text_extraction
# ============================================================


class TestRecordFailedTextExtraction:
    def test_updates_existing_doc(self, svc):
        session = MagicMock()
        resource = MagicMock()
        doc = MagicMock()

        with patch(f"{MODULE}.get_document_for_resource", return_value=doc):
            svc._record_failed_text_extraction(session, resource, "Some error")
            assert doc.error_message == "Some error"
            assert doc.extraction_method == "failed"

    def test_creates_new_failed_doc(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.id = 1
        resource.research_id = "res-1"
        resource.url = "https://example.com"
        resource.title = "Title"

        with (
            patch(f"{MODULE}.get_document_for_resource", return_value=None),
            patch(f"{MODULE}.get_source_type_id", return_value="src-1"),
        ):
            svc._record_failed_text_extraction(
                session, resource, "Download failed"
            )
            session.add.assert_called_once()

    def test_exception_swallowed(self, svc):
        session = MagicMock()
        resource = MagicMock()

        with patch(
            f"{MODULE}.get_document_for_resource",
            side_effect=RuntimeError("db"),
        ):
            # Should not raise
            svc._record_failed_text_extraction(session, resource, "error")


# ============================================================
# _try_library_text_extraction
# ============================================================


class TestTryLibraryTextExtraction:
    def test_no_doc_id_in_metadata_or_url(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.resource_metadata = {}
        resource.url = None
        result = svc._try_library_text_extraction(session, resource)
        assert result == (False, "Could not extract library document ID")

    def test_doc_id_from_metadata(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.resource_metadata = {
            "original_data": {"metadata": {"source_id": "doc-uuid-1"}}
        }
        resource.url = None

        doc = MagicMock()
        doc.text_content = "existing text"
        doc.extraction_method = "pdf_extraction"
        doc.id = "doc-uuid-1"
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )

        result = svc._try_library_text_extraction(session, resource)
        assert result == (True, None)
        assert resource.document_id == "doc-uuid-1"

    def test_doc_id_from_url(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.resource_metadata = {}
        resource.url = "/library/document/abc-123/pdf"

        doc = MagicMock()
        doc.text_content = "text"
        doc.extraction_method = "native_api"
        doc.id = "abc-123"
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )

        result = svc._try_library_text_extraction(session, resource)
        assert result == (True, None)

    def test_doc_not_found_in_db(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.resource_metadata = {}
        resource.url = "/library/document/missing-id"
        session.query.return_value.filter_by.return_value.first.return_value = (
            None
        )

        result = svc._try_library_text_extraction(session, resource)
        assert result[0] is False
        assert "not found" in result[1]

    def test_doc_has_failed_extraction_tries_pdf(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.resource_metadata = {}
        resource.url = "/library/document/doc-1"

        doc = MagicMock()
        doc.text_content = "text"
        doc.extraction_method = "failed"
        doc.id = "doc-1"
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )

        # PDF extraction path
        with patch(f"{MODULE}.PDFStorageManager") as mock_psm:
            mock_psm_instance = MagicMock()
            mock_psm.return_value = mock_psm_instance
            mock_psm_instance.load_pdf.return_value = b"pdf bytes"

            with patch.object(
                svc, "_extract_text_from_pdf", return_value="extracted"
            ):
                result = svc._try_library_text_extraction(session, resource)
                assert result == (True, None)
                assert doc.text_content == "extracted"

    def test_no_pdf_content(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.resource_metadata = {}
        resource.url = "/library/document/doc-1"

        doc = MagicMock()
        doc.text_content = None
        doc.extraction_method = None
        doc.id = "doc-1"
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )

        with patch(f"{MODULE}.PDFStorageManager") as mock_psm:
            mock_psm.return_value.load_pdf.return_value = None
            result = svc._try_library_text_extraction(session, resource)
            assert result[0] is False
            assert "no text or PDF" in result[1]

    def test_pdf_extraction_fails(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.resource_metadata = {}
        resource.url = "/library/document/doc-1"

        doc = MagicMock()
        doc.text_content = None
        doc.extraction_method = None
        doc.id = "doc-1"
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )

        with patch(f"{MODULE}.PDFStorageManager") as mock_psm:
            mock_psm.return_value.load_pdf.return_value = b"pdf"
            with patch.object(svc, "_extract_text_from_pdf", return_value=None):
                result = svc._try_library_text_extraction(session, resource)
                assert result[0] is False
                assert "Failed to extract" in result[1]

    def test_doc_id_from_document_id_key(self, svc):
        """Test fallback to document_id key in metadata."""
        session = MagicMock()
        resource = MagicMock()
        resource.resource_metadata = {
            "original_data": {
                "metadata": {"document_id": "doc-from-document-id"}
            }
        }
        resource.url = None

        doc = MagicMock()
        doc.text_content = "text"
        doc.extraction_method = "good"
        doc.id = "doc-from-document-id"
        session.query.return_value.filter_by.return_value.first.return_value = (
            doc
        )

        result = svc._try_library_text_extraction(session, resource)
        assert result == (True, None)


# ============================================================
# download_as_text
# ============================================================


class TestDownloadAsText:
    def _make_session_ctx(self, session):
        ctx = MagicMock()
        ctx.__enter__ = MagicMock(return_value=session)
        ctx.__exit__ = MagicMock(return_value=False)
        return ctx

    def test_resource_not_found(self, svc):
        session = MagicMock()
        session.query.return_value.filter_by.return_value.first.return_value = (
            None
        )

        with patch(
            f"{MODULE}.get_user_db_session",
            return_value=self._make_session_ctx(session),
        ):
            result = svc.download_as_text(999)
            assert result == (False, "Resource not found")

    def test_library_resource_by_source_type(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.source_type = "library"
        resource.url = "https://example.com"
        session.query.return_value.filter_by.return_value.first.return_value = (
            resource
        )

        with (
            patch(
                f"{MODULE}.get_user_db_session",
                return_value=self._make_session_ctx(session),
            ),
            patch.object(
                svc, "_try_library_text_extraction", return_value=(True, None)
            ),
        ):
            result = svc.download_as_text(1)
            assert result == (True, None)

    def test_library_resource_by_url(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.source_type = "web"
        resource.url = "/library/document/abc-123"
        session.query.return_value.filter_by.return_value.first.return_value = (
            resource
        )

        with (
            patch(
                f"{MODULE}.get_user_db_session",
                return_value=self._make_session_ctx(session),
            ),
            patch.object(
                svc, "_try_library_text_extraction", return_value=(True, None)
            ),
        ):
            result = svc.download_as_text(1)
            assert result == (True, None)

    def test_existing_text_found(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.source_type = "web"
        resource.url = "https://example.com/paper"
        session.query.return_value.filter_by.return_value.first.return_value = (
            resource
        )

        with (
            patch(
                f"{MODULE}.get_user_db_session",
                return_value=self._make_session_ctx(session),
            ),
            patch.object(svc, "_try_existing_text", return_value=(True, None)),
        ):
            result = svc.download_as_text(1)
            assert result == (True, None)

    def test_falls_through_to_fallback(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.source_type = "web"
        resource.url = "https://example.com/paper"
        session.query.return_value.filter_by.return_value.first.return_value = (
            resource
        )

        with (
            patch(
                f"{MODULE}.get_user_db_session",
                return_value=self._make_session_ctx(session),
            ),
            patch.object(svc, "_try_existing_text", return_value=None),
            patch.object(svc, "_try_legacy_text_file", return_value=None),
            patch.object(
                svc, "_try_existing_pdf_extraction", return_value=None
            ),
            patch.object(svc, "_try_api_text_extraction", return_value=None),
            patch.object(
                svc, "_fallback_pdf_extraction", return_value=(False, "fail")
            ),
        ):
            result = svc.download_as_text(1)
            assert result == (False, "fail")


# ============================================================
# download_resource
# ============================================================


class TestDownloadResource:
    def _make_session_ctx(self, session):
        ctx = MagicMock()
        ctx.__enter__ = MagicMock(return_value=session)
        ctx.__exit__ = MagicMock(return_value=False)
        return ctx

    def test_resource_not_found(self, svc):
        session = MagicMock()
        session.query.return_value.get.return_value = None

        with patch(
            f"{MODULE}.get_user_db_session",
            return_value=self._make_session_ctx(session),
        ):
            success, reason = svc.download_resource(999)
            assert success is False
            assert reason == "Resource not found"

    def test_already_downloaded(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.id = 1
        session.query.return_value.get.return_value = resource

        existing_doc = MagicMock()
        # First filter_by for existing doc
        session.query.return_value.filter_by.return_value.first.return_value = (
            existing_doc
        )

        with patch(
            f"{MODULE}.get_user_db_session",
            return_value=self._make_session_ctx(session),
        ):
            success, reason = svc.download_resource(1)
            assert success is True
            assert reason is None

    def test_download_success_triggers_auto_index(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.id = 1
        resource.url = "https://example.com/paper.pdf"
        session.query.return_value.get.return_value = resource

        # Return None for existing_doc check, then various other queries
        call_count = [0]

        def filter_by_side_effect(**kwargs):
            call_count[0] += 1
            result = MagicMock()
            if call_count[0] == 1:
                # existing_doc query (resource_id + COMPLETED) -> None
                result.first.return_value = None
            elif call_count[0] == 2:
                # queue_entry query -> None
                result.first.return_value = None
            elif call_count[0] == 3:
                # tracker query -> None
                result.first.return_value = None
            else:
                result.first.return_value = None
                result.order_by.return_value.first.return_value = None
            return result

        session.query.return_value.filter_by.side_effect = filter_by_side_effect

        with (
            patch(
                f"{MODULE}.get_user_db_session",
                return_value=self._make_session_ctx(session),
            ),
            patch.object(svc, "_download_pdf", return_value=(True, None, None)),
        ):
            # retry_manager is already mocked
            success, reason = svc.download_resource(1)
            assert success is True

    def test_download_failure_with_skip_reason(self, svc):
        session = MagicMock()
        resource = MagicMock()
        resource.id = 1
        resource.url = "https://example.com/paper.pdf"
        session.query.return_value.get.return_value = resource

        call_count = [0]

        def filter_by_side_effect(**kwargs):
            call_count[0] += 1
            result = MagicMock()
            result.first.return_value = None
            result.order_by.return_value.first.return_value = None
            return result

        session.query.return_value.filter_by.side_effect = filter_by_side_effect

        with (
            patch(
                f"{MODULE}.get_user_db_session",
                return_value=self._make_session_ctx(session),
            ),
            patch.object(
                svc, "_download_pdf", return_value=(False, "Rate limited", None)
            ),
        ):
            success, reason = svc.download_resource(1)
            assert success is False
            assert reason == "Rate limited"


# ============================================================
# _download_pdf
# ============================================================


class TestDownloadPdf:
    def test_no_downloader_handles_url(self, svc):
        resource = MagicMock()
        resource.url = "https://weird.com/something"
        tracker = MagicMock()
        tracker.url_hash = "hash1"
        tracker.download_attempts = MagicMock()
        tracker.download_attempts.count.return_value = 0
        session = MagicMock()

        d = MagicMock()
        d.can_handle.return_value = False
        svc.downloaders = [d]

        success, reason, _status_code = svc._download_pdf(
            resource, tracker, session
        )
        assert success is False
        assert "No compatible downloader" in reason

    def test_download_succeeds_creates_new_doc(self, svc):
        resource = MagicMock()
        resource.url = "https://example.com/paper.pdf"
        resource.id = 1
        resource.title = "Test Paper Title for Testing"
        resource.research_id = "res-1"

        tracker = MagicMock()
        tracker.url_hash = "hash1"
        tracker.download_attempts = MagicMock()
        tracker.download_attempts.count.return_value = 0

        session = MagicMock()

        result_obj = MagicMock()
        result_obj.is_success = True
        result_obj.content = b"%PDF-1.4 fake pdf content"
        result_obj.skip_reason = None

        downloader = MagicMock()
        downloader.can_handle.return_value = True
        downloader.download_with_result.return_value = result_obj
        downloader.__class__.__name__ = "DirectPDFDownloader"
        svc.downloaders = [downloader]

        svc.settings.get_setting.side_effect = lambda key, default=None: {
            "research_library.pdf_storage_mode": "filesystem",
            "research_library.max_pdf_size_mb": 100,
        }.get(key, default)

        mock_psm = MagicMock()
        mock_psm.save_pdf.return_value = ("/tmp/test.pdf", None)

        with (
            patch(
                f"{MODULE}.get_document_for_resource",
                side_effect=[None, MagicMock()],
            ),
            patch(f"{MODULE}.get_source_type_id", return_value="src-1"),
            patch(f"{MODULE}.get_default_library_id", return_value="lib-1"),
            patch(f"{MODULE}.PDFStorageManager", return_value=mock_psm),
            patch(f"{MODULE}.uuid.uuid4", return_value="new-doc-id"),
            patch.object(svc, "_extract_text_from_pdf", return_value="text"),
            patch.object(svc, "_save_text_with_db"),
        ):
            success, reason, _status_code = svc._download_pdf(
                resource, tracker, session
            )
            assert success is True
            assert reason is None

    def test_download_succeeds_updates_existing_doc(self, svc):
        resource = MagicMock()
        resource.url = "https://example.com/paper.pdf"
        resource.id = 1
        resource.title = "Test Paper"
        resource.research_id = "res-1"

        tracker = MagicMock()
        tracker.url_hash = "hash1"
        tracker.download_attempts = MagicMock()
        tracker.download_attempts.count.return_value = 0

        session = MagicMock()

        result_obj = MagicMock()
        result_obj.is_success = True
        result_obj.content = b"%PDF-1.4 fake pdf"
        result_obj.skip_reason = None

        downloader = MagicMock()
        downloader.can_handle.return_value = True
        downloader.download_with_result.return_value = result_obj
        downloader.__class__.__name__ = "DirectPDFDownloader"
        svc.downloaders = [downloader]

        svc.settings.get_setting.side_effect = lambda key, default=None: {
            "research_library.pdf_storage_mode": "database",
            "research_library.max_pdf_size_mb": 100,
        }.get(key, default)

        existing_doc = MagicMock()
        existing_doc.id = "existing-doc"
        mock_psm = MagicMock()
        mock_psm.save_pdf.return_value = ("database", None)

        with (
            patch(
                f"{MODULE}.get_document_for_resource",
                side_effect=[existing_doc, MagicMock()],
            ),
            patch(f"{MODULE}.PDFStorageManager", return_value=mock_psm),
            patch.object(svc, "_extract_text_from_pdf", return_value="text"),
            patch.object(svc, "_save_text_with_db"),
        ):
            success, reason, _status_code = svc._download_pdf(
                resource, tracker, session
            )
            assert success is True

    def test_download_exception(self, svc):
        resource = MagicMock()
        resource.url = "https://example.com/paper.pdf"
        resource.id = 1

        tracker = MagicMock()
        tracker.url_hash = "hash1"
        tracker.download_attempts = MagicMock()
        tracker.download_attempts.count.return_value = 0

        session = MagicMock()

        downloader = MagicMock()
        downloader.can_handle.return_value = True
        downloader.download_with_result.side_effect = RuntimeError(
            "Network error"
        )
        downloader.__class__.__name__ = "DirectPDFDownloader"
        svc.downloaders = [downloader]

        with patch(f"{MODULE}.sanitize_for_log", return_value="safe error"):
            success, reason, _status_code = svc._download_pdf(
                resource, tracker, session
            )
            assert success is False
            assert reason == "safe error"
            assert tracker.is_accessible is False

    def test_text_extraction_failure_doesnt_fail_download(self, svc):
        """Text extraction failure after successful PDF download should not fail the download."""
        resource = MagicMock()
        resource.url = "https://example.com/paper.pdf"
        resource.id = 1
        resource.title = "Test Paper Title for Testing"
        resource.research_id = "res-1"

        tracker = MagicMock()
        tracker.url_hash = "hash1"
        tracker.download_attempts = MagicMock()
        tracker.download_attempts.count.return_value = 0

        session = MagicMock()

        result_obj = MagicMock()
        result_obj.is_success = True
        result_obj.content = b"%PDF-1.4 fake pdf"
        result_obj.skip_reason = None

        downloader = MagicMock()
        downloader.can_handle.return_value = True
        downloader.download_with_result.return_value = result_obj
        downloader.__class__.__name__ = "DirectPDFDownloader"
        svc.downloaders = [downloader]

        svc.settings.get_setting.side_effect = lambda key, default=None: {
            "research_library.pdf_storage_mode": "none",
            "research_library.max_pdf_size_mb": 100,
        }.get(key, default)

        mock_psm = MagicMock()
        mock_psm.save_pdf.return_value = (None, None)

        with (
            patch(
                f"{MODULE}.get_document_for_resource",
                side_effect=[None, MagicMock()],
            ),
            patch(f"{MODULE}.get_source_type_id", return_value="src-1"),
            patch(f"{MODULE}.get_default_library_id", return_value="lib-1"),
            patch(f"{MODULE}.PDFStorageManager", return_value=mock_psm),
            patch(f"{MODULE}.uuid.uuid4", return_value="new-doc-id"),
            patch.object(
                svc,
                "_extract_text_from_pdf",
                side_effect=Exception("extraction error"),
            ),
        ):
            success, reason, _status_code = svc._download_pdf(
                resource, tracker, session
            )
            assert success is True  # Download still succeeds

    def test_skip_reason_from_downloader(self, svc):
        resource = MagicMock()
        resource.url = "https://example.com/paper.pdf"
        resource.id = 1

        tracker = MagicMock()
        tracker.url_hash = "hash1"
        tracker.download_attempts = MagicMock()
        tracker.download_attempts.count.return_value = 0

        session = MagicMock()

        result_obj = MagicMock()
        result_obj.is_success = False
        result_obj.content = None
        result_obj.skip_reason = "Access denied"

        from local_deep_research.research_library.downloaders import (
            GenericDownloader,
        )

        downloader = MagicMock(spec=GenericDownloader)
        downloader.can_handle.return_value = True
        downloader.download_with_result.return_value = result_obj
        downloader.__class__.__name__ = "GenericDownloader"
        svc.downloaders = [downloader]

        success, reason, _status_code = svc._download_pdf(
            resource, tracker, session
        )
        assert success is False
        assert reason == "Access denied"

    def test_empty_text_extraction(self, svc):
        """Text extraction returns None - warning logged but download succeeds."""
        resource = MagicMock()
        resource.url = "https://example.com/paper.pdf"
        resource.id = 1
        resource.title = "Test Paper Title for Testing"
        resource.research_id = "res-1"

        tracker = MagicMock()
        tracker.url_hash = "hash1"
        tracker.download_attempts = MagicMock()
        tracker.download_attempts.count.return_value = 0

        session = MagicMock()

        result_obj = MagicMock()
        result_obj.is_success = True
        result_obj.content = b"%PDF content"
        result_obj.skip_reason = None

        downloader = MagicMock()
        downloader.can_handle.return_value = True
        downloader.download_with_result.return_value = result_obj
        downloader.__class__.__name__ = "DirectPDFDownloader"
        svc.downloaders = [downloader]

        svc.settings.get_setting.side_effect = lambda key, default=None: {
            "research_library.pdf_storage_mode": "filesystem",
            "research_library.max_pdf_size_mb": 100,
        }.get(key, default)

        mock_psm = MagicMock()
        mock_psm.save_pdf.return_value = ("/tmp/test.pdf", None)

        with (
            patch(
                f"{MODULE}.get_document_for_resource",
                side_effect=[None, MagicMock()],
            ),
            patch(f"{MODULE}.get_source_type_id", return_value="src-1"),
            patch(f"{MODULE}.get_default_library_id", return_value="lib-1"),
            patch(f"{MODULE}.PDFStorageManager", return_value=mock_psm),
            patch(f"{MODULE}.uuid.uuid4", return_value="new-doc-id"),
            patch.object(svc, "_extract_text_from_pdf", return_value=None),
        ):
            success, reason, _status_code = svc._download_pdf(
                resource, tracker, session
            )
            assert success is True


# ============================================================
# _download_pubmed
# ============================================================


class TestDownloadPubmed:
    def test_pmc_article_success(self, svc):
        svc._last_pubmed_request = 0.0
        mock_resp = MagicMock()
        mock_resp.status_code = 200
        mock_resp.headers = {"content-type": "application/pdf"}
        mock_resp.content = b"%PDF big enough content" + b"x" * 1000

        with (
            patch(f"{MODULE}.safe_get", return_value=mock_resp),
            patch(f"{MODULE}.time.time", return_value=100.0),
            patch(f"{MODULE}.time.sleep"),
        ):
            result = svc._download_pubmed(
                "https://ncbi.nlm.nih.gov/pmc/articles/PMC12345/"
            )
            assert result == mock_resp.content

    def test_pmc_article_download_fails(self, svc):
        svc._last_pubmed_request = 0.0
        mock_resp = MagicMock()
        mock_resp.status_code = 404

        with (
            patch(f"{MODULE}.safe_get", side_effect=Exception("fail")),
            patch(f"{MODULE}.time.time", return_value=100.0),
            patch(f"{MODULE}.time.sleep"),
        ):
            # The inner try returns None on exception
            result = svc._download_pubmed(
                "https://ncbi.nlm.nih.gov/pmc/articles/PMC12345/"
            )
            # Falls through to _download_generic
            assert result is None

    def test_exception_returns_none(self, svc):
        svc._last_pubmed_request = 0.0
        with patch(f"{MODULE}.time.time", side_effect=Exception("bad")):
            assert (
                svc._download_pubmed("https://pubmed.ncbi.nlm.nih.gov/12345/")
                is None
            )

    def test_rate_limiting_applied(self, svc):
        svc._last_pubmed_request = 99.5
        svc._pubmed_delay = 1.0

        with (
            patch(f"{MODULE}.time.time", return_value=100.0),
            patch(f"{MODULE}.time.sleep") as mock_sleep,
            patch.object(svc, "_download_generic", return_value=None),
        ):
            svc._download_pubmed("https://example.com/pubmed")
            mock_sleep.assert_called_once()
            # Should sleep for 0.5s (1.0 - 0.5)
            assert abs(mock_sleep.call_args[0][0] - 0.5) < 0.01


# ============================================================
# PubMed downloader - additional
# ============================================================


class TestDownloadPubmedAdditional:
    def test_pubmed_url_with_pmid_europe_pmc_success(self, svc):
        svc._last_pubmed_request = 0.0

        with (
            patch(f"{MODULE}.time.time", return_value=100.0),
            patch(f"{MODULE}.time.sleep"),
            patch.object(svc, "_try_europe_pmc", return_value=b"pdf content"),
        ):
            result = svc._download_pubmed(
                "https://pubmed.ncbi.nlm.nih.gov/12345/"
            )
            assert result == b"pdf content"

    def test_pubmed_url_with_pmid_europe_pmc_fails_api_lookup(self, svc):
        svc._last_pubmed_request = 0.0

        api_resp = MagicMock()
        api_resp.status_code = 200
        api_resp.json.return_value = {"linksets": []}  # No linksets

        with (
            patch(f"{MODULE}.time.time", return_value=100.0),
            patch(f"{MODULE}.time.sleep"),
            patch.object(svc, "_try_europe_pmc", return_value=None),
            patch(f"{MODULE}.safe_get", return_value=api_resp),
            patch.object(svc, "_download_generic", return_value=None),
        ):
            result = svc._download_pubmed(
                "https://pubmed.ncbi.nlm.nih.gov/12345/"
            )
            assert result is None
