"""
Deep coverage tests for rag_routes.py targeting uncovered branches.

Covers:
- _get_rag_service_for_thread: default settings path, collection stored settings
  with string normalize_vectors, invalid text_separators JSON
- _background_index_worker: collection not found, force_reindex cleanup,
  cancellation mid-indexing, document indexing exception, outer exception
- _auto_index_documents_worker: successful indexing, skipped docs, per-doc exception,
  outer exception
- upload_to_collection: PDF database storage, pdf_upgrade paths,
  existing doc pdf_upgraded + already_in_collection, per-file exception,
  auto-index trigger with db_password
- collection_upload_page: database storage setting
- delete_collection: result not deleted with "not found" error vs generic error
- get_collection_documents: index file size formatting (B, KB, MB), no index
- configure_rag: text_separators as string input
- test_embedding: LLM-hint error detection vs generic error
- get_index_status: exception path
- cancel_indexing: exception path
- start_background_index: exception path
- _update_task_status: progress_total and progress_message updates
- _is_task_cancelled: task exists but not cancelled status
- get_rag_service: text_separators as non-string (already list)
"""

import uuid
from contextlib import contextmanager
from io import BytesIO
from unittest.mock import Mock, patch

import pytest
from flask import Flask, jsonify

from local_deep_research.web.auth.routes import auth_bp
from local_deep_research.research_library.routes.rag_routes import rag_bp

# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------

MODULE = "local_deep_research.research_library.routes.rag_routes"
_FACTORY = "local_deep_research.research_library.services.rag_service_factory"
_DB_CTX = "local_deep_research.database.session_context"
_DB_PASS = "local_deep_research.database.session_passwords"
_DOC_LOADERS = "local_deep_research.document_loaders"
_TEXT_PROC = "local_deep_research.text_processing"
_DEL_SVC = (
    "local_deep_research.research_library.deletion.services.collection_deletion"
)

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _uid():
    """Short unique identifier for test isolation."""
    return uuid.uuid4().hex[:12]


def _create_app():
    """Minimal Flask app with rag blueprint."""
    app = Flask(__name__)
    app.config["SECRET_KEY"] = f"test-{_uid()}"
    app.config["WTF_CSRF_ENABLED"] = False
    app.config["TESTING"] = True
    app.register_blueprint(auth_bp)
    app.register_blueprint(rag_bp)

    @app.errorhandler(500)
    def _handle_500(error):
        return jsonify({"error": "Internal server error"}), 500

    return app


def _mock_db_manager():
    """Mock db_manager so login_required passes."""
    m = Mock()
    m.is_user_connected.return_value = True
    m.connections = {"testuser": True}
    m.has_encryption = False
    return m


def _build_mock_query(all_result=None, first_result=None, count_result=0):
    """Build a chainable mock query."""
    q = Mock()
    q.all.return_value = all_result or []
    q.first.return_value = first_result
    q.count.return_value = count_result
    q.filter_by.return_value = q
    q.filter.return_value = q
    q.order_by.return_value = q
    q.outerjoin.return_value = q
    q.join.return_value = q
    q.limit.return_value = q
    q.offset.return_value = q
    q.delete.return_value = 0
    q.update.return_value = 0
    return q


def _make_settings_mock(overrides=None):
    """Create a mock settings manager."""
    mock_sm = Mock()
    defaults = {
        "local_search_embedding_model": "all-MiniLM-L6-v2",
        "local_search_embedding_provider": "sentence_transformers",
        "local_search_chunk_size": 1000,
        "local_search_chunk_overlap": 200,
        "local_search_splitter_type": "recursive",
        "local_search_text_separators": '["\n\n", "\n", ". ", " ", ""]',
        "local_search_distance_metric": "cosine",
        "local_search_normalize_vectors": True,
        "local_search_index_type": "flat",
        "research_library.upload_pdf_storage": "none",
        "research_library.storage_path": "/tmp/test_lib",
        "rag.indexing_batch_size": 15,
        "research_library.auto_index_enabled": True,
    }
    if overrides:
        defaults.update(overrides)
    mock_sm.get_setting.side_effect = lambda k, d=None, **kw: defaults.get(k, d)
    mock_sm.get_bool_setting.side_effect = lambda k, d=False, **kw: bool(
        defaults.get(k, d)
    )
    mock_sm.get_bool_setting.side_effect = lambda k, d=None: defaults.get(k, d)
    mock_sm.get_all_settings.return_value = {}
    mock_sm.set_setting = Mock()
    mock_sm.get_settings_snapshot.return_value = {}
    return mock_sm


def _make_db_session():
    """Create a standard mock db session."""
    s = Mock()
    s.query = Mock(return_value=_build_mock_query())
    s.commit = Mock()
    s.add = Mock()
    s.flush = Mock()
    s.expire_all = Mock()
    return s


@contextmanager
def _auth_client(
    app, mock_db_session=None, settings_overrides=None, extra_patches=None
):
    """Context manager providing an authenticated test client with mocking."""
    mock_db = _mock_db_manager()
    db_session = mock_db_session or _make_db_session()
    mock_sm = _make_settings_mock(settings_overrides)

    @contextmanager
    def fake_get_user_db_session(*a, **kw):
        yield db_session

    patches = [
        patch("local_deep_research.web.auth.decorators.db_manager", mock_db),
        patch(
            f"{_DB_CTX}.get_user_db_session",
            side_effect=fake_get_user_db_session,
        ),
        patch(f"{MODULE}.get_settings_manager", return_value=mock_sm),
        patch(f"{_FACTORY}.get_settings_manager", return_value=mock_sm),
        patch(
            f"{_FACTORY}.get_user_db_session",
            side_effect=fake_get_user_db_session,
        ),
        patch(
            "local_deep_research.utilities.db_utils.get_settings_manager",
            return_value=mock_sm,
        ),
        patch(f"{MODULE}.limiter", Mock(exempt=lambda f: f)),
        patch(f"{MODULE}.upload_rate_limit_user", lambda f: f),
        patch(f"{MODULE}.upload_rate_limit_ip", lambda f: f),
    ]
    if extra_patches:
        patches.extend(extra_patches)

    started = []
    try:
        for p in patches:
            started.append(p.start())
        with app.test_client() as client:
            with client.session_transaction() as sess:
                sess["username"] = "testuser"
                sess["session_id"] = "test-session-id"
            yield client, {"db_session": db_session, "settings": mock_sm}
    finally:
        for p in patches:
            p.stop()


@pytest.fixture
def app():
    """Minimal Flask app fixture."""
    return _create_app()


# ---------------------------------------------------------------------------
# _get_rag_service_for_thread
# ---------------------------------------------------------------------------


class TestGetRagServiceForThread:
    """Tests for _get_rag_service_for_thread helper."""

    def test_default_settings_when_no_collection(self):
        """Uses default settings when collection has no embedding_model."""
        from local_deep_research.research_library.routes.rag_routes import (
            _get_rag_service_for_thread,
        )

        mock_sm = _make_settings_mock()
        mock_coll = Mock()
        mock_coll.embedding_model = None  # No stored settings

        db_session = _make_db_session()
        q = _build_mock_query(first_result=mock_coll)
        db_session.query = Mock(return_value=q)

        @contextmanager
        def fake_session(*a, **kw):
            yield db_session

        mock_service = Mock()
        mock_service.__enter__ = Mock(return_value=mock_service)
        mock_service.__exit__ = Mock(return_value=False)

        with (
            patch(f"{_DB_CTX}.get_user_db_session", side_effect=fake_session),
            patch(
                "local_deep_research.research_library.services.rag_service_factory.get_user_db_session",
                side_effect=fake_session,
            ),
            patch(
                "local_deep_research.research_library.services.rag_service_factory.get_settings_manager",
                return_value=_make_settings_mock(),
            ),
            patch(
                "local_deep_research.research_library.services.rag_service_factory.LibraryRAGService",
                return_value=mock_service,
            ) as mock_rag_cls,
            patch(f"{MODULE}.SettingsManager", return_value=mock_sm),
            patch(f"{MODULE}.LibraryRAGService", return_value=mock_service),
            patch(
                "local_deep_research.web_search_engines.engines.local_embedding_manager.LocalEmbeddingManager"
            ) as mock_emb,
        ):
            mock_emb_inst = Mock()
            mock_emb.return_value = mock_emb_inst
            _get_rag_service_for_thread("coll-1", "testuser", "pass123")

        # Should have been called with defaults
        call_kwargs = mock_rag_cls.call_args.kwargs
        assert call_kwargs["embedding_model"] == "all-MiniLM-L6-v2"
        assert call_kwargs["embedding_provider"] == "sentence_transformers"

    def test_collection_stored_settings_with_string_normalize_vectors(self):
        """Handles normalize_vectors stored as string 'true'."""
        from local_deep_research.research_library.routes.rag_routes import (
            _get_rag_service_for_thread,
        )

        mock_sm = _make_settings_mock()
        mock_coll = Mock()
        mock_coll.embedding_model = "test-model"
        mock_coll.embedding_model_type = Mock(value="ollama")
        mock_coll.chunk_size = 500
        mock_coll.chunk_overlap = 100
        mock_coll.splitter_type = "recursive"
        mock_coll.text_separators = ["\n\n", "\n"]
        mock_coll.distance_metric = "cosine"
        mock_coll.normalize_vectors = "true"  # String, not bool
        mock_coll.index_type = "flat"

        db_session = _make_db_session()
        q = _build_mock_query(first_result=mock_coll)
        db_session.query = Mock(return_value=q)

        @contextmanager
        def fake_session(*a, **kw):
            yield db_session

        mock_service = Mock()

        with (
            patch(f"{_DB_CTX}.get_user_db_session", side_effect=fake_session),
            patch(
                "local_deep_research.research_library.services.rag_service_factory.get_user_db_session",
                side_effect=fake_session,
            ),
            patch(
                "local_deep_research.research_library.services.rag_service_factory.get_settings_manager",
                return_value=_make_settings_mock(),
            ),
            patch(
                "local_deep_research.research_library.services.rag_service_factory.LibraryRAGService",
                return_value=mock_service,
            ) as mock_rag_cls,
            patch(f"{MODULE}.SettingsManager", return_value=mock_sm),
            patch(f"{MODULE}.LibraryRAGService", return_value=mock_service),
            patch(
                "local_deep_research.web_search_engines.engines.local_embedding_manager.LocalEmbeddingManager"
            ) as mock_emb,
        ):
            mock_emb.return_value = Mock()
            _get_rag_service_for_thread("coll-1", "testuser", "pass123")

        call_kwargs = mock_rag_cls.call_args.kwargs
        assert call_kwargs["normalize_vectors"] is True
        assert call_kwargs["embedding_model"] == "test-model"

    def test_invalid_text_separators_json_uses_default(self):
        """Falls back to defaults when text_separators JSON is invalid."""
        from local_deep_research.research_library.routes.rag_routes import (
            _get_rag_service_for_thread,
        )

        mock_sm = _make_settings_mock(
            {"local_search_text_separators": "not valid json {{{"}
        )
        mock_coll = Mock()
        mock_coll.embedding_model = None

        db_session = _make_db_session()
        q = _build_mock_query(first_result=mock_coll)
        db_session.query = Mock(return_value=q)

        @contextmanager
        def fake_session(*a, **kw):
            yield db_session

        mock_service = Mock()

        with (
            patch(f"{_DB_CTX}.get_user_db_session", side_effect=fake_session),
            patch(
                "local_deep_research.research_library.services.rag_service_factory.get_user_db_session",
                side_effect=fake_session,
            ),
            patch(
                "local_deep_research.research_library.services.rag_service_factory.get_settings_manager",
                return_value=_make_settings_mock(),
            ),
            patch(
                "local_deep_research.research_library.services.rag_service_factory.LibraryRAGService",
                return_value=mock_service,
            ) as mock_rag_cls,
            patch(f"{MODULE}.SettingsManager", return_value=mock_sm),
            patch(f"{MODULE}.LibraryRAGService", return_value=mock_service),
            patch(
                "local_deep_research.web_search_engines.engines.local_embedding_manager.LocalEmbeddingManager"
            ) as mock_emb,
        ):
            mock_emb.return_value = Mock()
            _get_rag_service_for_thread("coll-1", "testuser", "pass123")

        call_kwargs = mock_rag_cls.call_args.kwargs
        assert call_kwargs["text_separators"] == ["\n\n", "\n", ". ", " ", ""]

    def test_normalize_vectors_none_falls_back_to_default(self):
        """When collection.normalize_vectors is None, uses default."""
        from local_deep_research.research_library.routes.rag_routes import (
            _get_rag_service_for_thread,
        )

        mock_sm = _make_settings_mock()
        mock_coll = Mock()
        mock_coll.embedding_model = "test-model"
        mock_coll.embedding_model_type = Mock(value="ollama")
        mock_coll.chunk_size = 500
        mock_coll.chunk_overlap = 100
        mock_coll.splitter_type = None
        mock_coll.text_separators = None
        mock_coll.distance_metric = None
        mock_coll.normalize_vectors = None  # None triggers fallback
        mock_coll.index_type = None

        db_session = _make_db_session()
        q = _build_mock_query(first_result=mock_coll)
        db_session.query = Mock(return_value=q)

        @contextmanager
        def fake_session(*a, **kw):
            yield db_session

        mock_service = Mock()

        with (
            patch(f"{_DB_CTX}.get_user_db_session", side_effect=fake_session),
            patch(
                "local_deep_research.research_library.services.rag_service_factory.get_user_db_session",
                side_effect=fake_session,
            ),
            patch(
                "local_deep_research.research_library.services.rag_service_factory.get_settings_manager",
                return_value=_make_settings_mock(),
            ),
            patch(
                "local_deep_research.research_library.services.rag_service_factory.LibraryRAGService",
                return_value=mock_service,
            ) as mock_rag_cls,
            patch(f"{MODULE}.SettingsManager", return_value=mock_sm),
            patch(f"{MODULE}.LibraryRAGService", return_value=mock_service),
            patch(
                "local_deep_research.web_search_engines.engines.local_embedding_manager.LocalEmbeddingManager"
            ) as mock_emb,
        ):
            mock_emb.return_value = Mock()
            _get_rag_service_for_thread("coll-1", "testuser", "pass123")

        call_kwargs = mock_rag_cls.call_args.kwargs
        # Should use default True when collection.normalize_vectors is None
        assert call_kwargs["normalize_vectors"] is True


# ---------------------------------------------------------------------------
# _auto_index_documents_worker
# ---------------------------------------------------------------------------


class TestAutoIndexDocumentsWorker:
    """Tests for _auto_index_documents_worker background worker."""

    def test_successful_indexing(self):
        """Worker indexes documents successfully."""
        from local_deep_research.research_library.routes.rag_routes import (
            _auto_index_documents_worker,
        )

        mock_service = Mock()
        mock_service.__enter__ = Mock(return_value=mock_service)
        mock_service.__exit__ = Mock(return_value=False)
        mock_service.index_document.return_value = {"status": "success"}

        with (
            patch(
                f"{MODULE}._get_rag_service_for_thread",
                return_value=mock_service,
            ),
            patch(f"{MODULE}.thread_cleanup", lambda f: f),
        ):
            _auto_index_documents_worker(
                ["doc1", "doc2"], "coll-1", "user", "pass"
            )

        assert mock_service.index_document.call_count == 2

    def test_skipped_documents(self):
        """Worker handles already-indexed documents."""
        from local_deep_research.research_library.routes.rag_routes import (
            _auto_index_documents_worker,
        )

        mock_service = Mock()
        mock_service.__enter__ = Mock(return_value=mock_service)
        mock_service.__exit__ = Mock(return_value=False)
        mock_service.index_document.return_value = {"status": "skipped"}

        with patch(
            f"{MODULE}._get_rag_service_for_thread", return_value=mock_service
        ):
            _auto_index_documents_worker(["doc1"], "coll-1", "user", "pass")

        mock_service.index_document.assert_called_once()

    def test_per_document_exception_continues(self):
        """Worker continues indexing after per-document exception."""
        from local_deep_research.research_library.routes.rag_routes import (
            _auto_index_documents_worker,
        )

        mock_service = Mock()
        mock_service.__enter__ = Mock(return_value=mock_service)
        mock_service.__exit__ = Mock(return_value=False)
        mock_service.index_document.side_effect = [
            RuntimeError("fail"),
            {"status": "success"},
        ]

        with patch(
            f"{MODULE}._get_rag_service_for_thread", return_value=mock_service
        ):
            # Should not raise
            _auto_index_documents_worker(
                ["doc1", "doc2"], "coll-1", "user", "pass"
            )

        assert mock_service.index_document.call_count == 2

    def test_outer_exception_logged(self):
        """Worker handles outer exception (e.g., service creation failure)."""
        from local_deep_research.research_library.routes.rag_routes import (
            _auto_index_documents_worker,
        )

        with patch(
            f"{MODULE}._get_rag_service_for_thread",
            side_effect=RuntimeError("service creation failed"),
        ):
            # Should not raise
            _auto_index_documents_worker(["doc1"], "coll-1", "user", "pass")


# ---------------------------------------------------------------------------
# _background_index_worker
# ---------------------------------------------------------------------------


class TestBackgroundIndexWorker:
    """Tests for _background_index_worker."""

    def _make_worker_mocks(
        self, collection=None, doc_links=None, force_reindex=False
    ):
        """Set up common mocks for background worker tests."""
        mock_service = Mock()
        mock_service.__enter__ = Mock(return_value=mock_service)
        mock_service.__exit__ = Mock(return_value=False)
        mock_service.embedding_model = "test-model"
        mock_service.embedding_provider = "sentence_transformers"
        mock_service.chunk_size = 1000
        mock_service.chunk_overlap = 200
        mock_service.splitter_type = "recursive"
        mock_service.text_separators = ["\n\n"]
        mock_service.distance_metric = "cosine"
        mock_service.normalize_vectors = True
        mock_service.index_type = "flat"

        db_session = _make_db_session()

        call_count = {"n": 0}

        def query_side_effect(*args):
            call_count["n"] += 1
            q = _build_mock_query()
            if call_count["n"] == 1:
                q.first.return_value = collection
            if doc_links is not None:
                q.all.return_value = doc_links
            return q

        db_session.query = Mock(side_effect=query_side_effect)

        return mock_service, db_session

    def test_collection_not_found(self):
        """Worker reports failure when collection not found."""
        from local_deep_research.research_library.routes.rag_routes import (
            _background_index_worker,
        )

        mock_service, db_session = self._make_worker_mocks(collection=None)

        @contextmanager
        def fake_session(*a, **kw):
            yield db_session

        with (
            patch(
                f"{MODULE}._get_rag_service_for_thread",
                return_value=mock_service,
            ),
            patch(f"{_DB_CTX}.get_user_db_session", side_effect=fake_session),
            patch(
                "local_deep_research.research_library.services.rag_service_factory.get_user_db_session",
                side_effect=fake_session,
            ),
            patch(
                "local_deep_research.research_library.services.rag_service_factory.get_settings_manager",
                return_value=_make_settings_mock(),
            ),
            patch(f"{MODULE}._update_task_status") as mock_update,
        ):
            _background_index_worker("task-1", "coll-1", "user", "pass", False)

        # Should have been called with "failed" status
        mock_update.assert_called_with(
            "user",
            "pass",
            "task-1",
            status="failed",
            error_message="Collection not found",
        )

    def test_outer_exception_updates_task(self):
        """Worker updates task to failed when outer exception occurs."""
        from local_deep_research.research_library.routes.rag_routes import (
            _background_index_worker,
        )

        with (
            patch(
                f"{MODULE}._get_rag_service_for_thread",
                side_effect=RuntimeError("service boom"),
            ),
            patch(f"{MODULE}._update_task_status") as mock_update,
        ):
            _background_index_worker("task-1", "coll-1", "user", "pass", False)

        # Last call should be the failure update
        last_call = mock_update.call_args
        assert last_call.kwargs.get("status") == "failed" or (
            len(last_call.args) > 3 and "failed" in str(last_call)
        )


# ---------------------------------------------------------------------------
# delete_collection edge cases -- moved to delete_routes tests
# (route removed from rag_routes.py, canonical endpoint in delete_routes.py)
# ---------------------------------------------------------------------------


# ---------------------------------------------------------------------------
# upload_to_collection: PDF storage and edge cases
# ---------------------------------------------------------------------------


class TestUploadPdfStorageDatabase:
    """Tests for upload with pdf_storage='database'."""

    def test_upload_new_doc_with_pdf_storage(self, app):
        """Uploads a PDF with database storage enabled."""
        mock_coll = Mock()
        mock_coll.id = "coll-1"

        db_session = _make_db_session()
        call_count = {"n": 0}

        def query_side_effect(model):
            call_count["n"] += 1
            q = _build_mock_query()
            if call_count["n"] == 1:
                q.first.return_value = mock_coll
            elif call_count["n"] == 2:
                q.first.return_value = None  # No existing doc
            elif call_count["n"] == 3:
                mock_source = Mock()
                mock_source.id = "src-1"
                q.first.return_value = mock_source  # Source type exists
            return q

        db_session.query = Mock(side_effect=query_side_effect)

        mock_password_store = Mock()
        mock_password_store.get_session_password.return_value = None

        mock_pdf_manager = Mock()
        mock_pdf_manager.save_pdf = Mock()

        with _auth_client(
            app,
            mock_db_session=db_session,
            settings_overrides={
                "research_library.upload_pdf_storage": "database"
            },
            extra_patches=[
                patch(
                    f"{_DOC_LOADERS}.extract_text_from_bytes",
                    return_value="PDF text",
                ),
                patch(
                    f"{_DOC_LOADERS}.is_extension_supported", return_value=True
                ),
                patch(
                    f"{_TEXT_PROC}.remove_surrogates", side_effect=lambda x: x
                ),
                patch(
                    f"{_DB_PASS}.session_password_store", mock_password_store
                ),
                patch(
                    "local_deep_research.research_library.services.pdf_storage_manager.PDFStorageManager",
                    return_value=mock_pdf_manager,
                ),
            ],
        ) as (client, ctx):
            data = {
                "files": (BytesIO(b"%PDF-test-content"), "test.pdf"),
                "pdf_storage": "database",
            }
            resp = client.post(
                "/library/api/collections/coll-1/upload",
                data=data,
                content_type="multipart/form-data",
            )
            assert resp.status_code == 200
            rdata = resp.get_json()
            assert rdata["success"] is True

    def test_existing_doc_pdf_upgrade_not_in_collection(self, app):
        """Existing doc gets PDF upgrade and added to collection."""
        mock_coll = Mock()
        mock_coll.id = "coll-1"

        existing_doc = Mock()
        existing_doc.id = "doc-existing"
        existing_doc.filename = "test.pdf"

        db_session = _make_db_session()
        call_count = {"n": 0}

        def query_side_effect(model):
            call_count["n"] += 1
            q = _build_mock_query()
            if call_count["n"] == 1:
                q.first.return_value = mock_coll
            elif call_count["n"] == 2:
                q.first.return_value = existing_doc
            elif call_count["n"] == 3:
                q.first.return_value = None  # Not in collection
            return q

        db_session.query = Mock(side_effect=query_side_effect)

        mock_password_store = Mock()
        mock_password_store.get_session_password.return_value = None
        mock_pdf_manager = Mock()
        mock_pdf_manager.upgrade_to_pdf.return_value = True

        with _auth_client(
            app,
            mock_db_session=db_session,
            settings_overrides={
                "research_library.upload_pdf_storage": "database"
            },
            extra_patches=[
                patch(
                    f"{_DB_PASS}.session_password_store", mock_password_store
                ),
                patch(
                    "local_deep_research.research_library.services.pdf_storage_manager.PDFStorageManager",
                    return_value=mock_pdf_manager,
                ),
            ],
        ) as (client, ctx):
            data = {"files": (BytesIO(b"%PDF-data"), "test.pdf")}
            resp = client.post(
                "/library/api/collections/coll-1/upload",
                data=data,
                content_type="multipart/form-data",
            )
            assert resp.status_code == 200
            rdata = resp.get_json()
            assert (
                rdata["uploaded"][0]["status"]
                == "added_to_collection_pdf_upgraded"
            )
            assert rdata["uploaded"][0]["pdf_upgraded"] is True

    def test_existing_doc_pdf_upgrade_already_in_collection(self, app):
        """Existing doc already in collection gets pdf_upgraded status."""
        mock_coll = Mock()
        mock_coll.id = "coll-1"

        existing_doc = Mock()
        existing_doc.id = "doc-existing"
        existing_doc.filename = "test.pdf"

        db_session = _make_db_session()
        call_count = {"n": 0}

        def query_side_effect(model):
            call_count["n"] += 1
            q = _build_mock_query()
            if call_count["n"] == 1:
                q.first.return_value = mock_coll
            elif call_count["n"] == 2:
                q.first.return_value = existing_doc
            elif call_count["n"] == 3:
                q.first.return_value = Mock()  # Already in collection
            return q

        db_session.query = Mock(side_effect=query_side_effect)

        mock_password_store = Mock()
        mock_password_store.get_session_password.return_value = None
        mock_pdf_manager = Mock()
        mock_pdf_manager.upgrade_to_pdf.return_value = True

        with _auth_client(
            app,
            mock_db_session=db_session,
            settings_overrides={
                "research_library.upload_pdf_storage": "database"
            },
            extra_patches=[
                patch(
                    f"{_DB_PASS}.session_password_store", mock_password_store
                ),
                patch(
                    "local_deep_research.research_library.services.pdf_storage_manager.PDFStorageManager",
                    return_value=mock_pdf_manager,
                ),
            ],
        ) as (client, ctx):
            data = {"files": (BytesIO(b"%PDF-data"), "test.pdf")}
            resp = client.post(
                "/library/api/collections/coll-1/upload",
                data=data,
                content_type="multipart/form-data",
            )
            assert resp.status_code == 200
            rdata = resp.get_json()
            assert rdata["uploaded"][0]["status"] == "pdf_upgraded"

    def test_upload_auto_index_triggered_with_password(self, app):
        """Auto-index is triggered when db_password is available."""
        mock_coll = Mock()
        mock_coll.id = "coll-1"

        db_session = _make_db_session()
        call_count = {"n": 0}

        def query_side_effect(model):
            call_count["n"] += 1
            q = _build_mock_query()
            if call_count["n"] == 1:
                q.first.return_value = mock_coll
            elif call_count["n"] == 2:
                q.first.return_value = None  # No existing doc
            elif call_count["n"] == 3:
                mock_source = Mock()
                mock_source.id = "src-1"
                q.first.return_value = mock_source
            return q

        db_session.query = Mock(side_effect=query_side_effect)

        mock_password_store = Mock()
        mock_password_store.get_session_password.return_value = (
            "db-password-123"
        )

        with _auth_client(
            app,
            mock_db_session=db_session,
            extra_patches=[
                patch(
                    f"{_DOC_LOADERS}.extract_text_from_bytes",
                    return_value="Text",
                ),
                patch(
                    f"{_DOC_LOADERS}.is_extension_supported", return_value=True
                ),
                patch(
                    f"{_TEXT_PROC}.remove_surrogates", side_effect=lambda x: x
                ),
                patch(
                    f"{_DB_PASS}.session_password_store", mock_password_store
                ),
                patch(f"{MODULE}.trigger_auto_index"),
            ],
        ) as (client, ctx):
            data = {"files": (BytesIO(b"text content"), "doc.txt")}
            resp = client.post(
                "/library/api/collections/coll-1/upload",
                data=data,
                content_type="multipart/form-data",
            )
            assert resp.status_code == 200

    def test_upload_pdf_save_failure_continues(self, app):
        """Upload continues even if PDF save to database fails."""
        mock_coll = Mock()
        mock_coll.id = "coll-1"

        db_session = _make_db_session()
        call_count = {"n": 0}

        def query_side_effect(model):
            call_count["n"] += 1
            q = _build_mock_query()
            if call_count["n"] == 1:
                q.first.return_value = mock_coll
            elif call_count["n"] == 2:
                q.first.return_value = None
            elif call_count["n"] == 3:
                mock_source = Mock()
                mock_source.id = "src-1"
                q.first.return_value = mock_source
            return q

        db_session.query = Mock(side_effect=query_side_effect)

        mock_password_store = Mock()
        mock_password_store.get_session_password.return_value = None
        mock_pdf_manager = Mock()
        mock_pdf_manager.save_pdf.side_effect = RuntimeError("Storage failed")

        with _auth_client(
            app,
            mock_db_session=db_session,
            settings_overrides={
                "research_library.upload_pdf_storage": "database"
            },
            extra_patches=[
                patch(
                    f"{_DOC_LOADERS}.extract_text_from_bytes",
                    return_value="PDF text",
                ),
                patch(
                    f"{_DOC_LOADERS}.is_extension_supported", return_value=True
                ),
                patch(
                    f"{_TEXT_PROC}.remove_surrogates", side_effect=lambda x: x
                ),
                patch(
                    f"{_DB_PASS}.session_password_store", mock_password_store
                ),
                patch(
                    "local_deep_research.research_library.services.pdf_storage_manager.PDFStorageManager",
                    return_value=mock_pdf_manager,
                ),
            ],
        ) as (client, ctx):
            data = {
                "files": (BytesIO(b"%PDF-content"), "test.pdf"),
                "pdf_storage": "database",
            }
            resp = client.post(
                "/library/api/collections/coll-1/upload",
                data=data,
                content_type="multipart/form-data",
            )
            assert resp.status_code == 200
            rdata = resp.get_json()
            assert rdata["success"] is True
            assert rdata["uploaded"][0]["pdf_stored"] is False


# ---------------------------------------------------------------------------
# collection_upload_page: database storage setting
# ---------------------------------------------------------------------------


class TestCollectionUploadPageStorageSettings:
    """Edge cases for collection_upload_page storage setting."""

    def test_database_storage_setting(self, app):
        """Upload page passes database storage setting to template."""
        with _auth_client(
            app,
            settings_overrides={
                "research_library.upload_pdf_storage": "database"
            },
            extra_patches=[
                patch(
                    f"{MODULE}.render_template", return_value="<html>ok</html>"
                ),
            ],
        ) as (client, ctx):
            resp = client.get("/library/collections/coll-1/upload")
            assert resp.status_code == 200


# ---------------------------------------------------------------------------
# test_embedding: error hint detection
# ---------------------------------------------------------------------------


class TestTestEmbeddingErrorHints:
    """Tests for LLM-hint detection in test_embedding error path."""

    def test_llm_hint_does_not_support(self, app):
        """Error containing 'does not support' triggers LLM hint message."""
        with _auth_client(
            app,
            extra_patches=[
                patch(
                    "local_deep_research.embeddings.embeddings_config.get_embedding_function",
                    side_effect=ValueError("model does not support embeddings"),
                ),
            ],
        ) as (client, ctx):
            resp = client.post(
                "/library/api/rag/test-embedding",
                json={"provider": "ollama", "model": "llama3"},
                content_type="application/json",
            )
            assert resp.status_code == 500
            data = resp.get_json()
            assert "LLM (language model)" in data["error"]

    def test_llm_hint_list_index_out_of_range(self, app):
        """Error with 'list index out of range' triggers LLM hint."""
        with _auth_client(
            app,
            extra_patches=[
                patch(
                    "local_deep_research.embeddings.embeddings_config.get_embedding_function",
                    side_effect=IndexError("list index out of range"),
                ),
            ],
        ) as (client, ctx):
            resp = client.post(
                "/library/api/rag/test-embedding",
                json={"provider": "ollama", "model": "bad-model"},
                content_type="application/json",
            )
            assert resp.status_code == 500
            data = resp.get_json()
            assert "LLM (language model)" in data["error"]

    def test_generic_error_no_llm_hint(self, app):
        """Error without LLM hints gives generic message."""
        with _auth_client(
            app,
            extra_patches=[
                patch(
                    "local_deep_research.embeddings.embeddings_config.get_embedding_function",
                    side_effect=ConnectionError("connection refused"),
                ),
            ],
        ) as (client, ctx):
            resp = client.post(
                "/library/api/rag/test-embedding",
                json={"provider": "ollama", "model": "embed-model"},
                content_type="application/json",
            )
            assert resp.status_code == 500
            data = resp.get_json()
            assert "If you are unsure" in data["error"]


# ---------------------------------------------------------------------------
# get_collection_documents: index file size formatting
# ---------------------------------------------------------------------------


class TestCollectionDocumentsNoIndex:
    """Test get_collection_documents when no RAG index exists."""

    def test_no_rag_index_returns_null_size(self, app):
        """When no RAG index exists, index_file_size is None."""
        mock_coll = Mock()
        mock_coll.id = "coll-1"
        mock_coll.name = "Test Collection"
        mock_coll.description = "Desc"
        mock_coll.embedding_model = None
        mock_coll.embedding_model_type = None
        mock_coll.embedding_dimension = None
        mock_coll.chunk_size = None
        mock_coll.chunk_overlap = None
        mock_coll.splitter_type = None
        mock_coll.distance_metric = None
        mock_coll.index_type = None
        mock_coll.normalize_vectors = None
        mock_coll.collection_type = "user_uploads"

        db_session = _make_db_session()
        call_count = {"n": 0}

        def query_side_effect(*args):
            call_count["n"] += 1
            q = _build_mock_query()
            if call_count["n"] == 1:
                q.first.return_value = mock_coll
            elif call_count["n"] == 2:
                q.all.return_value = []
            elif call_count["n"] == 3:
                q.first.return_value = None  # No RAG index
            return q

        db_session.query = Mock(side_effect=query_side_effect)

        with _auth_client(app, mock_db_session=db_session) as (client, ctx):
            resp = client.get("/library/api/collections/coll-1/documents")
            assert resp.status_code == 200
            data = resp.get_json()
            assert data["collection"]["index_file_size"] is None
            assert data["collection"]["index_file_size_bytes"] is None

    def test_collection_not_found_returns_404(self, app):
        """Returns 404 when collection doesn't exist."""
        db_session = _make_db_session()
        q = _build_mock_query(first_result=None)
        db_session.query = Mock(return_value=q)

        with _auth_client(app, mock_db_session=db_session) as (client, ctx):
            resp = client.get("/library/api/collections/nonexistent/documents")
            assert resp.status_code == 404


# ---------------------------------------------------------------------------
# configure_rag: text_separators as string
# ---------------------------------------------------------------------------


class TestConfigureRagTextSeparatorsString:
    """Tests for configure_rag with text_separators as string input."""

    def test_text_separators_string_with_collection(self, app):
        """text_separators passed as JSON string for collection config."""
        mock_rag_service = Mock()
        mock_rag_service.__enter__ = Mock(return_value=mock_rag_service)
        mock_rag_service.__exit__ = Mock(return_value=False)
        mock_rag_index = Mock()
        mock_rag_index.index_hash = "abc123"
        mock_rag_service._get_or_create_rag_index.return_value = mock_rag_index

        with _auth_client(
            app,
            extra_patches=[
                patch(
                    f"{MODULE}.LibraryRAGService", return_value=mock_rag_service
                ),
            ],
        ) as (client, ctx):
            resp = client.post(
                "/library/api/rag/configure",
                json={
                    "embedding_model": "test-model",
                    "embedding_provider": "ollama",
                    "chunk_size": 500,
                    "chunk_overlap": 100,
                    "collection_id": "coll-1",
                    "text_separators": '["\\n\\n", "\\n"]',  # String, not list
                },
                content_type="application/json",
            )
            assert resp.status_code == 200
            data = resp.get_json()
            assert data["success"] is True
            assert data["index_hash"] == "abc123"


# ---------------------------------------------------------------------------
# get_index_status / cancel_indexing / start_background_index exception paths
# ---------------------------------------------------------------------------


class TestRouteExceptionPaths:
    """Tests for exception handling in route handlers."""

    def test_get_index_status_exception(self, app):
        """get_index_status returns 500 on db exception inside try block."""
        mock_password_store = Mock()
        mock_password_store.get_session_password.return_value = "pass"

        @contextmanager
        def failing_session(*a, **kw):
            raise RuntimeError("db boom")
            yield  # noqa: E501, F841

        with _auth_client(
            app,
            extra_patches=[
                patch(
                    f"{_DB_PASS}.session_password_store", mock_password_store
                ),
                patch(
                    f"{_DB_CTX}.get_user_db_session",
                    side_effect=failing_session,
                ),
            ],
        ) as (client, ctx):
            resp = client.get("/library/api/collections/coll-1/index/status")
            assert resp.status_code == 500
            data = resp.get_json()
            assert data["status"] == "error"

    def test_cancel_indexing_exception(self, app):
        """cancel_indexing returns 500 on db exception inside try block."""
        mock_password_store = Mock()
        mock_password_store.get_session_password.return_value = "pass"

        @contextmanager
        def failing_session(*a, **kw):
            raise RuntimeError("db boom")
            yield  # noqa: E501, F841

        with _auth_client(
            app,
            extra_patches=[
                patch(
                    f"{_DB_PASS}.session_password_store", mock_password_store
                ),
                patch(
                    f"{_DB_CTX}.get_user_db_session",
                    side_effect=failing_session,
                ),
            ],
        ) as (client, ctx):
            resp = client.post("/library/api/collections/coll-1/index/cancel")
            assert resp.status_code == 500

    def test_start_background_index_exception(self, app):
        """start_background_index returns 500 on db exception inside try block."""
        mock_password_store = Mock()
        mock_password_store.get_session_password.return_value = "pass"

        @contextmanager
        def failing_session(*a, **kw):
            raise RuntimeError("db boom")
            yield  # noqa: E501, F841

        with _auth_client(
            app,
            extra_patches=[
                patch(
                    f"{_DB_PASS}.session_password_store", mock_password_store
                ),
                patch(
                    f"{_DB_CTX}.get_user_db_session",
                    side_effect=failing_session,
                ),
            ],
        ) as (client, ctx):
            resp = client.post(
                "/library/api/collections/coll-1/index/start",
                json={"force_reindex": False},
                content_type="application/json",
            )
            assert resp.status_code == 500


# ---------------------------------------------------------------------------
# get_rag_service: text_separators already a list
# ---------------------------------------------------------------------------


class TestGetRagServiceTextSeparatorsList:
    """Test get_rag_service when text_separators is already a list (not string)."""

    def test_text_separators_already_list(self, app):
        """When text_separators setting is already a list, no JSON parsing needed."""
        mock_service = Mock()
        mock_service.get_rag_stats.return_value = {}

        rag_patch = patch(
            f"{_FACTORY}.LibraryRAGService", return_value=mock_service
        )

        with _auth_client(
            app,
            settings_overrides={"local_search_text_separators": ["\n\n", "\n"]},
            extra_patches=[
                rag_patch,
                patch(
                    "local_deep_research.database.library_init.get_default_library_id",
                    return_value="lib-1",
                ),
            ],
        ) as (client, ctx):
            resp = client.get("/library/api/rag/stats")
            assert resp.status_code == 200


# ---------------------------------------------------------------------------
# _update_task_status: edge case with completed_at
# ---------------------------------------------------------------------------


class TestUpdateTaskStatusEdgeCases:
    """Edge cases for _update_task_status."""

    def test_sets_completed_at_on_completed_status(self):
        """When status is 'completed', completed_at is set."""
        from local_deep_research.research_library.routes.rag_routes import (
            _update_task_status,
        )

        mock_task = Mock()
        mock_task.status = "processing"
        db_session = _make_db_session()
        q = _build_mock_query(first_result=mock_task)
        db_session.query = Mock(return_value=q)

        @contextmanager
        def fake_session(*a, **kw):
            yield db_session

        with patch(f"{_DB_CTX}.get_user_db_session", side_effect=fake_session):
            _update_task_status(
                "user",
                "pass",
                "task-1",
                status="completed",
                progress_message="Done",
            )

        assert mock_task.status == "completed"
        assert mock_task.completed_at is not None
        assert mock_task.progress_message == "Done"

    def test_updates_progress_total_only(self):
        """Can update progress_total without changing status."""
        from local_deep_research.research_library.routes.rag_routes import (
            _update_task_status,
        )

        mock_task = Mock()
        mock_task.status = "processing"
        db_session = _make_db_session()
        q = _build_mock_query(first_result=mock_task)
        db_session.query = Mock(return_value=q)

        @contextmanager
        def fake_session(*a, **kw):
            yield db_session

        with patch(f"{_DB_CTX}.get_user_db_session", side_effect=fake_session):
            _update_task_status(
                "user",
                "pass",
                "task-1",
                progress_total=50,
            )

        assert mock_task.progress_total == 50
        # status should not have been changed
        assert mock_task.status == "processing"


# ---------------------------------------------------------------------------
# _is_task_cancelled: non-cancelled status
# ---------------------------------------------------------------------------


class TestIsTaskCancelledEdgeCases:
    """Edge case for _is_task_cancelled."""

    def test_task_exists_but_processing(self):
        """Task exists but is still processing, returns False."""
        from local_deep_research.research_library.routes.rag_routes import (
            _is_task_cancelled,
        )

        mock_task = Mock()
        mock_task.status = "processing"
        db_session = _make_db_session()
        q = _build_mock_query(first_result=mock_task)
        db_session.query = Mock(return_value=q)

        @contextmanager
        def fake_session(*a, **kw):
            yield db_session

        with patch(f"{_DB_CTX}.get_user_db_session", side_effect=fake_session):
            result = _is_task_cancelled("user", "pass", "task-1")

        assert result is False


# ---------------------------------------------------------------------------
# get_index_status: task for different collection
# ---------------------------------------------------------------------------


class TestGetIndexStatusNullDates:
    """Test get_index_status null date handling."""

    def test_task_with_null_dates(self, app):
        """Task with null created_at and completed_at returns None for dates."""
        mock_task = Mock()
        mock_task.task_id = "task-1"
        mock_task.status = "completed"
        mock_task.progress_current = 5
        mock_task.progress_total = 5
        mock_task.progress_message = "Done"
        mock_task.error_message = None
        mock_task.created_at = None
        mock_task.completed_at = None
        mock_task.metadata_json = {"collection_id": "coll-1"}

        db_session = _make_db_session()
        q = _build_mock_query(first_result=mock_task)
        db_session.query = Mock(return_value=q)

        mock_password_store = Mock()
        mock_password_store.get_session_password.return_value = None

        with _auth_client(
            app,
            mock_db_session=db_session,
            extra_patches=[
                patch(
                    f"{_DB_PASS}.session_password_store", mock_password_store
                ),
            ],
        ) as (client, ctx):
            resp = client.get("/library/api/collections/coll-1/index/status")
            assert resp.status_code == 200
            data = resp.get_json()
            assert data["created_at"] is None
            assert data["completed_at"] is None


# ---------------------------------------------------------------------------
# cancel_indexing: null metadata_json handling
# ---------------------------------------------------------------------------


class TestCancelIndexingNullMetadata:
    """Test cancel_indexing with null metadata_json."""

    def test_null_metadata_json(self, app):
        """Task with null metadata_json returns 404 (not for this collection)."""
        mock_task = Mock()
        mock_task.task_id = "task-1"
        mock_task.status = "processing"
        mock_task.metadata_json = None  # null metadata

        db_session = _make_db_session()
        q = _build_mock_query(first_result=mock_task)
        db_session.query = Mock(return_value=q)

        mock_password_store = Mock()
        mock_password_store.get_session_password.return_value = None

        with _auth_client(
            app,
            mock_db_session=db_session,
            extra_patches=[
                patch(
                    f"{_DB_PASS}.session_password_store", mock_password_store
                ),
            ],
        ) as (client, ctx):
            resp = client.post("/library/api/collections/coll-1/index/cancel")
            assert resp.status_code == 404


# ---------------------------------------------------------------------------
# get_rag_service: no collection_id (fallback path)
# ---------------------------------------------------------------------------


class TestGetRagServiceNoCollection:
    """Test get_rag_service fallback when no collection found in DB."""

    def test_collection_id_provided_but_not_found(self, app):
        """When collection_id is provided but collection not in DB, uses defaults."""
        db_session = _make_db_session()
        q = _build_mock_query(first_result=None)  # Collection not found
        db_session.query = Mock(return_value=q)

        mock_service = Mock()
        mock_service.get_rag_stats.return_value = {}

        with _auth_client(
            app,
            mock_db_session=db_session,
            extra_patches=[
                patch(
                    f"{_FACTORY}.LibraryRAGService", return_value=mock_service
                ),
                patch(
                    "local_deep_research.database.library_init.get_default_library_id",
                    return_value="default-lib",
                ),
            ],
        ) as (client, ctx):
            resp = client.get(
                "/library/api/rag/stats?collection_id=nonexistent"
            )
            # Should use default settings (fallback path at end of get_rag_service)
            assert resp.status_code == 200
