"""
Routes for Research Library and Download Manager

Provides web endpoints for:
- Library browsing and management
- Download manager interface
- API endpoints for downloads and queries
"""

import json
from io import BytesIO
from pathlib import Path
from urllib.parse import urlparse
from flask import (
    Blueprint,
    g,
    jsonify,
    request,
    session,
    Response,
    send_file,
    stream_with_context,
)
from loguru import logger

from ...web.auth.decorators import login_required
from ...web.utils.templates import render_template_with_defaults
from ...database.session_context import get_user_db_session
from ...database.models.research import ResearchResource
from ...database.models.library import (
    Document as Document,
    DocumentStatus,
    DownloadQueue as LibraryDownloadQueue,
    Collection,
)
from ...library.download_management import ResourceFilter
from ..services.download_service import DownloadService
from ..services.library_service import LibraryService
from ..services.pdf_storage_manager import PDFStorageManager
from ..utils import handle_api_error
from ...utilities.db_utils import get_settings_manager
from ...config.paths import get_library_directory

# Create Blueprint
library_bp = Blueprint("library", __name__, url_prefix="/library")


# Error handler for authentication errors
@library_bp.errorhandler(Exception)
def handle_web_api_exception(error):
    """Handle WebAPIException and its subclasses."""
    from ...web.exceptions import WebAPIException

    if isinstance(error, WebAPIException):
        return jsonify(error.to_dict()), error.status_code
    # Re-raise other exceptions
    raise error


def is_downloadable_domain(url: str) -> bool:
    """Check if URL is from a downloadable academic domain using proper URL parsing."""
    try:
        if not url:
            return False

        parsed = urlparse(url.lower())
        hostname = parsed.hostname or ""
        path = parsed.path or ""
        query = parsed.query or ""

        # Check for direct PDF files
        if path.endswith(".pdf") or ".pdf?" in url.lower():
            return True

        # List of downloadable academic domains
        downloadable_domains = [
            "arxiv.org",
            "biorxiv.org",
            "medrxiv.org",
            "ncbi.nlm.nih.gov",
            "pubmed.ncbi.nlm.nih.gov",
            "europepmc.org",
            "semanticscholar.org",
            "researchgate.net",
            "academia.edu",
            "sciencedirect.com",
            "springer.com",
            "nature.com",
            "wiley.com",
            "ieee.org",
            "acm.org",
            "plos.org",
            "frontiersin.org",
            "mdpi.com",
            "acs.org",
            "rsc.org",
            "tandfonline.com",
            "sagepub.com",
            "oxford.com",
            "cambridge.org",
            "bmj.com",
            "nejm.org",
            "thelancet.com",
            "jamanetwork.com",
            "annals.org",
            "ahajournals.org",
            "cell.com",
            "science.org",
            "pnas.org",
            "elifesciences.org",
            "embopress.org",
            "journals.asm.org",
            "microbiologyresearch.org",
            "jvi.asm.org",
            "genome.cshlp.org",
            "genetics.org",
            "g3journal.org",
            "plantphysiol.org",
            "plantcell.org",
            "aspb.org",
            "bioone.org",
            "company-of-biologists.org",
            "biologists.org",
            "jeb.biologists.org",
            "dmm.biologists.org",
            "bio.biologists.org",
            "doi.org",
            "ssrn.com",
            "openreview.net",
        ]

        # Check if hostname matches any downloadable domain
        for domain in downloadable_domains:
            if hostname == domain or hostname.endswith("." + domain):
                return True

        # Special case for PubMed which might appear in path
        if "pubmed" in hostname or "/pubmed/" in path:
            return True

        # Check for PDF in path or query parameters
        if "/pdf/" in path or "type=pdf" in query or "format=pdf" in query:
            return True

        return False

    except Exception as e:
        logger.warning(f"Error parsing URL {url}: {e}")
        return False


def get_authenticated_user_password(
    username: str, flask_session_id: str = None
) -> str:
    """
    Get authenticated user password from session store with fallback to g.user_password.

    Args:
        username: The username to get password for
        flask_session_id: Optional Flask session ID. If not provided, uses session.get("_id")

    Returns:
        str: The user's password

    Raises:
        AuthenticationRequiredError: If no password is available for the user
    """
    from ...database.session_passwords import session_password_store
    from ...web.exceptions import AuthenticationRequiredError

    session_id = flask_session_id or session.get("session_id")

    # Try session password store first
    try:
        user_password = session_password_store.get_session_password(
            username, session_id
        )
        if user_password:
            logger.debug(
                f"Retrieved user password from session store for user {username}"
            )
            return user_password
    except Exception:
        logger.exception("Failed to get user password from session store")

    # Fallback to g.user_password (set by middleware if temp_auth was used)
    user_password = getattr(g, "user_password", None)
    if user_password:
        logger.debug(
            f"Retrieved user password from g.user_password fallback for user {username}"
        )
        return user_password

    # No password available
    logger.error(f"No user password available for user {username}")
    raise AuthenticationRequiredError(
        message="Authentication required: Please refresh the page and log in again to access encrypted database features.",
        username=username,
    )


# ============= Page Routes =============


@library_bp.route("/")
@login_required
def library_page():
    """Main library page showing downloaded documents."""
    username = session.get("username")
    service = LibraryService(username)

    # Get library settings
    from ...utilities.db_utils import get_settings_manager

    settings = get_settings_manager()
    pdf_storage_mode = settings.get_setting(
        "research_library.pdf_storage_mode", "database"
    )
    # Enable PDF storage button if mode is not "none"
    enable_pdf_storage = pdf_storage_mode != "none"
    shared_library = settings.get_setting(
        "research_library.shared_library", False
    )

    # Get statistics
    stats = service.get_library_stats()

    # Get documents with optional filters
    domain_filter = request.args.get("domain")
    research_filter = request.args.get("research")
    collection_filter = request.args.get("collection")  # New collection filter

    documents = service.get_documents(
        research_id=research_filter,
        domain=domain_filter,
        collection_id=collection_filter,
        limit=100,
    )

    # Get unique domains for filter dropdown
    unique_domains = service.get_unique_domains()

    # Get research list for filter dropdown
    research_list = service.get_research_list_with_stats()

    # Get collections list for filter dropdown
    collections = service.get_all_collections()

    return render_template_with_defaults(
        "pages/library.html",
        stats=stats,
        documents=documents,
        unique_domains=unique_domains,
        research_list=research_list,
        collections=collections,
        selected_collection=collection_filter,
        storage_path=stats.get("storage_path", ""),
        enable_pdf_storage=enable_pdf_storage,
        pdf_storage_mode=pdf_storage_mode,
        shared_library=shared_library,
    )


@library_bp.route("/document/<string:document_id>")
@login_required
def document_details_page(document_id):
    """Document details page showing all metadata and links."""
    username = session.get("username")
    service = LibraryService(username)

    # Get document details
    document = service.get_document_by_id(document_id)

    if not document:
        return "Document not found", 404

    return render_template_with_defaults(
        "pages/document_details.html", document=document
    )


@library_bp.route("/download-manager")
@login_required
def download_manager_page():
    """Download manager page for selecting and downloading research PDFs."""
    username = session.get("username")
    service = LibraryService(username)

    # Get library settings
    from ...utilities.db_utils import get_settings_manager

    settings = get_settings_manager()
    pdf_storage_mode = settings.get_setting(
        "research_library.pdf_storage_mode", "database"
    )
    # Enable PDF storage button if mode is not "none"
    enable_pdf_storage = pdf_storage_mode != "none"
    shared_library = settings.get_setting(
        "research_library.shared_library", False
    )

    # Get research sessions with statistics
    research_list = service.get_research_list_with_stats()

    # Calculate summary statistics
    total_researches = len(research_list)
    total_resources = sum(r["total_resources"] for r in research_list)
    already_downloaded = sum(r["downloaded_count"] for r in research_list)
    available_to_download = (
        sum(r["downloadable_count"] for r in research_list) - already_downloaded
    )

    # Enrich research data with domain breakdowns
    for research in research_list:
        # Get PDF sources for this research
        documents = service.get_documents(
            research_id=research["id"], file_type="pdf"
        )
        research["pdf_sources"] = documents[:10]  # Preview first 10

        # Domain statistics
        domains = {}
        for doc in documents:
            domain = doc.get("domain", "unknown")
            if domain not in domains:
                domains[domain] = {"total": 0, "pdfs": 0, "downloaded": 0}
            domains[domain]["total"] += 1
            if doc["file_type"] == "pdf":
                domains[domain]["pdfs"] += 1
            if doc["download_status"] == "completed":
                domains[domain]["downloaded"] += 1

        research["domains"] = domains

    return render_template_with_defaults(
        "pages/download_manager.html",
        research_list=research_list,
        total_researches=total_researches,
        total_resources=total_resources,
        already_downloaded=already_downloaded,
        available_to_download=available_to_download,
        enable_pdf_storage=enable_pdf_storage,
        pdf_storage_mode=pdf_storage_mode,
        shared_library=shared_library,
    )


# ============= API Routes =============


@library_bp.route("/api/stats")
@login_required
def get_library_stats():
    """Get library statistics."""
    username = session.get("username")
    service = LibraryService(username)
    stats = service.get_library_stats()
    return jsonify(stats)


@library_bp.route("/api/collections/list")
@login_required
def get_collections_list():
    """Get list of all collections for dropdown selection."""
    username = session.get("username")

    with get_user_db_session(username) as db_session:
        collections = (
            db_session.query(Collection).order_by(Collection.name).all()
        )

        return jsonify(
            {
                "success": True,
                "collections": [
                    {
                        "id": col.id,
                        "name": col.name,
                        "description": col.description,
                    }
                    for col in collections
                ],
            }
        )


@library_bp.route("/api/documents")
@login_required
def get_documents():
    """Get documents with filtering."""
    username = session.get("username")
    service = LibraryService(username)

    # Get filter parameters
    research_id = request.args.get("research_id")
    domain = request.args.get("domain")
    file_type = request.args.get("file_type")
    favorites_only = request.args.get("favorites") == "true"
    search_query = request.args.get("search")
    limit = int(request.args.get("limit", 100))
    offset = int(request.args.get("offset", 0))

    documents = service.get_documents(
        research_id=research_id,
        domain=domain,
        file_type=file_type,
        favorites_only=favorites_only,
        search_query=search_query,
        limit=limit,
        offset=offset,
    )

    return jsonify({"documents": documents})


@library_bp.route(
    "/api/document/<string:document_id>/favorite", methods=["POST"]
)
@login_required
def toggle_favorite(document_id):
    """Toggle favorite status of a document."""
    username = session.get("username")
    service = LibraryService(username)
    is_favorite = service.toggle_favorite(document_id)
    return jsonify({"favorite": is_favorite})


@library_bp.route("/api/document/<string:document_id>", methods=["DELETE"])
@login_required
def delete_document(document_id):
    """Delete a document from library."""
    username = session.get("username")
    service = LibraryService(username)
    success = service.delete_document(document_id)
    return jsonify({"success": success})


@library_bp.route("/api/document/<string:document_id>/pdf-url")
@login_required
def get_pdf_url(document_id):
    """Get URL for viewing PDF."""
    # Return URL that will serve the PDF
    return jsonify(
        {
            "url": f"/library/api/document/{document_id}/pdf",
            "title": "Document",  # Could fetch actual title
        }
    )


@library_bp.route("/document/<string:document_id>/pdf")
@login_required
def view_pdf_page(document_id):
    """Page for viewing PDF file - uses PDFStorageManager for retrieval."""
    username = session.get("username")

    with get_user_db_session(username) as db_session:
        # Get document from database
        document = db_session.query(Document).filter_by(id=document_id).first()

        if not document:
            logger.warning(
                f"Document ID {document_id} not found in database for user {username}"
            )
            return "Document not found", 404

        logger.info(
            f"Document {document_id}: title='{document.title}', "
            f"file_path={document.file_path}"
        )

        # Get settings for PDF storage manager
        settings = get_settings_manager(db_session)
        storage_mode = settings.get_setting(
            "research_library.pdf_storage_mode", "none"
        )
        library_root = Path(
            settings.get_setting(
                "research_library.storage_path",
                str(get_library_directory()),
            )
        ).expanduser()

        # Use PDFStorageManager to load PDF (handles database and filesystem)
        pdf_manager = PDFStorageManager(library_root, storage_mode)
        pdf_bytes = pdf_manager.load_pdf(document, db_session)

        if pdf_bytes:
            logger.info(
                f"Serving PDF for document {document_id} ({len(pdf_bytes)} bytes)"
            )
            return send_file(
                BytesIO(pdf_bytes),
                mimetype="application/pdf",
                as_attachment=False,
                download_name=document.filename or "document.pdf",
            )

        # No PDF found anywhere
        logger.warning(f"No PDF available for document {document_id}")
        return "PDF not available", 404


@library_bp.route("/api/document/<string:document_id>/pdf")
@login_required
def serve_pdf_api(document_id):
    """API endpoint for serving PDF file (kept for backward compatibility)."""
    return view_pdf_page(document_id)


@library_bp.route("/document/<string:document_id>/txt")
@login_required
def view_text_page(document_id):
    """Page for viewing text content."""
    username = session.get("username")

    with get_user_db_session(username) as db_session:
        # Get document by ID (text now stored in Document.text_content)
        document = db_session.query(Document).filter_by(id=document_id).first()

        if not document:
            logger.warning(f"Document not found for document ID {document_id}")
            return "Document not found", 404

        if not document.text_content:
            logger.warning(f"Document {document_id} has no text content")
            return "Text content not available", 404

        logger.info(
            f"Serving text content for document {document_id}: {len(document.text_content)} characters"
        )

        # Render as HTML page
        return render_template_with_defaults(
            "pages/document_text.html",
            document_id=document_id,
            title=document.title or "Document Text",
            text_content=document.text_content,
            extraction_method=document.extraction_method,
            word_count=document.word_count,
        )


@library_bp.route("/api/document/<string:document_id>/text")
@login_required
def serve_text_api(document_id):
    """API endpoint for serving text content (kept for backward compatibility)."""
    username = session.get("username")

    with get_user_db_session(username) as db_session:
        # Get document by ID (text now stored in Document.text_content)
        document = db_session.query(Document).filter_by(id=document_id).first()

        if not document:
            logger.warning(f"Document not found for document ID {document_id}")
            return jsonify({"error": "Document not found"}), 404

        if not document.text_content:
            logger.warning(f"Document {document_id} has no text content")
            return jsonify({"error": "Text content not available"}), 404

        logger.info(
            f"Serving text content for document {document_id}: {len(document.text_content)} characters"
        )

        return jsonify(
            {
                "text_content": document.text_content,
                "title": document.title or "Document",
                "extraction_method": document.extraction_method,
                "word_count": document.word_count,
            }
        )


@library_bp.route("/api/open-folder", methods=["POST"])
@login_required
def open_folder():
    """Open folder containing a document.

    Security: This endpoint is disabled for server deployments.
    It only makes sense for desktop usage where the server and client are on the same machine.
    """
    return jsonify(
        {
            "status": "error",
            "message": "This feature is disabled. It is only available in desktop mode.",
        }
    ), 403


@library_bp.route("/api/download/<int:resource_id>", methods=["POST"])
@login_required
def download_single_resource(resource_id):
    """Download a single resource."""
    username = session.get("username")
    user_password = get_authenticated_user_password(username)

    with DownloadService(username, user_password) as service:
        success, error = service.download_resource(resource_id)
        if success:
            return jsonify({"success": True})
        else:
            logger.warning(
                f"Download failed for resource {resource_id}: {error}"
            )
            return jsonify(
                {
                    "success": False,
                    "error": "Download failed. Please try again or contact support.",
                }
            ), 500


@library_bp.route("/api/download-text/<int:resource_id>", methods=["POST"])
@login_required
def download_text_single(resource_id):
    """Download a single resource as text file."""
    try:
        username = session.get("username")
        user_password = get_authenticated_user_password(username)

        with DownloadService(username, user_password) as service:
            success, error = service.download_as_text(resource_id)

            # Sanitize error message - don't expose internal details
            if not success:
                if error:
                    logger.warning(
                        f"Download as text failed for resource {resource_id}: {error}"
                    )
                return jsonify(
                    {"success": False, "error": "Failed to download resource"}
                )

            return jsonify({"success": True, "error": None})
    except Exception as e:
        return handle_api_error(
            f"downloading resource {resource_id} as text", e
        )


@library_bp.route("/api/download-all-text", methods=["POST"])
@login_required
def download_all_text():
    """Download all undownloaded resources as text files."""
    username = session.get("username")
    # Capture Flask session ID to avoid scoping issues in nested function
    flask_session_id = session.get("session_id")

    def generate():
        # Get user password for database operations
        from ...web.exceptions import AuthenticationRequiredError

        try:
            user_password = get_authenticated_user_password(
                username, flask_session_id
            )
        except AuthenticationRequiredError:
            logger.warning(f"Authentication expired for user {username}")
            return

        download_service = DownloadService(username, user_password)
        try:
            # Get all undownloaded resources
            with get_user_db_session(username) as session:
                # Get resources that don't have text files yet
                resources = session.query(ResearchResource).all()

                # Filter resources that need text extraction
                txt_path = Path(download_service.library_root) / "txt"
                resources_to_process = []

                # Pre-scan directory once to get all existing resource IDs
                existing_resource_ids = set()
                if txt_path.exists():
                    for txt_file in txt_path.glob("*.txt"):
                        # Extract resource ID from filename pattern *_{id}.txt
                        parts = txt_file.stem.rsplit("_", 1)
                        if len(parts) == 2:
                            try:
                                existing_resource_ids.add(int(parts[1]))
                            except ValueError:
                                pass

                for resource in resources:
                    # Check if text file already exists using preloaded set
                    if resource.id not in existing_resource_ids:
                        resources_to_process.append(resource)

                total = len(resources_to_process)
                current = 0

                logger.info(f"Found {total} resources needing text extraction")

                for resource in resources_to_process:
                    current += 1
                    progress = (
                        int((current / total) * 100) if total > 0 else 100
                    )

                    file_name = (
                        resource.title[:50]
                        if resource
                        else f"document_{current}.txt"
                    )

                    try:
                        success, error = download_service.download_as_text(
                            resource.id
                        )

                        if success:
                            status = "success"
                            error_msg = None
                        else:
                            status = "failed"
                            error_msg = error or "Text extraction failed"

                    except Exception as e:
                        logger.exception(
                            f"Error extracting text for resource {resource.id}"
                        )
                        status = "failed"
                        error_msg = (
                            f"Text extraction failed - {type(e).__name__}"
                        )

                    # Send update
                    update = {
                        "progress": progress,
                        "current": current,
                        "total": total,
                        "file": file_name,
                        "url": resource.url,  # Add the URL for UI display
                        "status": status,
                        "error": error_msg,
                    }
                    yield f"data: {json.dumps(update)}\n\n"

                # Send completion
                yield f"data: {json.dumps({'complete': True, 'total': total})}\n\n"
        finally:
            download_service.close()

    return Response(
        stream_with_context(generate()), mimetype="text/event-stream"
    )


@library_bp.route("/api/download-research/<research_id>", methods=["POST"])
@login_required
def download_research_pdfs(research_id):
    """Queue all PDFs from a research session for download."""
    username = session.get("username")
    user_password = get_authenticated_user_password(username)

    with DownloadService(username, user_password) as service:
        # Get optional collection_id from request body
        data = request.json or {}
        collection_id = data.get("collection_id")

        queued = service.queue_research_downloads(research_id, collection_id)

        # Start processing queue (in production, this would be a background task)
        # For now, we'll process synchronously
        # TODO: Integrate with existing queue processor

        return jsonify({"success": True, "queued": queued})


@library_bp.route("/api/download-bulk", methods=["POST"])
@login_required
def download_bulk():
    """Download PDFs or extract text from multiple research sessions."""
    username = session.get("username")
    data = request.json
    research_ids = data.get("research_ids", [])
    mode = data.get("mode", "pdf")  # pdf or text_only
    collection_id = data.get(
        "collection_id"
    )  # Optional: target collection for downloads

    if not research_ids:
        return jsonify({"error": "No research IDs provided"}), 400

    # Capture Flask session ID to avoid scoping issues in nested function
    flask_session_id = session.get("session_id")

    def generate():
        """Generate progress updates as Server-Sent Events."""
        # Get user password for database operations
        from ...web.exceptions import AuthenticationRequiredError

        try:
            user_password = get_authenticated_user_password(
                username, flask_session_id
            )
        except AuthenticationRequiredError:
            return

        download_service = DownloadService(username, user_password)
        try:
            # Count total pending queue items across all research IDs
            total = 0
            current = 0

            with get_user_db_session(username) as session:
                for research_id in research_ids:
                    count = (
                        session.query(LibraryDownloadQueue)
                        .filter_by(
                            research_id=research_id,
                            status=DocumentStatus.PENDING,
                        )
                        .count()
                    )
                    total += count
                    logger.debug(
                        f"[PROGRESS_DEBUG] Research {research_id}: {count} pending items in queue"
                    )

            logger.info(
                f"[PROGRESS_DEBUG] Total pending downloads across all research: {total}"
            )
            yield f"data: {json.dumps({'progress': 0, 'current': 0, 'total': total})}\n\n"

            # Process each research
            for research_id in research_ids:
                # Get queued downloads for this research
                with get_user_db_session(username) as session:
                    # Get pending queue items for this research
                    queue_items = (
                        session.query(LibraryDownloadQueue)
                        .filter_by(
                            research_id=research_id,
                            status=DocumentStatus.PENDING,
                        )
                        .all()
                    )

                    # If no items queued yet, queue them now
                    if not queue_items:
                        try:
                            download_service.queue_research_downloads(
                                research_id, collection_id
                            )
                            # Re-fetch queue items
                            queue_items = (
                                session.query(LibraryDownloadQueue)
                                .filter_by(
                                    research_id=research_id, status="pending"
                                )
                                .all()
                            )
                        except Exception:
                            logger.exception(
                                f"Error queueing downloads for research {research_id}"
                            )
                            # Continue with empty queue_items
                            queue_items = []

                    # Process each queued item
                    for queue_item in queue_items:
                        logger.debug(
                            f"[PROGRESS_DEBUG] Before increment: current={current} (type: {type(current)}), total={total} (type: {type(total)})"
                        )
                        current += 1
                        logger.debug(
                            f"[PROGRESS_DEBUG] After increment: current={current} (type: {type(current)})"
                        )

                        # Check for division issues
                        if total is None:
                            logger.error(
                                "[PROGRESS_DEBUG] ERROR: total is None! Setting to 0 to avoid crash"
                            )
                            total = 0

                        progress = (
                            int((current / total) * 100) if total > 0 else 100
                        )
                        logger.debug(
                            f"[PROGRESS_DEBUG] Calculated progress: {progress}%"
                        )

                        # Get resource info
                        resource = session.query(ResearchResource).get(
                            queue_item.resource_id
                        )
                        file_name = (
                            resource.title[:50]
                            if resource
                            else f"document_{current}.pdf"
                        )

                        # Attempt actual download with error handling
                        skip_reason = None
                        status = "skipped"  # Default to skipped
                        success = False
                        error_msg = None

                        try:
                            logger.debug(
                                f"Attempting {'PDF download' if mode == 'pdf' else 'text extraction'} for resource {queue_item.resource_id}"
                            )

                            # Call appropriate service method based on mode
                            if mode == "pdf":
                                result = download_service.download_resource(
                                    queue_item.resource_id
                                )
                            else:  # text_only
                                result = download_service.download_as_text(
                                    queue_item.resource_id
                                )

                            # Handle new tuple return format
                            if isinstance(result, tuple):
                                success, skip_reason = result
                            else:
                                success = result
                                skip_reason = None

                            status = "success" if success else "skipped"
                            if skip_reason and not success:
                                error_msg = skip_reason
                                logger.info(
                                    f"{'Download' if mode == 'pdf' else 'Text extraction'} skipped for resource {queue_item.resource_id}: {skip_reason}"
                                )

                            logger.debug(
                                f"{'Download' if mode == 'pdf' else 'Text extraction'} result: success={success}, status={status}, skip_reason={skip_reason}"
                            )
                        except Exception as e:
                            # Log error but continue processing
                            error_msg = str(e)
                            error_type = type(e).__name__
                            logger.info(
                                f"CAUGHT Download exception for resource {queue_item.resource_id}: {error_type}: {error_msg}"
                            )
                            # Check if this is a skip reason (not a real error)
                            # Use error category + categorized message for user display
                            if any(
                                phrase in error_msg.lower()
                                for phrase in [
                                    "paywall",
                                    "subscription",
                                    "not available",
                                    "not found",
                                    "no free",
                                    "embargoed",
                                    "forbidden",
                                    "not accessible",
                                ]
                            ):
                                status = "skipped"
                                skip_reason = f"Document not accessible (paywall or access restriction) - {error_type}"
                            elif any(
                                phrase in error_msg.lower()
                                for phrase in [
                                    "failed to download",
                                    "could not",
                                    "invalid",
                                    "server",
                                ]
                            ):
                                status = "failed"
                                skip_reason = f"Download failed - {error_type}"
                            else:
                                status = "failed"
                                skip_reason = (
                                    f"Processing failed - {error_type}"
                                )
                            success = False

                        # Ensure skip_reason is set if we have an error message
                        if error_msg and not skip_reason:
                            skip_reason = f"Processing failed - {error_type}"
                            logger.debug(
                                f"Setting skip_reason from error_msg: {error_msg}"
                            )

                        # Send progress update
                        update_data = {
                            "progress": progress,
                            "current": current,
                            "total": total,
                            "file": file_name,
                            "status": status,
                        }
                        # Add skip reason if available
                        if skip_reason:
                            update_data["error"] = skip_reason
                            logger.info(
                                f"Sending skip reason to UI: {skip_reason}"
                            )

                        logger.info(f"Update data being sent: {update_data}")
                        yield f"data: {json.dumps(update_data)}\n\n"

            yield f"data: {json.dumps({'progress': 100, 'current': total, 'total': total, 'complete': True})}\n\n"
        finally:
            download_service.close()

    return Response(
        stream_with_context(generate()),
        mimetype="text/event-stream",
        headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
    )


@library_bp.route("/api/research-list")
@login_required
def get_research_list():
    """Get list of research sessions with download stats."""
    username = session.get("username")
    service = LibraryService(username)
    research_list = service.get_research_list_with_stats()
    return jsonify({"research": research_list})


@library_bp.route("/api/sync-library", methods=["POST"])
@login_required
def sync_library():
    """Sync library database with filesystem."""
    username = session.get("username")
    service = LibraryService(username)
    stats = service.sync_library_with_filesystem()
    return jsonify(stats)


@library_bp.route("/api/mark-redownload", methods=["POST"])
@login_required
def mark_for_redownload():
    """Mark documents for re-download."""
    username = session.get("username")
    service = LibraryService(username)

    data = request.json
    document_ids = data.get("document_ids", [])

    if not document_ids:
        return jsonify({"error": "No document IDs provided"}), 400

    count = service.mark_for_redownload(document_ids)
    return jsonify({"success": True, "marked": count})


@library_bp.route("/api/queue-all-undownloaded", methods=["POST"])
@login_required
def queue_all_undownloaded():
    """Queue all articles that haven't been downloaded yet."""
    username = session.get("username")

    logger.info(f"queue_all_undownloaded called for user {username}")

    with get_user_db_session(username) as db_session:
        # Find all resources that don't have a completed download
        undownloaded = (
            db_session.query(ResearchResource)
            .outerjoin(
                Document,
                (ResearchResource.id == Document.resource_id)
                & (Document.status == "completed"),
            )
            .filter(Document.id.is_(None))
            .all()
        )

        logger.info(f"Found {len(undownloaded)} total undownloaded resources")

        # Get user password for encrypted database access
        user_password = get_authenticated_user_password(username)

        resource_filter = ResourceFilter(username, user_password)
        filter_results = resource_filter.filter_downloadable_resources(
            undownloaded
        )

        # Get detailed filtering summary
        filter_summary = resource_filter.get_filter_summary(undownloaded)
        skipped_info = resource_filter.get_skipped_resources_info(undownloaded)

        logger.info(f"Filter results: {filter_summary.to_dict()}")

        queued_count = 0
        research_ids = set()
        skipped_count = 0

        # Convert filter_results to dict for O(1) lookup instead of O(n²)
        filter_results_by_id = {r.resource_id: r for r in filter_results}

        for resource in undownloaded:
            # Check if resource passed the smart filter
            filter_result = filter_results_by_id.get(resource.id)

            if not filter_result or not filter_result.can_retry:
                skipped_count += 1
                if filter_result:
                    logger.debug(
                        f"Skipping resource {resource.id} due to retry policy: {filter_result.reason}"
                    )
                else:
                    logger.debug(
                        f"Skipping resource {resource.id} - no filter result available"
                    )
                continue

            # Check if it's downloadable using proper URL parsing
            if not resource.url:
                skipped_count += 1
                continue

            is_downloadable = is_downloadable_domain(resource.url)

            # Log what we're checking
            if resource.url and "pubmed" in resource.url.lower():
                logger.info(f"Found PubMed URL: {resource.url[:100]}")

            if not is_downloadable:
                skipped_count += 1
                logger.debug(
                    f"Skipping non-downloadable URL: {resource.url[:100] if resource.url else 'None'}"
                )
                continue

            # Check if already in queue (any status)
            existing_queue = (
                db_session.query(LibraryDownloadQueue)
                .filter_by(resource_id=resource.id)
                .first()
            )

            if existing_queue:
                # If it exists but isn't pending, reset it to pending
                if existing_queue.status != DocumentStatus.PENDING:
                    existing_queue.status = DocumentStatus.PENDING
                    existing_queue.completed_at = None
                    queued_count += 1
                    research_ids.add(resource.research_id)
                    logger.debug(
                        f"Reset queue entry for resource {resource.id} to pending"
                    )
                else:
                    # Already pending, still count it
                    queued_count += 1
                    research_ids.add(resource.research_id)
                    logger.debug(
                        f"Resource {resource.id} already pending in queue"
                    )
            else:
                # Add new entry to queue
                queue_entry = LibraryDownloadQueue(
                    resource_id=resource.id,
                    research_id=resource.research_id,
                    priority=0,
                    status=DocumentStatus.PENDING,
                )
                db_session.add(queue_entry)
                queued_count += 1
                research_ids.add(resource.research_id)
                logger.debug(
                    f"Added new queue entry for resource {resource.id}"
                )

        db_session.commit()

        logger.info(
            f"Queued {queued_count} articles for download, skipped {skipped_count} resources (including {filter_summary.permanently_failed_count} permanently failed and {filter_summary.temporarily_failed_count} temporarily failed)"
        )

        # Note: Removed synchronous download processing here to avoid blocking the HTTP request
        # Downloads will be processed via the SSE streaming endpoint or background tasks

        return jsonify(
            {
                "success": True,
                "queued": queued_count,
                "research_ids": list(research_ids),
                "total_undownloaded": len(undownloaded),
                "skipped": skipped_count,
                "filter_summary": filter_summary.to_dict(),
                "skipped_details": skipped_info,
            }
        )


@library_bp.route("/api/get-research-sources/<research_id>", methods=["GET"])
@login_required
def get_research_sources(research_id):
    """Get all sources for a research with snippets."""
    username = session.get("username")

    sources = []
    with get_user_db_session(username) as db_session:
        # Get all resources for this research
        resources = (
            db_session.query(ResearchResource)
            .filter_by(research_id=research_id)
            .order_by(ResearchResource.created_at)
            .all()
        )

        for idx, resource in enumerate(resources, 1):
            # Check if document exists
            document = (
                db_session.query(Document)
                .filter_by(resource_id=resource.id)
                .first()
            )

            # Get domain from URL
            domain = ""
            if resource.url:
                try:
                    from urllib.parse import urlparse

                    domain = urlparse(resource.url).hostname or ""
                except (ValueError, AttributeError):
                    # urlparse can raise ValueError for malformed URLs
                    pass

            source_data = {
                "number": idx,
                "resource_id": resource.id,
                "url": resource.url,
                "title": resource.title or f"Source {idx}",
                "snippet": resource.content_preview or "",
                "domain": domain,
                "relevance_score": getattr(resource, "relevance_score", None),
                "downloaded": False,
                "document_id": None,
                "file_type": None,
            }

            if document and document.status == "completed":
                source_data.update(
                    {
                        "downloaded": True,
                        "document_id": document.id,
                        "file_type": document.file_type,
                        "download_date": document.created_at.isoformat()
                        if document.created_at
                        else None,
                    }
                )

            sources.append(source_data)

    return jsonify({"success": True, "sources": sources, "total": len(sources)})


@library_bp.route("/api/check-downloads", methods=["POST"])
@login_required
def check_downloads():
    """Check download status for a list of URLs."""
    username = session.get("username")
    data = request.json
    research_id = data.get("research_id")
    urls = data.get("urls", [])

    if not research_id or not urls:
        return jsonify({"error": "Missing research_id or urls"}), 400

    download_status = {}

    with get_user_db_session(username) as db_session:
        # Get all resources for this research
        resources = (
            db_session.query(ResearchResource)
            .filter_by(research_id=research_id)
            .filter(ResearchResource.url.in_(urls))
            .all()
        )

        for resource in resources:
            # Check if document exists
            document = (
                db_session.query(Document)
                .filter_by(resource_id=resource.id)
                .first()
            )

            if document and document.status == "completed":
                download_status[resource.url] = {
                    "downloaded": True,
                    "document_id": document.id,
                    "file_path": document.file_path,
                    "file_type": document.file_type,
                    "title": document.title or resource.title,
                }
            else:
                download_status[resource.url] = {
                    "downloaded": False,
                    "resource_id": resource.id,
                }

    return jsonify({"download_status": download_status})


@library_bp.route("/api/download-source", methods=["POST"])
@login_required
def download_source():
    """Download a single source from a research."""
    username = session.get("username")
    user_password = get_authenticated_user_password(username)
    data = request.json
    research_id = data.get("research_id")
    url = data.get("url")

    if not research_id or not url:
        return jsonify({"error": "Missing research_id or url"}), 400

    # Check if URL is downloadable
    if not is_downloadable_domain(url):
        return jsonify({"error": "URL is not from a downloadable domain"}), 400

    with get_user_db_session(username) as db_session:
        # Find the resource
        resource = (
            db_session.query(ResearchResource)
            .filter_by(research_id=research_id, url=url)
            .first()
        )

        if not resource:
            return jsonify({"error": "Resource not found"}), 404

        # Check if already downloaded
        existing = (
            db_session.query(Document)
            .filter_by(resource_id=resource.id)
            .first()
        )

        if existing and existing.download_status == "completed":
            return jsonify(
                {
                    "success": True,
                    "message": "Already downloaded",
                    "document_id": existing.id,
                }
            )

        # Add to download queue
        queue_entry = (
            db_session.query(LibraryDownloadQueue)
            .filter_by(resource_id=resource.id)
            .first()
        )

        if not queue_entry:
            queue_entry = LibraryDownloadQueue(
                resource_id=resource.id,
                research_id=resource.research_id,
                priority=1,  # Higher priority for manual downloads
                status=DocumentStatus.PENDING,
            )
            db_session.add(queue_entry)
        else:
            queue_entry.status = DocumentStatus.PENDING
            queue_entry.priority = 1

        db_session.commit()

        # Start download immediately
        with DownloadService(username, user_password) as service:
            success, message = service.download_resource(resource.id)

            if success:
                return jsonify(
                    {"success": True, "message": "Download completed"}
                )
            else:
                # Log internal message, but show only generic message to user
                return jsonify({"success": False, "message": "Download failed"})
