#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
OneCite Processing Pipeline Module
Implementation of various modules for the 4-stage processing pipeline
"""

import re
import os
import logging
import time
from typing import List, Dict, Optional, Callable, Any
import requests
from bs4 import BeautifulSoup
import bibtexparser
from thefuzz import fuzz
from scholarly import scholarly

from .core import RawEntry, IdentifiedEntry, CompletedEntry
from .exceptions import ParseError, ResolverError


class ParserModule:
    """Stage 1: Parse and Extract Module"""
    
    def __init__(self):
        self.logger = logging.getLogger(__name__)
    
    def parse(self, input_content: str, input_type: str) -> List[RawEntry]:
        """
        Parse input content into a list of raw entries
        
        Args:
            input_content: Input content string
            input_type: Input type ('txt' or 'bib')
        
        Returns:
            List of raw entries
        """
        self.logger.info(f"Starting to parse {input_type} format input content")
        
        if input_type.lower() == 'bib':
            return self._parse_bibtex(input_content)
        elif input_type.lower() == 'txt':
            return self._parse_text(input_content)
        else:
            raise ParseError(f"Unsupported input type: {input_type}")
    
    def _parse_bibtex(self, bibtex_content: str) -> List[RawEntry]:
        """Parse BibTeX format content"""
        entries = []
        try:
            bib_database = bibtexparser.loads(bibtex_content)
            for i, entry in enumerate(bib_database.entries):
                raw_entry: RawEntry = {
                    'id': i,
                    'raw_text': str(entry),
                    'doi': entry.get('doi'),
                    'url': entry.get('url'),
                    'query_string': None
                }
                
                # If no DOI is available, generate query string
                if not raw_entry['doi']:
                    query_parts = []
                    if 'title' in entry:
                        query_parts.append(entry['title'])
                    if 'author' in entry:
                        query_parts.append(entry['author'])
                    if 'year' in entry:
                        query_parts.append(entry['year'])
                    raw_entry['query_string'] = ' '.join(query_parts)
                
                entries.append(raw_entry)
            
            self.logger.info(f"Successfully parsed {len(entries)} BibTeX entries")
            return entries
            
        except Exception as e:
            self.logger.error(f"BibTeX parsing failed: {str(e)}")
            raise ParseError(f"BibTeX parsing failed: {str(e)}")
    
    def _parse_text(self, text_content: str) -> List[RawEntry]:
        """Parse plain text format content"""
        entries = []
        
        # Split text blocks using double newlines
        text_blocks = text_content.split('\n\n')
        
        for i, block in enumerate(text_blocks):
            block = block.strip()
            if not block:
                continue
            
            raw_entry: RawEntry = {
                'id': i,
                'raw_text': block,
                'doi': None,
                'url': None,
                'query_string': None
            }
            
            # Find DOI (standard format)
            doi_match = re.search(r'10\.\d{4,}/[^\s,}]+', block)
            if doi_match:
                raw_entry['doi'] = doi_match.group()
            else:
                # Try to find article ID patterns that might be convertible to DOI
                # Common patterns: e0000429, PMC123456, etc.
                article_id_match = re.search(r'\b[eE]\d{7}\b', block)  # PLOS style: e0000429
                if article_id_match:
                    article_id = article_id_match.group()
                    # Note potential PLOS article ID but don't assume specific journal
                    # Let CrossRef resolve the actual DOI during identification
                    self.logger.info(f"Entry {i} found potential PLOS article ID {article_id}, will attempt resolution via CrossRef")
                    # Add to query string instead of assuming DOI
                    if not raw_entry['query_string']:
                        raw_entry['query_string'] = block
            
            # Find URL
            url_match = re.search(r'https?://[^\s]+', block)
            if url_match:
                raw_entry['url'] = url_match.group()
            
            # If no DOI or URL found, build a concise query string from title/author/year
            if not raw_entry['doi'] and not raw_entry['url']:
                lines = [ln.strip() for ln in block.splitlines() if ln.strip()]
                title_text = lines[0] if lines else block
                authors_text = lines[1] if len(lines) > 1 else ''
                year_match = re.search(r'(19|20)\d{2}', block)
                year_text = year_match.group(0) if year_match else ''

                query_parts: List[str] = []
                if title_text:
                    query_parts.append(title_text)
                if authors_text:
                    query_parts.append(authors_text)
                if year_text:
                    query_parts.append(year_text)

                raw_entry['query_string'] = ' '.join(query_parts) or block
            
            entries.append(raw_entry)
        
        self.logger.info(f"Successfully parsed {len(entries)} text entries")
        return entries


class IdentifierModule:
    """Stage 2: Identification and Standardization Module"""
    
    def __init__(self, use_google_scholar: bool = False):
        self.logger = logging.getLogger(__name__)
        self.crossref_base_url = "https://api.crossref.org/works"
        self.use_google_scholar = use_google_scholar
        
        # Well-known papers that might not have DOIs
        self.well_known_papers = {
            'attention is all you need': {
                'title': 'Attention Is All You Need',
                'authors': ['Vaswani, Ashish', 'Shazeer, Noam', 'Parmar, Niki', 'Uszkoreit, Jakob', 
                           'Jones, Llion', 'Gomez, Aidan N', 'Kaiser, Lukasz', 'Polosukhin, Illia'],
                'year': '2017',
                'journal': 'Advances in Neural Information Processing Systems',
                'arxiv_id': '1706.03762',
                'url': 'https://arxiv.org/abs/1706.03762',
                'type': 'conference'
            }
        }
    
    def identify(self, raw_entries: List[RawEntry], 
                interactive_callback: Callable[[List[Dict]], int]) -> List[IdentifiedEntry]:
        """
        Identify and standardize entries, finding DOI for each entry
        
        Args:
            raw_entries: List of raw entries
            interactive_callback: Interactive callback function
        
        Returns:
            List of identified entries
        """
        self.logger.info(f"Starting to identify {len(raw_entries)} entries")
        identified_entries = []
        
        for entry in raw_entries:
            identified_entry = self._identify_single_entry(entry, interactive_callback)
            identified_entries.append(identified_entry)
        
        successful_count = sum(1 for e in identified_entries if e['status'] == 'identified')
        self.logger.info(f"Identification completed: {successful_count}/{len(identified_entries)} entries successfully identified")
        
        return identified_entries
    
    def _identify_single_entry(self, raw_entry: RawEntry, 
                              interactive_callback: Callable[[List[Dict]], int]) -> IdentifiedEntry:
        """Identify a single entry"""
        identified_entry: IdentifiedEntry = {
            'id': raw_entry['id'],
            'raw_text': raw_entry['raw_text'],
            'doi': None,
            'arxiv_id': None,
            'url': None,
            'metadata': {},
            'status': 'identification_failed'
        }
        
        # If valid DOI already exists, verify it against CrossRef API
        if raw_entry.get('doi'):
            if self._validate_doi(raw_entry['doi']):
                # DOI format is valid, now verify it exists and get real metadata
                real_metadata = self._verify_doi_and_get_metadata(raw_entry['doi'])
                if real_metadata:
                    # Compare user input with real metadata for AI detection
                    consistency_score = self._check_doi_content_consistency(raw_entry['raw_text'], real_metadata)
                    
                    identified_entry['doi'] = raw_entry['doi']
                    identified_entry['metadata'] = real_metadata
                    identified_entry['metadata']['consistency_score'] = consistency_score
                    identified_entry['status'] = 'identified'
                    
                    if consistency_score < 70:
                        self.logger.warning(f"Entry {raw_entry['id']} DOI verified but content inconsistent (score: {consistency_score}). Possible AI-generated fake reference.")
                        identified_entry['metadata']['warning'] = 'low_consistency'
                        
                        # Reject the reference if consistency score is too low
                        # But allow DOI-only entries to pass through
                        if consistency_score < 20 and len(raw_entry['raw_text'].strip()) > 20:
                            self.logger.error(f"Entry {raw_entry['id']} consistency score too low ({consistency_score}), marking as failed")
                            identified_entry['status'] = 'identification_failed'
                            return identified_entry
                    else:
                        self.logger.info(f"Entry {raw_entry['id']} DOI verified with good consistency (score: {consistency_score})")
                    
                    return identified_entry
                else:
                    self.logger.warning(f"Entry {raw_entry['id']} has valid DOI format but DOI does not exist: {raw_entry['doi']}")
                    # Continue to fuzzy search as fallback
        
        # Check for arXiv ID in raw text
        arxiv_id = self._extract_arxiv_id(raw_entry['raw_text'])
        if arxiv_id:
            identified_entry['arxiv_id'] = arxiv_id
            identified_entry['status'] = 'identified'
            self.logger.info(f"Entry {raw_entry['id']} has arXiv ID: {arxiv_id}")
            return identified_entry
        
        # Try to extract DOI or arXiv ID from URL
        if raw_entry.get('url'):
            # Check if it's an arXiv URL
            if 'arxiv.org' in raw_entry['url']:
                arxiv_id = self._extract_arxiv_id_from_url(raw_entry['url'])
                if arxiv_id:
                    identified_entry['arxiv_id'] = arxiv_id
                    identified_entry['url'] = raw_entry['url']
                    identified_entry['status'] = 'identified'
                    self.logger.info(f"Entry {raw_entry['id']} extracted arXiv ID from URL: {arxiv_id}")
                    return identified_entry
            else:
                # Try to extract DOI
                extracted_doi = self._extract_doi_from_url(raw_entry['url'])
                if extracted_doi:
                    identified_entry['doi'] = extracted_doi
                    identified_entry['status'] = 'identified'
                    self.logger.info(f"Entry {raw_entry['id']} extracted DOI from URL: {extracted_doi}")
                    return identified_entry
                
                # Try to extract metadata from PDF or HTML page
                url_metadata = self._extract_metadata_from_url(raw_entry['url'])
                if url_metadata:
                    identified_entry['metadata'] = url_metadata
                    identified_entry['status'] = 'identified'
                    identified_entry['url'] = raw_entry['url']
                    self.logger.info(f"Entry {raw_entry['id']} extracted metadata from URL")
                    return identified_entry
                
                # Store URL for conference papers
                identified_entry['url'] = raw_entry['url']
        
        # Fuzzy search
        if raw_entry.get('query_string'):
            return self._fuzzy_search(raw_entry, interactive_callback)
        
        self.logger.warning(f"Entry {raw_entry['id']} identification failed")
        return identified_entry
    
    def _validate_doi(self, doi: str) -> bool:
        """Validate DOI format"""
        doi_pattern = r'^10\.\d{4,}/.+'
        return bool(re.match(doi_pattern, doi))
    
    def _verify_doi_and_get_metadata(self, doi: str) -> Optional[Dict]:
        """Verify DOI exists in CrossRef and get real metadata for comparison"""
        try:
            url = f"{self.crossref_base_url}/{doi}"
            headers = {'Accept': 'application/json'}
            
            response = requests.get(url, headers=headers, timeout=10)
            response.raise_for_status()
            
            data = response.json()
            work = data.get('message', {})
            
            # Extract real metadata from CrossRef
            real_metadata = {
                'source': 'crossref_verification',
                'doi': work.get('DOI'),
                'title': work.get('title', [''])[0] if work.get('title') else '',
                'authors': [f"{a.get('given', '')} {a.get('family', '')}" 
                          for a in work.get('author', [])],
                'year': work.get('published-print', {}).get('date-parts', [[None]])[0][0] or
                       work.get('published-online', {}).get('date-parts', [[None]])[0][0],
                'journal': work.get('container-title', [''])[0] if work.get('container-title') else '',
                'volume': work.get('volume'),
                'number': work.get('issue'),
                'pages': work.get('page'),
                'publisher': work.get('publisher'),
                'citations': work.get('is-referenced-by-count', 0),
                'url': work.get('URL')
            }
            
            self.logger.info(f"DOI {doi} verified successfully in CrossRef")
            return real_metadata
            
        except requests.exceptions.HTTPError as e:
            if e.response.status_code == 404:
                self.logger.warning(f"DOI {doi} not found in CrossRef (404)")
                return None
            else:
                self.logger.error(f"HTTP error verifying DOI {doi}: {str(e)}")
                return None
        except Exception as e:
            self.logger.error(f"Failed to verify DOI {doi}: {str(e)}")
            return None
    
    def _check_doi_content_consistency(self, user_input: str, real_metadata: Dict) -> float:
        """Check consistency between user input and real DOI metadata to detect AI-generated fake references"""
        try:
            # Normalize user input
            user_input_lower = user_input.lower()
            
            # Extract real information
            real_title = real_metadata.get('title', '').lower()
            real_authors = [author.lower() for author in real_metadata.get('authors', [])]
            real_year = str(real_metadata.get('year', ''))
            real_journal = real_metadata.get('journal', '').lower()
            
            # Calculate consistency scores for different fields
            scores = []
            
            # Title consistency (most important)
            if real_title:
                title_score = max(
                    fuzz.ratio(user_input_lower, real_title),
                    fuzz.partial_ratio(user_input_lower, real_title),
                    fuzz.token_set_ratio(user_input_lower, real_title)
                )
                scores.append(('title', title_score, 0.4))  # 40% weight
            
            # Author consistency
            if real_authors:
                author_scores = []
                for real_author in real_authors:
                    author_score = max(
                        fuzz.partial_ratio(user_input_lower, real_author),
                        fuzz.token_set_ratio(user_input_lower, real_author)
                    )
                    author_scores.append(author_score)
                best_author_score = max(author_scores) if author_scores else 0
                scores.append(('author', best_author_score, 0.3))  # 30% weight
            
            # Year consistency
            if real_year and real_year in user_input:
                scores.append(('year', 100, 0.2))  # 20% weight
            elif real_year:
                scores.append(('year', 0, 0.2))
            
            # Journal consistency
            if real_journal:
                journal_score = max(
                    fuzz.partial_ratio(user_input_lower, real_journal),
                    fuzz.token_set_ratio(user_input_lower, real_journal)
                )
                scores.append(('journal', journal_score, 0.1))  # 10% weight
            
            # Calculate weighted average
            if not scores:
                return 0.0
            
            total_weighted_score = sum(score * weight for _, score, weight in scores)
            total_weight = sum(weight for _, _, weight in scores)
            
            final_score = total_weighted_score / total_weight if total_weight > 0 else 0.0
            
            # Log detailed scores for debugging
            score_details = {field: score for field, score, _ in scores}
            self.logger.info(f"DOI consistency check details: {score_details}, final: {final_score:.2f}")
            
            return round(final_score, 2)
            
        except Exception as e:
            self.logger.error(f"Error in DOI content consistency check: {str(e)}")
            return 0.0
    
    def _extract_arxiv_id(self, text: str) -> Optional[str]:
        """Extract arXiv ID from text"""
        # Match both old (e.g., 1706.03762) and new (e.g., arxiv:1706.03762) formats
        arxiv_patterns = [
            r'arxiv[:\s]*(\d{4}\.\d{4,5})',  # New format
            r'\b(\d{4}\.\d{4,5})\b',  # Standalone ID
            r'arXiv:(\d{4}\.\d{4,5})',  # With arXiv prefix
        ]
        
        for pattern in arxiv_patterns:
            match = re.search(pattern, text, re.IGNORECASE)
            if match:
                return match.group(1)
        return None
    
    def _extract_arxiv_id_from_url(self, url: str) -> Optional[str]:
        """Extract arXiv ID from arXiv URL"""
        # Match patterns like https://arxiv.org/abs/1706.03762
        match = re.search(r'arxiv\.org/(?:abs|pdf)/(\d{4}\.\d{4,5})', url)
        if match:
            return match.group(1)
        return None
    
    def _extract_doi_from_url(self, url: str) -> Optional[str]:
        """Extract DOI from URL page"""
        try:
            response = requests.get(url, timeout=10, headers={'User-Agent': 'OneCite/1.0'})
            soup = BeautifulSoup(response.content, 'html.parser')
            
            # Look for DOI in meta tags
            doi_meta = soup.find('meta', attrs={'name': 'citation_doi'}) or \
                      soup.find('meta', attrs={'name': 'dc.identifier'}) or \
                      soup.find('meta', attrs={'property': 'citation_doi'})
            
            if doi_meta and 'content' in doi_meta.attrs:
                doi = doi_meta['content']
                if self._validate_doi(doi):
                    return doi
            
            # Search for DOI in page content
            doi_match = re.search(r'10\.\d{4,}/[^\s"<>,}]+', response.text)
            if doi_match:
                doi = doi_match.group()
                if self._validate_doi(doi):
                    return doi
                    
        except Exception as e:
            self.logger.warning(f"Failed to extract DOI from URL {url}: {str(e)}")
        
        return None
    
    def _extract_metadata_from_url(self, url: str) -> Optional[Dict]:
        """Extract metadata from PDF or HTML page"""
        try:
            response = requests.get(url, timeout=15, headers={'User-Agent': 'OneCite/1.0'})
            response.raise_for_status()
            
            # Check if it's a PDF
            content_type = response.headers.get('content-type', '').lower()
            if 'pdf' in content_type or url.lower().endswith('.pdf'):
                return self._extract_from_pdf_content(response.content)
            else:
                return self._extract_from_html_content(response.content)
                
        except Exception as e:
            self.logger.warning(f"Failed to extract metadata from URL {url}: {str(e)}")
            return None
    
    def _extract_from_html_content(self, content: bytes) -> Optional[Dict]:
        """Extract metadata from HTML content"""
        try:
            soup = BeautifulSoup(content, 'html.parser')
            metadata = {}
            
            # Look for academic metadata in meta tags
            meta_mappings = {
                'title': ['citation_title', 'dc.title', 'og:title'],
                'author': ['citation_author', 'dc.creator', 'author'],
                'journal': ['citation_journal_title', 'dc.source', 'citation_conference_title'],
                'year': ['citation_publication_date', 'citation_date', 'dc.date'],
                'abstract': ['citation_abstract', 'dc.description', 'description'],
                'volume': ['citation_volume'],
                'pages': ['citation_firstpage', 'citation_lastpage']
            }
            
            authors = []
            for field, tag_names in meta_mappings.items():
                for tag_name in tag_names:
                    metas = soup.find_all('meta', attrs={'name': tag_name}) + \
                           soup.find_all('meta', attrs={'property': tag_name})
                    
                    for meta in metas:
                        if meta.get('content'):
                            content_value = meta['content'].strip()
                            if not content_value:
                                continue
                                
                            if field == 'author':
                                authors.append(content_value)
                            elif field == 'year':
                                year_match = re.search(r'\b(19|20)\d{2}\b', content_value)
                                if year_match:
                                    metadata[field] = int(year_match.group())
                            elif field == 'journal':
                                # Don't overwrite if already found
                                if field not in metadata:
                                    metadata[field] = content_value
                            else:
                                metadata[field] = content_value
                            
                            # For non-author fields, break after finding first valid value
                            if field != 'author':
                                break
                    
                    # For non-author fields, break after finding value from any tag
                    if field != 'author' and field in metadata:
                        break
            
            # Process authors
            if authors:
                # Clean up author names and join them
                cleaned_authors = []
                for author in authors:
                    # Remove extra whitespace and common prefixes
                    author = re.sub(r'^\s*(by\s+)?', '', author, flags=re.IGNORECASE).strip()
                    if author and len(author) > 2:
                        cleaned_authors.append(author)
                
                if cleaned_authors:
                    metadata['author'] = ' and '.join(cleaned_authors)
            
            # If no title found, try page title
            if 'title' not in metadata:
                title_tag = soup.find('title')
                if title_tag:
                    title = title_tag.get_text().strip()
                    # Clean up common title suffixes
                    title = re.sub(r'\s*[-|]\s*(PDF|Download|Paper|Abstract).*$', '', title, flags=re.IGNORECASE)
                    if len(title) > 10:
                        metadata['title'] = title
            
            # If still no authors, try to extract from page content
            if 'author' not in metadata:
                authors_from_content = self._extract_authors_from_content(soup)
                if authors_from_content:
                    metadata['author'] = authors_from_content
            
            # Extract year from title or content if not found
            if 'year' not in metadata:
                year_from_content = self._extract_year_from_content(soup, metadata.get('title', ''))
                if year_from_content:
                    metadata['year'] = year_from_content
            
            return metadata if len(metadata) >= 1 else None
            
        except Exception as e:
            self.logger.warning(f"Failed to extract from HTML: {str(e)}")
            return None
    
    def _extract_authors_from_content(self, soup: BeautifulSoup) -> Optional[str]:
        """Extract authors from page content when meta tags are not available"""
        try:
            # Look for author-related elements
            author_selectors = [
                '[class*="author"]',
                '[class*="byline"]', 
                '[id*="author"]',
                '.authors',
                '.author-list'
            ]
            
            for selector in author_selectors:
                elements = soup.select(selector)
                for elem in elements:
                    text = elem.get_text().strip()
                    if text and 10 <= len(text) <= 200:
                        # Clean up the text
                        text = re.sub(r'^\s*(authors?|by)\s*:?\s*', '', text, flags=re.IGNORECASE)
                        # Look for name patterns
                        if re.search(r'[A-Z][a-z]+\s+[A-Z][a-z]+', text):
                            return text
            
            # Try pattern matching in the full text
            page_text = soup.get_text()
            
            # Pattern 1: "By Author Name"
            by_pattern = r'[Bb]y\s+([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+(?:\s*,\s*[A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)*)'
            match = re.search(by_pattern, page_text)
            if match:
                return match.group(1)
            
            # Pattern 2: "Authors: Name1, Name2"
            authors_pattern = r'[Aa]uthors?\s*:\s*([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*(?:\s*,\s*[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)*)'
            match = re.search(authors_pattern, page_text)
            if match:
                return match.group(1)
                
        except Exception as e:
            self.logger.warning(f"Failed to extract authors from content: {str(e)}")
        
        return None
    
    def _extract_year_from_content(self, soup: BeautifulSoup, title: str = '') -> Optional[int]:
        """Extract publication year from content"""
        try:
            # First try to find year in title
            if title:
                year_match = re.search(r'\b(19|20)\d{2}\b', title)
                if year_match:
                    return int(year_match.group())
            
            # Look for year in specific elements
            year_selectors = [
                '[class*="year"]',
                '[class*="date"]',
                '.publication-date',
                '.pub-date'
            ]
            
            for selector in year_selectors:
                elements = soup.select(selector)
                for elem in elements:
                    text = elem.get_text()
                    year_match = re.search(r'\b(19|20)\d{2}\b', text)
                    if year_match:
                        return int(year_match.group())
            
            # Try to find year in the first few paragraphs
            paragraphs = soup.find_all('p')[:5]
            for p in paragraphs:
                text = p.get_text()
                year_match = re.search(r'\b(19|20)\d{2}\b', text)
                if year_match:
                    year = int(year_match.group())
                    # Only accept reasonable years for academic papers
                    if 1950 <= year <= 2030:
                        return year
                        
        except Exception as e:
            self.logger.warning(f"Failed to extract year from content: {str(e)}")
        
        return None
    
    def _extract_from_pdf_content(self, content: bytes) -> Optional[Dict]:
        """Extract metadata from PDF content"""
        try:
            import PyPDF2
            import io
            
            pdf_file = io.BytesIO(content)
            pdf_reader = PyPDF2.PdfReader(pdf_file)
            metadata = {}
            
            # Extract from PDF metadata
            if pdf_reader.metadata:
                pdf_meta = pdf_reader.metadata
                if pdf_meta.get('/Title'):
                    title = str(pdf_meta['/Title']).strip()
                    if len(title) > 5:
                        metadata['title'] = title
                if pdf_meta.get('/Author'):
                    author = str(pdf_meta['/Author']).strip()
                    if len(author) > 3:
                        metadata['author'] = author
            
            # Extract from first page text
            if len(pdf_reader.pages) > 0:
                try:
                    first_page_text = pdf_reader.pages[0].extract_text()
                    if first_page_text:
                        lines = [line.strip() for line in first_page_text.split('\n') if line.strip()]
                        
                        # Try to find title (usually one of the first few lines)
                        if 'title' not in metadata:
                            for line in lines[:5]:
                                if 20 <= len(line) <= 200 and not line.isupper():
                                    # Skip lines that look like headers/footers
                                    if not re.search(r'(page|abstract|introduction|©|\d+)', line.lower()):
                                        metadata['title'] = line
                                        break
                        
                        # Try to extract year
                        year_match = re.search(r'\b(19|20)\d{2}\b', first_page_text)
                        if year_match:
                            metadata['year'] = int(year_match.group())
                            
                except Exception as e:
                    self.logger.warning(f"Failed to extract text from PDF: {str(e)}")
            
            return metadata if metadata else None
            
        except ImportError:
            self.logger.warning("PyPDF2 not available for PDF parsing")
            return None
        except Exception as e:
            self.logger.warning(f"Failed to extract from PDF: {str(e)}")
            return None
    
    def _fuzzy_search(self, raw_entry: RawEntry, 
                     interactive_callback: Callable[[List[Dict]], int]) -> IdentifiedEntry:
        """Perform fuzzy search"""
        query_string = raw_entry['query_string']
        
        # Check if it's a well-known paper first
        query_lower = query_string.lower()
        for key, paper_data in self.well_known_papers.items():
            if key in query_lower or fuzz.ratio(key, query_lower) > 85:
                self.logger.info(f"Entry {raw_entry['id']} matched well-known paper: {paper_data['title']}")
                return {
                    'id': raw_entry['id'],
                    'raw_text': raw_entry['raw_text'],
                    'doi': None,
                    'arxiv_id': paper_data.get('arxiv_id'),
                    'url': paper_data.get('url'),
                    'metadata': paper_data,
                    'status': 'identified'
                }
        
        # Multi-source query with intelligent fallback
        candidates = []
        
        # CrossRef search (primary, fast and reliable)
        crossref_results = self._search_crossref(query_string)
        candidates.extend(crossref_results)
        
        # If CrossRef didn't find good results, try Google Scholar as fallback
        if len(crossref_results) == 0 or (crossref_results and max(c.get('citations', 0) for c in crossref_results) < 10):
            self.logger.info("CrossRef results insufficient, checking Google Scholar")
            if self.use_google_scholar:
                self.logger.info("Google Scholar enabled, searching...")
                scholar_results = self._search_google_scholar(query_string)
                candidates.extend(scholar_results)
            else:
                self.logger.info("Google Scholar disabled by user, skipping")
        else:
            self.logger.info(f"CrossRef found {len(crossref_results)} good results, skipping Google Scholar")
        
        if not candidates:
            self.logger.warning(f"Entry {raw_entry['id']}: no candidate results found")
            return {
                'id': raw_entry['id'],
                'raw_text': raw_entry['raw_text'],
                'doi': None,
                'arxiv_id': None,
                'url': None,
                'metadata': {},
                'status': 'identification_failed'
            }
        
        # Calculate match scores
        scored_candidates = self._score_candidates(candidates, query_string)
        scored_candidates.sort(key=lambda x: x['match_score'], reverse=True)

        best_candidate = scored_candidates[0]
        # Prefer candidates with DOI when scores are close
        doi_candidates = [c for c in scored_candidates if c.get('doi')]
        if doi_candidates:
            best_doi_candidate = doi_candidates[0]
            if (not best_candidate.get('doi') or
                best_doi_candidate['match_score'] >= best_candidate['match_score'] - 5):
                best_candidate = best_doi_candidate

        # If best does not have DOI but looks strong, try title-only CrossRef lookup to resolve DOI
        if (not best_candidate.get('doi')) and best_candidate.get('title') and best_candidate.get('match_score', 0) >= 85:
            try:
                resolved = self._resolve_doi_via_crossref_title(best_candidate['title'], query_string)
                if resolved and resolved.get('doi'):
                    best_candidate = resolved
            except Exception:
                pass
        
        # Decision logic
        if best_candidate['match_score'] >= 80:
            # High confidence: auto adopt
            if len(scored_candidates) == 1 or best_candidate['match_score'] - scored_candidates[1]['match_score'] > 10:
                self.logger.info(f"Entry {raw_entry['id']} high confidence match: {best_candidate.get('doi', 'no-doi')}")
                return {
                    'id': raw_entry['id'],
                    'raw_text': raw_entry['raw_text'],
                    'doi': best_candidate.get('doi'),
                    'arxiv_id': best_candidate.get('arxiv_id'),
                    'url': best_candidate.get('url'),
                    'metadata': best_candidate,
                    'status': 'identified'
                }
        
        if 70 <= best_candidate['match_score'] < 80:
            # Medium confidence: trigger interactive mode
            top_candidates = scored_candidates[:5]  # Top 5 candidates
            try:
                user_choice = interactive_callback(top_candidates)
                if 0 <= user_choice < len(top_candidates):
                    chosen_candidate = top_candidates[user_choice]
                    self.logger.info(f"Entry {raw_entry['id']} user selection: {chosen_candidate.get('doi', 'no-doi')}")
                    return {
                        'id': raw_entry['id'],
                        'raw_text': raw_entry['raw_text'],
                        'doi': chosen_candidate.get('doi'),
                        'arxiv_id': chosen_candidate.get('arxiv_id'),
                        'url': chosen_candidate.get('url'),
                        'metadata': chosen_candidate,
                        'status': 'identified'
                    }
                else:
                    # Non-interactive or user skipped: fallback to best candidate if sufficiently strong
                    if best_candidate['match_score'] >= 75:
                        self.logger.info(
                            f"Entry {raw_entry['id']} fallback adopt best candidate (score={best_candidate['match_score']}): {best_candidate.get('doi', 'no-doi')}"
                        )
                        return {
                            'id': raw_entry['id'],
                            'raw_text': raw_entry['raw_text'],
                            'doi': best_candidate.get('doi'),
                            'arxiv_id': best_candidate.get('arxiv_id'),
                            'url': best_candidate.get('url'),
                            'metadata': best_candidate,
                            'status': 'identified'
                        }
                    self.logger.info(f"Entry {raw_entry['id']} user skipped")
            except Exception as e:
                self.logger.error(f"Interactive callback failed: {str(e)}")
                # Fallback in case interactive path is unavailable
                if best_candidate['match_score'] >= 75:
                    self.logger.info(
                        f"Entry {raw_entry['id']} fallback adopt best candidate after interactive error (score={best_candidate['match_score']}): {best_candidate.get('doi', 'no-doi')}"
                    )
                    return {
                        'id': raw_entry['id'],
                        'raw_text': raw_entry['raw_text'],
                        'doi': best_candidate.get('doi'),
                        'arxiv_id': best_candidate.get('arxiv_id'),
                        'url': best_candidate.get('url'),
                        'metadata': best_candidate,
                        'status': 'identified'
                    }
        
        # Low confidence but if score is decent and has title, mark as identified
        # Further lower threshold for better recall - many valid papers are being rejected
        if best_candidate['match_score'] >= 50 and best_candidate.get('title'):
            self.logger.info(f"Entry {raw_entry['id']} adopting best candidate with score {best_candidate['match_score']}")
            return {
                'id': raw_entry['id'],
                'raw_text': raw_entry['raw_text'],
                'doi': best_candidate.get('doi'),
                'arxiv_id': best_candidate.get('arxiv_id'),
                'url': best_candidate.get('url'),
                'metadata': best_candidate,
                'status': 'identified'
            }

        # Debug: Log the best candidate score for analysis
        self.logger.info(f"Entry {raw_entry['id']} best candidate score: {best_candidate.get('match_score', 0)}")
        if 'score_breakdown' in best_candidate:
            self.logger.info(f"Entry {raw_entry['id']} score breakdown: {best_candidate['score_breakdown']}")
        
        # Low confidence: mark as failed
        self.logger.warning(f"Entry {raw_entry['id']} low confidence match, marking as failed")
        return {
            'id': raw_entry['id'],
            'raw_text': raw_entry['raw_text'],
            'doi': None,
            'arxiv_id': None,
            'url': None,
            'metadata': {},
            'status': 'identification_failed'
        }

    def _resolve_doi_via_crossref_title(self, candidate_title: str, original_query: str) -> Optional[Dict]:
        """Try to resolve DOI by querying CrossRef with title only (plus hints).
        Returns a candidate dict with DOI if found and strongly matched.
        """
        try:
            url = f"{self.crossref_base_url}"
            # Build a focused query using title and optional year tokens from original query
            year_match = re.search(r"(19|20)\d{2}", original_query)
            year_text = year_match.group(0) if year_match else ''
            focused_query = candidate_title
            if year_text:
                focused_query = f"{candidate_title} {year_text}"
            params = {
                'query.title': candidate_title,
                'query.bibliographic': focused_query,
                'rows': 5,
                'mailto': 'omnicite@example.com'
            }
            response = requests.get(url, params=params, timeout=5)
            response.raise_for_status()
            data = response.json()
            items = data.get('message', {}).get('items', [])
            best_item = None
            best_score = -1
            for item in items:
                title = (item.get('title', [''])[0] or '').lower()
                if not title:
                    continue
                # Use robust fuzzy comparison against candidate title
                base = candidate_title.lower()
                score = max(
                    fuzz.ratio(base, title),
                    fuzz.partial_ratio(base, title),
                    fuzz.token_set_ratio(base, title)
                )
                if score > best_score and item.get('DOI'):
                    best_score = score
                    best_item = item
            if best_item and best_score >= 90:
                return {
                    'source': 'crossref',
                    'doi': best_item.get('DOI'),
                    'title': (best_item.get('title', [''])[0] or ''),
                    'authors': [f"{a.get('given', '')} {a.get('family', '')}" for a in best_item.get('author', [])],
                    'year': best_item.get('published-print', {}).get('date-parts', [[None]])[0][0] or
                            best_item.get('published-online', {}).get('date-parts', [[None]])[0][0],
                    'journal': best_item.get('container-title', [''])[0] if best_item.get('container-title') else '',
                    'citations': best_item.get('is-referenced-by-count', 0)
                }
        except Exception:
            return None
        return None
    
    def _search_crossref(self, query: str, limit: int = 15) -> List[Dict]:
        """Enhanced CrossRef search with better query optimization"""
        try:
            # 优化查询参数
            params = {
                'query': query,
                'query.bibliographic': query,
                'query.title': query,  # 添加标题查询
                'rows': limit,
                'sort': 'relevance',  # 按相关性排序
                'mailto': 'omnicite@example.com'
            }

            # 尝试多个查询策略
            search_strategies = [
                params,  # 标准查询
                {**params, 'query.author': query.split('.')[0] if '.' in query else query},  # 重点查询作者
                {**params, 'filter': 'type:journal-article,proceedings-article,book-chapter'},  # 只查询学术内容
            ]

            all_results = []
            seen_dois = set()

            for i, strategy_params in enumerate(search_strategies):
                try:
                    self.logger.debug(f"CrossRef search strategy {i+1}")
                    url = f"{self.crossref_base_url}"
                    response = requests.get(url, params=strategy_params, timeout=15)  # 进一步增加超时时间
                    response.raise_for_status()
                    data = response.json()

                    for item in data.get('message', {}).get('items', []):
                        doi = item.get('DOI')
                        if not doi or doi in seen_dois:
                            continue

                        # 更完整的数据提取
                        result = {
                            'source': 'crossref',
                            'doi': doi,
                            'title': item.get('title', [''])[0] if item.get('title') else '',
                            'authors': [],
                            'year': None,
                            'journal': '',
                            'citations': item.get('is-referenced-by-count', 0),
                            'type': item.get('type', ''),
                            'url': f"https://doi.org/{doi}",
                            'publisher': item.get('publisher', ''),
                            'volume': item.get('volume', ''),
                            'issue': item.get('issue', ''),
                            'pages': item.get('page', '')
                        }

                        # 处理作者信息
                        for author in item.get('author', []):
                            if author.get('family') or author.get('given'):
                                author_name = f"{author.get('given', '')} {author.get('family', '')}".strip()
                                if author_name:
                                    result['authors'].append(author_name)

                        # 处理出版年份
                        published_dates = [
                            item.get('published-print', {}).get('date-parts'),
                            item.get('published-online', {}).get('date-parts'),
                            item.get('issued', {}).get('date-parts')
                        ]

                        for date_parts in published_dates:
                            if date_parts and date_parts[0] and date_parts[0][0]:
                                result['year'] = date_parts[0][0]
                                break

                        # 处理期刊/会议名称
                        container_titles = item.get('container-title', [])
                        if container_titles:
                            result['journal'] = container_titles[0]

                        # 会议论文特殊处理
                        if result['type'] == 'proceedings-article':
                            event = item.get('event')
                            if event and event.get('name'):
                                result['journal'] = event['name'][0] if isinstance(event['name'], list) else event['name']

                        # 只保留有足够信息的结果
                        if result['title'] and len(result['title']) > 5 and result['authors']:
                            all_results.append(result)
                            seen_dois.add(doi)

                        if len(all_results) >= limit:
                            break

                    if len(all_results) >= limit:
                        break

                except requests.exceptions.Timeout:
                    self.logger.warning(f"CrossRef search strategy {i+1} timed out")
                    time.sleep(5)  # 超时后等待5秒再继续
                    continue
                except Exception as e:
                    self.logger.warning(f"CrossRef search strategy {i+1} failed: {str(e)}")
                    time.sleep(2)  # 错误后等待2秒再继续
                    continue

            self.logger.info(f"CrossRef search returned {len(all_results)} unique results")
            return all_results

        except Exception as e:
            self.logger.error(f"CrossRef search failed: {str(e)}")
            return []
    
    def _search_google_scholar(self, query: str, limit: int = 5) -> List[Dict]:
        """Search in Google Scholar (with intelligent retry and captcha handling)"""
        try:
            import threading
            import time

            # Add delay between requests to avoid rate limiting
            if hasattr(self, '_last_scholar_request'):
                time_since_last = time.time() - self._last_scholar_request
                if time_since_last < 10.0:  # 大幅增加到10秒间隔
                    time.sleep(10.0 - time_since_last)

            self._last_scholar_request = time.time()

            # 智能重试机制 - 减少重试次数，增加延迟
            max_retries = 2  # 减少到2次重试
            for attempt in range(max_retries):
                if attempt > 0:
                    # 大幅递增延迟：30秒，60秒
                    retry_delay = 30 * (attempt + 1)
                    self.logger.info(f"Google Scholar retry attempt {attempt + 1}/{max_retries} after {retry_delay}s delay")
                    time.sleep(retry_delay)

                results = []
                search_completed = [False]
                error_occurred = [None]
                captcha_solved = [False]

                def search_worker():
                    try:
                        self.logger.info(f"Google Scholar search attempt {attempt + 1}: {query[:50]}...")
                        search_query = scholarly.search_pubs(query)

                        count = 0
                        for pub in search_query:
                            if count >= limit:
                                break

                            # 动态超时检查
                            elapsed = time.time() - self._last_scholar_request
                            if elapsed > 20:  # 延长到20秒
                                self.logger.warning("Google Scholar search taking too long, stopping")
                                break

                            try:
                                # Extract more fields from Google Scholar
                                bib = pub.get('bib', {})

                                result = {
                                    'source': 'google_scholar',
                                    'doi': None,
                                    'title': bib.get('title', '') or pub.get('title', ''),
                                    'authors': bib.get('author', []) if isinstance(bib.get('author'), list) else
                                              (bib.get('author').split(' and ') if bib.get('author') else []),
                                    'year': bib.get('pub_year', '') or pub.get('year'),
                                    'journal': bib.get('venue', '') or pub.get('venue', '') or pub.get('journal', ''),
                                    'citations': pub.get('num_citations', 0),
                                    'url': pub.get('pub_url', '') or pub.get('url', ''),
                                    'arxiv_id': None
                                }

                                # Try to extract arXiv ID from eprint or other fields
                                if 'eprint' in pub:
                                    arxiv_match = re.search(r'(\d{4}\.\d{4,5})', pub['eprint'])
                                    if arxiv_match:
                                        result['arxiv_id'] = arxiv_match.group(1)

                                # Extract DOI from URL if available
                                if result['url'] and 'doi.org' in result['url']:
                                    doi_match = re.search(r'doi\.org/(.+)', result['url'])
                                    if doi_match:
                                        result['doi'] = doi_match.group(1)

                                # For conference papers, venue often contains conference name
                                if result['journal'] and ('conference' in result['journal'].lower() or
                                                         'proceedings' in result['journal'].lower() or
                                                         'nips' in result['journal'].lower() or
                                                         'neurips' in result['journal'].lower()):
                                    result['type'] = 'conference'

                                # 过滤掉空的或不完整的结果
                                if result['title'] and len(result['title']) > 5:
                                    results.append(result)
                                    count += 1

                            except Exception as e:
                                self.logger.warning(f"Error processing Google Scholar result: {str(e)}")
                                continue

                        search_completed[0] = True
                        self.logger.info(f"Google Scholar search completed, found {len(results)} valid results")

                    except Exception as e:
                        error_msg = str(e)
                        error_occurred[0] = error_msg
                        search_completed[0] = True

                        # 检测验证码和限流错误
                        is_captcha_error = any(keyword in error_msg.lower() for keyword in [
                            'captcha', 'blocked', 'rate', 'too many', '429', 'forbidden', 'access denied'
                        ])

                        if is_captcha_error:
                            self.logger.warning(f"Google Scholar captcha/rate limit detected: {error_msg}")
                        else:
                            self.logger.warning(f"Google Scholar search error: {error_msg}")

                # Start search thread
                search_thread = threading.Thread(target=search_worker)
                search_thread.daemon = True
                search_thread.start()

                # 等待搜索完成，动态调整等待时间 - 大幅增加超时时间
                max_wait_iterations = 120  # 60秒 (增加3倍)
                for i in range(max_wait_iterations):
                    if search_completed[0]:
                        break
                    time.sleep(0.5)

                # 检查搜索结果
                if search_completed[0]:
                    if error_occurred[0]:
                        error_msg = error_occurred[0]
                        is_captcha_error = any(keyword in error_msg.lower() for keyword in [
                            'captcha', 'blocked', 'rate', 'too many', '429', 'forbidden', 'access denied'
                        ])

                        if is_captcha_error and attempt < max_retries - 1:
                            # 验证码错误且还有重试机会，等待更长时间让验证码系统冷却
                            self.logger.info("Captcha error detected, will retry with extended backoff...")
                            time.sleep(60)  # 额外等待60秒让系统冷却
                            continue
                        else:
                            # 其他错误或已达最大重试次数
                            self.logger.warning(f"Google Scholar search failed after retries: {error_msg}")
                            return []
                    else:
                        # 成功获取结果
                        self.logger.info(f"Google Scholar search succeeded with {len(results)} results")
                        return results

                else:
                    # 搜索超时 - 可能是验证码问题，增加额外延迟
                    if attempt < max_retries - 1:
                        self.logger.warning(f"Google Scholar search timed out, likely due to captcha. Adding cooling period...")
                        time.sleep(120)  # 超时时额外等待2分钟
                        continue
                    else:
                        self.logger.warning(f"Google Scholar search failed after {max_retries} attempts (timeout)")
                        return []

            return []

        except Exception as e:
            self.logger.warning(f"Google Scholar search failed: {str(e)}")
            return []
    
    def _score_candidates(self, candidates: List[Dict], query_string: str) -> List[Dict]:
        """Enhanced candidate scoring with domain-specific optimizations"""
        scored_candidates = []

        # Normalize query for robust title matching
        normalized_query = query_string.strip()
        # Try to derive a probable title part: cut at first 4-digit year
        title_part = re.split(r'\b(19|20)\d{2}\b', normalized_query)[0].strip() or normalized_query
        # Remove common "et al." noise
        title_part = re.sub(r'\bet\s*al\.?\b', '', title_part, flags=re.IGNORECASE).strip()

        # Enhanced domain-specific synonyms mapping
        synonyms = {
            'nips': 'neural information processing systems',
            'neurips': 'neural information processing systems',
            'cvpr': 'computer vision and pattern recognition',
            'iclr': 'international conference on learning representations',
            'icml': 'international conference on machine learning',
            # Health information systems domain
            'dhis2': 'district health information system 2',
            'dhis': 'district health information system',
            'who': 'world health organization',
            'bmj': 'british medical journal',
            'plos': 'public library of science',
            'bmc': 'biomed central'
        }

        normalized_query_lower = normalized_query.lower()
        for k, v in synonyms.items():
            if k in normalized_query_lower and v not in normalized_query_lower:
                normalized_query += f" {v}"

        # Also normalize candidate journal/venue names
        def normalize_venue(venue):
            if not venue:
                return ""
            venue_lower = venue.lower()
            for k, v in synonyms.items():
                if k in venue_lower:
                    return venue.replace(k, v).replace(k.upper(), v)
            return venue

        # Extract key components from query for better matching
        query_year = None
        year_match = re.search(r'\b(19|20)\d{2}\b', normalized_query)
        if year_match:
            query_year = int(year_match.group(0))

        for candidate in candidates:
            scores = {}

            # Title similarity (enhanced scoring)
            candidate_title = candidate.get('title', '').lower()
            base_title = title_part.lower()

            if candidate_title and base_title:
                # Use multiple fuzzy measures and take the best
                ratio = fuzz.ratio(base_title, candidate_title)
                partial = fuzz.partial_ratio(base_title, candidate_title)
                token_sort = fuzz.token_sort_ratio(base_title, candidate_title)
                token_set = fuzz.token_set_ratio(base_title, candidate_title)

                title_score = max(ratio, partial, token_sort, token_set)

                # Bonus for exact phrase matches
                if base_title in candidate_title or candidate_title in base_title:
                    title_score = min(title_score + 20, 100)

                scores['title'] = title_score
            else:
                scores['title'] = 0

            # Author matching (improved)
            author_score = 0
            if candidate.get('authors'):
                authors_text = ' '.join(candidate['authors']).lower()
                query_lower = normalized_query.lower()

                # Exact author name matching
                exact_author_match = False
                for author in candidate['authors']:
                    author_clean = author.lower().strip()
                    if author_clean in query_lower:
                        exact_author_match = True
                        break

                if exact_author_match:
                    author_score = 80
                else:
                    # Fuzzy author matching
                    author_score = fuzz.partial_ratio(query_lower, authors_text)

                # Bonus for multiple authors
                if len(candidate['authors']) > 1:
                    author_score = min(author_score + 10, 100)

            scores['author'] = author_score

            # Year matching (critical for academic papers)
            year_score = 0
            if candidate.get('year') and query_year:
                candidate_year = int(candidate['year']) if isinstance(candidate['year'], str) and candidate['year'].isdigit() else candidate['year']
                if candidate_year:
                    year_diff = abs(candidate_year - query_year)
                    if year_diff == 0:
                        year_score = 100  # Perfect year match
                    elif year_diff <= 2:
                        year_score = 70  # Close year match
                    elif year_diff <= 5:
                        year_score = 30  # Reasonable year match

            scores['year'] = year_score

            # Venue/Journal matching
            venue_score = 0
            venue_lower = ""  # Initialize venue_lower
            if candidate.get('journal'):
                normalized_venue = normalize_venue(candidate['journal'])
                venue_lower = normalized_venue.lower()

                # Check for venue mentions in query
                if venue_lower and venue_lower in normalized_query_lower:
                    venue_score = 60
                else:
                    # Fuzzy venue matching
                    venue_ratio = fuzz.partial_ratio(normalized_query_lower, venue_lower)
                    if venue_ratio > 60:
                        venue_score = venue_ratio * 0.8

            scores['venue'] = venue_score

            # Source reliability score
            source_score = 0
            if candidate.get('source') == 'crossref':
                source_score = 90  # CrossRef is highly reliable
            elif candidate.get('source') == 'google_scholar':
                source_score = 70  # Google Scholar is good but less structured
            elif candidate.get('source') == 'pubmed':
                source_score = 95  # PubMed for medical literature

            scores['source'] = source_score

            # Citation score (adjusted for field and age)
            citation_score = 0
            if candidate.get('citations') is not None:
                citations = candidate['citations']
                if citations > 1000:
                    citation_score = 100
                elif citations > 100:
                    citation_score = 80
                elif citations > 10:
                    citation_score = 50
                elif citations > 0:
                    citation_score = 20
                # New papers get some base score even with 0 citations

            scores['citations'] = citation_score

            # Domain-specific bonuses
            domain_bonus = 0
            authors_text = ' '.join(candidate['authors']).lower() if candidate.get('authors') else ''
            candidate_text = f"{candidate_title} {authors_text} {venue_lower}".lower()

            # Health information systems keywords
            health_keywords = ['health information', 'dhis', 'district health', 'routine health',
                             'health system', 'health data', 'public health', 'epidemiology']

            if any(keyword in candidate_text for keyword in health_keywords):
                domain_bonus += 15

            # WHO/International organization bonus
            if 'world health organization' in candidate_text or 'who' in candidate_text:
                domain_bonus += 20

            # BMC/PLOS bonus (open access, high quality)
            if any(pub in venue_lower for pub in ['bmc', 'plos', 'biomed central', 'public library of science']):
                domain_bonus += 10

            scores['domain'] = domain_bonus

            # Weighted total score calculation (adjusted for version tolerance)
            # Lower year weight, higher title and author weight
            match_score = (
                scores['title'] * 0.40 +      # Title most important (increased)
                scores['author'] * 0.30 +     # Author very important (increased)
                scores['year'] * 0.10 +       # Year less critical (decreased)
                scores['venue'] * 0.10 +      # Venue helpful
                scores['source'] * 0.05 +     # Source reliability
                scores['citations'] * 0.03 +  # Citations least important
                scores['domain'] * 0.02       # Domain bonus
            )

            candidate_copy = candidate.copy()
            candidate_copy['match_score'] = round(match_score, 2)
            candidate_copy['score_breakdown'] = scores
            scored_candidates.append(candidate_copy)

        # Sort by match score descending
        scored_candidates.sort(key=lambda x: x['match_score'], reverse=True)

        return scored_candidates


class EnricherModule:
    """Stage 3: Enrichment and Validation Module"""
    
    def __init__(self, use_google_scholar: bool = False):
        self.logger = logging.getLogger(__name__)
        self.crossref_base_url = "https://api.crossref.org/works"
        self.use_google_scholar = use_google_scholar
    
    def enrich(self, identified_entries: List[IdentifiedEntry], 
               template: Dict) -> List[CompletedEntry]:
        """
        Enrich entries to obtain complete bibliographic information.
        
        Args:
            identified_entries: List of identified entries
            template: Template configuration
        
        Returns:
            List of completed records
        """
        self.logger.info(f"Starting enrichment for {len(identified_entries)} entries")
        completed_entries = []
        
        for entry in identified_entries:
            if entry['status'] == 'identified':
                # Process entries with DOI, arXiv ID, or other metadata
                if entry.get('doi') or entry.get('arxiv_id') or entry.get('metadata'):
                    completed_entry = self._enrich_single_entry(entry, template)
                    completed_entries.append(completed_entry)
                else:
                    # Entries without any identifier
                    failed_entry = {
                        'id': entry['id'],
                        'doi': '',
                        'status': 'enrichment_failed',
                        'bib_key': '',
                        'bib_data': {}
                    }
                    completed_entries.append(failed_entry)
            else:
                # Entries that were not identified are marked as failed
                failed_entry = {
                    'id': entry['id'],
                    'doi': '',
                    'status': 'enrichment_failed',
                    'bib_key': '',
                    'bib_data': {}
                }
                completed_entries.append(failed_entry)
        
        successful_count = sum(1 for e in completed_entries if e['status'] == 'completed')
        self.logger.info(f"Enrichment completed: {successful_count}/{len(completed_entries)} entries successfully completed")
        
        return completed_entries
    
    def _enrich_single_entry(self, identified_entry: IdentifiedEntry, 
                            template: Dict) -> CompletedEntry:
        """Enrich a single entry"""
        doi = identified_entry.get('doi')
        arxiv_id = identified_entry.get('arxiv_id')
        metadata = identified_entry.get('metadata', {})
        
        try:
            base_record = None
            
            # Try to get metadata from various sources
            if doi:
                # Get base record from CrossRef
                base_record = self._get_crossref_metadata(doi)
            elif arxiv_id:
                # Get base record from arXiv
                base_record = self._get_arxiv_metadata(arxiv_id)
            elif metadata:
                # Use metadata from search results
                base_record = self._convert_search_metadata(metadata)
            
            if not base_record:
                return {
                    'id': identified_entry['id'],
                    'doi': doi or '',
                    'status': 'enrichment_failed',
                    'bib_key': '',
                    'bib_data': {}
                }
            
            # Generate BibTeX key
            bib_key = self._generate_bibtex_key(base_record)
            
            # Complete missing fields according to the template
            completed_data = self._complete_fields(base_record, template)
            
            # Set the entry type based on content
            if metadata.get('type') == 'conference' or 'conference' in completed_data.get('journal', '').lower():
                completed_data['ENTRYTYPE'] = 'inproceedings'
            else:
                completed_data['ENTRYTYPE'] = template.get('entry_type', '@article').lstrip('@')
            
            completed_data['ID'] = bib_key
            
            # Add DOI if available
            if doi:
                completed_data['doi'] = doi
            
            # Add arXiv ID if available
            if arxiv_id:
                completed_data['arxiv'] = arxiv_id
                if not completed_data.get('url'):
                    completed_data['url'] = f'https://arxiv.org/abs/{arxiv_id}'
            
            self.logger.info(f"Entry {identified_entry['id']} enrichment successful")
            
            return {
                'id': identified_entry['id'],
                'doi': doi or '',
                'status': 'completed',
                'bib_key': bib_key,
                'bib_data': completed_data
            }
            
        except Exception as e:
            self.logger.error(f"Entry {identified_entry['id']} enrichment failed: {str(e)}")
            return {
                'id': identified_entry['id'],
                'doi': doi or '',
                'status': 'enrichment_failed',
                'bib_key': '',
                'bib_data': {}
            }
    
    def _get_crossref_metadata(self, doi: str) -> Optional[Dict]:
        """Get metadata from the CrossRef API"""
        try:
            url = f"{self.crossref_base_url}/{doi}"
            headers = {'Accept': 'application/json'}
            
            response = requests.get(url, headers=headers, timeout=10)
            response.raise_for_status()
            
            data = response.json()
            work = data.get('message', {})
            
            # Convert to a standard format
            metadata = {
                'doi': work.get('DOI'),
                'title': work.get('title', [''])[0] if work.get('title') else '',
                'author': self._format_authors(work.get('author', [])),
                'journal': work.get('container-title', [''])[0] if work.get('container-title') else '',
                'year': self._extract_year(work),
                'volume': work.get('volume'),
                'number': work.get('issue'),
                'pages': work.get('page'),
                'publisher': work.get('publisher'),
                'url': work.get('URL')
            }
            
            return {k: v for k, v in metadata.items() if v is not None}
            
        except Exception as e:
            self.logger.error(f"Failed to get CrossRef metadata for {doi}: {str(e)}")
            return None
    
    def _get_arxiv_metadata(self, arxiv_id: str) -> Optional[Dict]:
        """Get metadata from arXiv API (with timeout protection)"""
        try:
            import feedparser
            import requests
            url = f"http://export.arxiv.org/api/query?id_list={arxiv_id}"
            
            # Use requests to get content with timeout
            response = requests.get(url, timeout=10)
            response.raise_for_status()
            
            feed = feedparser.parse(response.content)
            if not feed.entries:
                return None
            
            entry = feed.entries[0]
            
            # Extract authors
            authors = []
            for author in entry.get('authors', []):
                name = author.get('name', '')
                if name:
                    # Convert "First Last" to "Last, First"
                    parts = name.split()
                    if len(parts) >= 2:
                        authors.append(f"{parts[-1]}, {' '.join(parts[:-1])}")
                    else:
                        authors.append(name)
            
            # Extract year from published date
            published = entry.get('published', '')
            year = published[:4] if len(published) >= 4 else None
            
            metadata = {
                'arxiv': arxiv_id,
                'title': entry.get('title', '').replace('\n', ' ').strip(),
                'author': ' and '.join(authors),
                'year': year,
                'journal': 'arXiv preprint',
                'url': f'https://arxiv.org/abs/{arxiv_id}',
                'abstract': entry.get('summary', '').replace('\n', ' ').strip()
            }
            
            return {k: v for k, v in metadata.items() if v}
            
        except Exception as e:
            self.logger.error(f"Failed to get arXiv metadata for {arxiv_id}: {str(e)}")
            return None
    
    def _convert_search_metadata(self, metadata: Dict) -> Optional[Dict]:
        """Convert search result metadata to standard format"""
        try:
            # Handle authors - they might be in list or string format
            authors = metadata.get('authors', []) or metadata.get('author', '')
            if isinstance(authors, list):
                formatted_authors = ' and '.join(authors)
            elif isinstance(authors, str) and authors.strip():
                formatted_authors = authors.strip()
            else:
                formatted_authors = ''
            
            # Determine if it's a conference paper
            journal = metadata.get('journal', '')
            if metadata.get('type') == 'conference' or any(conf in journal.lower() 
                for conf in ['conference', 'proceedings', 'symposium', 'workshop', 'nips', 'neurips']):
                # For conference papers, use booktitle instead of journal
                result = {
                    'title': metadata.get('title', ''),
                    'author': formatted_authors,
                    'booktitle': journal,
                    'year': str(metadata.get('year', '')),
                }
            else:
                result = {
                    'title': metadata.get('title', ''),
                    'author': formatted_authors,
                    'journal': journal,
                    'year': str(metadata.get('year', '')),
                }
            
            # Add optional fields
            if metadata.get('doi'):
                result['doi'] = metadata['doi']
            if metadata.get('url'):
                result['url'] = metadata['url']
            if metadata.get('arxiv_id'):
                result['arxiv'] = metadata['arxiv_id']
            if metadata.get('pages'):
                result['pages'] = metadata['pages']
            if metadata.get('volume'):
                result['volume'] = metadata['volume']
            if metadata.get('number'):
                result['number'] = metadata['number']
            
            return {k: v for k, v in result.items() if v}
            
        except Exception as e:
            self.logger.error(f"Failed to convert search metadata: {str(e)}")
            return None
    
    def _format_authors(self, authors: List[Dict]) -> str:
        """Format the author list"""
        formatted_authors = []
        for author in authors:
            given = author.get('given', '')
            family = author.get('family', '')
            if family:
                if given:
                    formatted_authors.append(f"{family}, {given}")
                else:
                    formatted_authors.append(family)
        
        return ' and '.join(formatted_authors)
    
    def _extract_year(self, work: Dict) -> Optional[str]:
        """Extract publication year"""
        # Try multiple date fields
        date_fields = ['published-print', 'published-online', 'created']
        for field in date_fields:
            if field in work:
                date_parts = work[field].get('date-parts', [[]])
                if date_parts and date_parts[0]:
                    return str(date_parts[0][0])
        return None
    
    def _generate_bibtex_key(self, metadata: Dict) -> str:
        """Generate BibTeX key"""
        # Format: First author's surname + year + first word of the title
        key_parts = []
        
        # First author's surname
        if metadata.get('author'):
            first_author = metadata['author'].split(' and ')[0]
            if ',' in first_author:
                family_name = first_author.split(',')[0].strip()
            else:
                family_name = first_author.split()[-1]
            key_parts.append(re.sub(r'[^\w]', '', family_name))
        
        # Year
        if metadata.get('year'):
            key_parts.append(metadata['year'])
        
        # First word of title
        if metadata.get('title'):
            title_words = metadata['title'].split()
            if title_words:
                first_word = re.sub(r'[^\w]', '', title_words[0])
                key_parts.append(first_word)
        
        return ''.join(key_parts) or 'unknown'
    
    def _complete_fields(self, base_record: Dict, template: Dict) -> Dict:
        """Complete missing fields according to template"""
        completed_data = base_record.copy()
        
        # Check required fields in template
        for field_config in template.get('fields', []):
            field_name = field_config['name']
            
            # If field is missing and has completion strategy
            if field_name not in completed_data or not completed_data[field_name]:
                if 'source_priority' in field_config:
                    value = self._fetch_missing_field(field_name, field_config['source_priority'], base_record)
                    if value:
                        completed_data[field_name] = value
        
        return completed_data
    
    def _fetch_missing_field(self, field_name: str, source_priority: List[str], base_record: Dict) -> Optional[str]:
        """Get missing fields according to priority strategy"""
        for source in source_priority:
            if source == 'crossref_api':
                # Already got from CrossRef, skip
                continue
            elif source == 'google_scholar_scraper':
                # Only use Google Scholar if enabled
                if self.use_google_scholar:
                    value = self._fetch_from_google_scholar(field_name, base_record)
                    if value:
                        return value
                else:
                    self.logger.info(f"Google Scholar disabled, skipping field {field_name} completion")
            elif source == 'user_prompt':
                # User input not handled here, left to frontend
                continue
        
        return None
    
    def _fetch_from_google_scholar(self, field_name: str, base_record: Dict) -> Optional[str]:
        """Get field value from Google Scholar (with improved timeout protection)"""
        try:
            # Search using title and authors
            query = base_record.get('title', '')
            if not query:
                return None
            
            # Add delay between requests to avoid rate limiting
            import threading
            import time
            
            if hasattr(self, '_last_scholar_request'):
                time_since_last = time.time() - self._last_scholar_request
                if time_since_last < 2.0:  # 至少等待2秒
                    time.sleep(2.0 - time_since_last)
            
            self._last_scholar_request = time.time()
            
            result_container = [None]
            search_completed = [False]
            
            def search_worker():
                try:
                    search_query = scholarly.search_pubs(query)
                    pub = next(search_query, None)
                    
                    if pub and field_name in pub:
                        result_container[0] = str(pub[field_name])
                    
                    search_completed[0] = True
                except Exception as e:
                    self.logger.warning(f"Google Scholar field search failed: {str(e)}")
                    search_completed[0] = True
            
            # Start search thread
            search_thread = threading.Thread(target=search_worker)
            search_thread.daemon = True
            search_thread.start()
            
            # Wait up to 5 seconds with periodic checks (field completion is not critical)
            for _ in range(10):  # 10 * 0.5 = 5 seconds
                if search_completed[0]:
                    break
                time.sleep(0.5)
            
            if not search_completed[0]:
                self.logger.warning(f"Google Scholar field search timed out for {field_name}")
                return None
            
            return result_container[0]
                
        except Exception as e:
            self.logger.warning(f"Getting field {field_name} from Google Scholar failed: {str(e)}")
        
        return None


class FormatterModule:
    """Stage 4: Formatting and Generation Module"""
    
    def __init__(self):
        self.logger = logging.getLogger(__name__)
    
    def format(self, completed_entries: List[CompletedEntry], 
               output_format: str) -> Dict[str, Any]:
        """
        Format completed records to specified output format
        
        Args:
            completed_entries: List of completed records
            output_format: Output format
        
        Returns:
            Formatting results and report
        """
        self.logger.info(f"Starting to format {len(completed_entries)} entries to {output_format} format")
        
        formatted_strings = []
        failed_entries = []
        
        for entry in completed_entries:
            if entry['status'] == 'completed':
                try:
                    if output_format.lower() == 'bibtex':
                        formatted_string = self._format_bibtex(entry)
                    elif output_format.lower() == 'apa':
                        formatted_string = self._format_apa(entry)
                    elif output_format.lower() == 'mla':
                        formatted_string = self._format_mla(entry)
                    else:
                        # Default to BibTeX
                        formatted_string = self._format_bibtex(entry)
                    
                    formatted_strings.append(formatted_string)
                    
                except Exception as e:
                    self.logger.error(f"Formatting entry {entry['id']} failed: {str(e)}")
                    failed_entries.append({
                        'id': entry['id'],
                        'error': str(e),
                        'doi': entry.get('doi', 'unknown')
                    })
            else:
                failed_entries.append({
                    'id': entry['id'],
                    'error': 'Entry processing failed',
                    'status': entry['status']
                })
        
        report = {
            'total': len(completed_entries),
            'succeeded': len(formatted_strings),
            'failed_entries': failed_entries
        }
        
        self.logger.info(f"Formatting completed: {len(formatted_strings)}/{len(completed_entries)} entries successful")
        
        return {
            'results': formatted_strings,
            'report': report
        }
    
    def _format_bibtex(self, entry: CompletedEntry) -> str:
        """Format to BibTeX format"""
        bib_data = entry['bib_data']
        entry_type = bib_data.get('ENTRYTYPE', 'article')
        entry_id = bib_data.get('ID', entry['bib_key'])
        
        lines = [f"@{entry_type}{{{entry_id},"]
        
        for key, value in bib_data.items():
            if key not in ['ENTRYTYPE', 'ID'] and value:
                # Clean and format values
                clean_value = str(value).replace('{', '').replace('}', '')
                if key in ['volume', 'number', 'year']:
                    lines.append(f"  {key} = {clean_value},")
                else:
                    lines.append(f'  {key} = "{clean_value}",')
        
        lines.append('}')
        return '\n'.join(lines)
    
    def _format_apa(self, entry: CompletedEntry) -> str:
        """Format to APA format"""
        bib_data = entry['bib_data']
        parts = []
        
        # Authors
        if bib_data.get('author'):
            authors = bib_data['author'].replace(' and ', ', ')
            parts.append(authors)
        
        # Year
        if bib_data.get('year'):
            parts.append(f"({bib_data['year']})")
        
        # Title
        if bib_data.get('title'):
            parts.append(f"{bib_data['title']}.")
        
        # Journal information
        if bib_data.get('journal'):
            journal_part = f"*{bib_data['journal']}*"
            if bib_data.get('volume'):
                journal_part += f", {bib_data['volume']}"
            if bib_data.get('number'):
                journal_part += f"({bib_data['number']})"
            if bib_data.get('pages'):
                journal_part += f", {bib_data['pages']}"
            parts.append(journal_part + ".")
        
        return ' '.join(parts)
    
    def _format_mla(self, entry: CompletedEntry) -> str:
        """Format to MLA format (simplified implementation)"""
        # Simplified MLA format, should be more complex in practice
        return self._format_apa(entry)
