#!/usr/bin/env python3
"""
Local Semantic Scholar Database Client for Reference Verification

This module provides functionality to verify non-arXiv references using a local Semantic Scholar database.
It can check if a reference's metadata (authors, year, title) matches what's in the local database.

Usage:
    from local_semantic_scholar import LocalNonArxivReferenceChecker
    
    # Initialize the checker
    checker = LocalNonArxivReferenceChecker(db_path="semantic_scholar_db/semantic_scholar.db")
    
    # Verify a reference
    reference = {
        'title': 'Title of the paper',
        'authors': ['Author 1', 'Author 2'],
        'year': 2020,
        'url': 'https://example.com/paper',
        'raw_text': 'Full citation text'
    }
    
    verified_data, errors = checker.verify_reference(reference)
"""

import json
import logging
import re
import sqlite3
import time
from typing import Dict, List, Tuple, Optional, Any, Union

# Import utility functions
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from refchecker.utils.doi_utils import extract_doi_from_url, compare_dois, construct_doi_url
from refchecker.utils.error_utils import create_author_error, create_doi_error, create_venue_warning, format_title_mismatch
from refchecker.utils.text_utils import normalize_author_name, normalize_paper_title, is_name_match, compare_authors, calculate_title_similarity, compare_titles_with_latex_cleaning, strip_latex_commands, are_venues_substantially_different
from refchecker.utils.url_utils import extract_arxiv_id_from_url, get_best_available_url, construct_semantic_scholar_url
from refchecker.utils.db_utils import process_semantic_scholar_result, process_semantic_scholar_results
from refchecker.config.settings import get_config

# Set up logging
logger = logging.getLogger(__name__)

# Get configuration
config = get_config()
SIMILARITY_THRESHOLD = config["text_processing"]["similarity_threshold"]

def log_query_debug(query: str, params: list, execution_time: float, result_count: int, strategy: str):
    """Log database query details in debug mode"""
    if logger.isEnabledFor(logging.DEBUG):
        logger.debug(f"DB Query Strategy: {strategy}")
        logger.debug(f"DB Query: {query}")
        logger.debug(f"DB Params: {params}")
        logger.debug(f"DB Execution Time: {execution_time:.3f}s")
        logger.debug(f"DB Result Count: {result_count}")
    else:
        # Always log strategy and result count for INFO level
        logger.debug(f"DB Query [{strategy}]: {result_count} results in {execution_time:.3f}s")

class LocalNonArxivReferenceChecker:
    """
    A class to verify non-arXiv references using a local Semantic Scholar database
    """
    
    def __init__(self, db_path: str = "semantic_scholar_db/semantic_scholar.db"):
        """
        Initialize the local Semantic Scholar database client
        
        Args:
            db_path: Path to the SQLite database
        
        Raises:
            FileNotFoundError: If the database file does not exist
            ValueError: If the database is missing the required 'papers' table
        """
        if not os.path.exists(db_path):
            raise FileNotFoundError(f"Local Semantic Scholar database not found: {db_path}")
        self.db_path = db_path
        self.conn = sqlite3.connect(db_path, check_same_thread=False)
        self.conn.row_factory = sqlite3.Row  # Return rows as dictionaries
        # Validate that the required 'papers' table exists
        cursor = self.conn.execute(
            "SELECT name FROM sqlite_master WHERE type='table' AND name='papers'"
        )
        if cursor.fetchone() is None:
            self.conn.close()
            raise ValueError(
                f"Database at {db_path} is missing the required 'papers' table. "
                f"Ensure --db-path points to a valid Semantic Scholar database."
            )
        # Optimise for read-heavy workloads (reference lookups are read-only)
        self.conn.execute("PRAGMA journal_mode=WAL")
        self.conn.execute("PRAGMA synchronous=NORMAL")
        self.conn.execute("PRAGMA cache_size=-64000")   # 64 MB page cache
        self.conn.execute("PRAGMA temp_store=MEMORY")
    
    # DOI extraction now handled by utility function
    
    # Title normalization now handled by utility function
    
    # Author name normalization now handled by utility function
    
    # Author comparison now handled by utility function
    
    # Name matching now handled by utility function
    
    def get_paper_by_doi(self, doi: str) -> Optional[Dict[str, Any]]:
        """
        Get paper data by DOI from the local database
        
        Args:
            doi: DOI of the paper
            
        Returns:
            Paper data dictionary or None if not found
        """
        # Reject truncated/partial DOIs (e.g., "10.1016/j")
        if not doi or len(doi.split('/', 1)[-1]) < 2:
            return None

        cursor = self.conn.cursor()
        
        # Query the database for the paper with the given DOI using the column-based schema
        query = '''
        SELECT * FROM papers
        WHERE externalIds_DOI = ?
        '''
        params = (doi,)
        
        start_time = time.time()
        cursor.execute(query, params)
        row = cursor.fetchone()
        execution_time = time.time() - start_time
        
        result_count = 1 if row else 0
        log_query_debug(query, list(params), execution_time, result_count, "DOI lookup")
        
        if not row:
            return None
        
        # Convert row to dictionary and process using utility function
        paper_data = process_semantic_scholar_result(dict(row))
        
        return paper_data
    
    def get_paper_by_arxiv_id(self, arxiv_id: str) -> Optional[Dict[str, Any]]:
        """
        Get paper data by arXiv ID from the local database
        
        Args:
            arxiv_id: arXiv ID of the paper
            
        Returns:
            Paper data dictionary or None if not found
        """
        cursor = self.conn.cursor()
        
        # Query the database for the paper with the given arXiv ID using the column-based schema
        query = '''
        SELECT * FROM papers
        WHERE externalIds_ArXiv = ?
        '''
        params = (arxiv_id,)
        
        start_time = time.time()
        cursor.execute(query, params)
        row = cursor.fetchone()
        execution_time = time.time() - start_time
        
        result_count = 1 if row else 0
        log_query_debug(query, list(params), execution_time, result_count, "arXiv ID lookup")
        
        if not row:
            return None
        
        # Convert row to dictionary and process using utility function
        paper_data = process_semantic_scholar_result(dict(row))
        
        return paper_data
    
    def search_papers_by_title(self, title: str, year: Optional[int] = None) -> List[Dict[str, Any]]:
        """
        Search for papers by title in the local database with optimized performance
        
        Args:
            title: Paper title
            year: Publication year (optional)
            
        Returns:
            List of paper data dictionaries
        """
        cursor = self.conn.cursor()
        
        # Clean up the title for searching
        title_cleaned = title.replace('%', '').strip()
        title_lower = title_cleaned.lower()
        title_normalized = normalize_paper_title(title_cleaned)
        
        results = []
        
        # Strategy 1: Try normalized title match first (fastest and most accurate)
        try:
            cursor.execute("PRAGMA table_info(papers)")
            columns = [row[1] for row in cursor.fetchall()]
            
            if 'normalized_paper_title' in columns and title_normalized:
                query = "SELECT * FROM papers WHERE normalized_paper_title = ?"
                params = [title_normalized]
                    
                start_time = time.time()
                cursor.execute(query, params)
                results.extend([dict(row) for row in cursor.fetchall()])
                execution_time = time.time() - start_time
                
                log_query_debug(query, params, execution_time, len(results), "normalized title match")
                
                if results:
                    logger.debug(f"Found {len(results)} results using normalized title match")
                    return process_semantic_scholar_results(results)
                
                # Strategy 1b: The checker's normalize_paper_title() strips
                # prefixes like "Taichi:", "D-nerf:", etc. but the DB stores
                # the full normalized title. Try DB-style normalization
                # (lowercase + strip non-alphanumeric, no prefix removal).
                db_style_normalized = re.sub(r'[^a-z0-9]', '', title_lower)
                if db_style_normalized and db_style_normalized != title_normalized:
                    start_time = time.time()
                    cursor.execute(query, [db_style_normalized])
                    results.extend([dict(row) for row in cursor.fetchall()])
                    execution_time = time.time() - start_time
                    
                    log_query_debug(query, [db_style_normalized], execution_time, len(results), "DB-style normalized title match")
                    
                    if results:
                        logger.debug(f"Found {len(results)} results using DB-style normalized title match")
                        return process_semantic_scholar_results(results)
        except Exception as e:
            logger.warning(f"Error in normalized title search: {e}")
        
        return process_semantic_scholar_results(results)
    
    # Result processing now handled by utility function
    
    def find_best_match(self, title: str, authors: List[str], year: Optional[int] = None) -> Optional[Dict[str, Any]]:
        """
        Find the best matching paper in the local database
        
        Args:
            title: Paper title
            authors: List of author names
            year: Publication year (optional)
            
        Returns:
            Best matching paper data dictionary or None if not found
        """
        logger.debug(f"Local DB: Finding best match for title: '{title}', authors: {authors}, year: {year}")
        
        # Search by title
        title_results = self.search_papers_by_title(title, year)
        
        logger.debug(f"Local DB: Title search returned {len(title_results)} results")
        
        if title_results:
            # Find the best match by title similarity with stable sorting
            scored_results = []
            
            for result in title_results:
                result_title = result.get('title', '')
                
                # Calculate similarity score using utility function
                score = calculate_title_similarity(title, result_title)
                
                # Check author match
                if authors and result.get('authors'):
                    # Compare first author
                    first_author = normalize_author_name(authors[0])
                    result_first_author = normalize_author_name(result['authors'][0].get('name', ''))
                    
                    if is_name_match(first_author, result_first_author):
                        score += 0.2
                
                # Check year match
                if year and result.get('year') == year:
                    score += 0.1
                
                logger.debug(f"Local DB: Candidate match score {score:.2f} for '{result_title}'")
                
                scored_results.append((score, result))
            
            # Sort by score (descending), then by title for stable ordering when scores are equal
            scored_results.sort(key=lambda x: (-x[0], x[1].get('title', '')))
            
            if scored_results:
                best_score, best_match = scored_results[0]
            
            # If we found a good match, return it
            if best_score >= SIMILARITY_THRESHOLD:
                logger.debug(f"Local DB: Found good title match with score {best_score:.2f}")
                return best_match
            else:
                logger.debug(f"Local DB: Best title match score {best_score:.2f} below threshold ({SIMILARITY_THRESHOLD})")
        
        logger.debug("Local DB: No good match found")
        return None
    
    def verify_reference(self, reference: Dict[str, Any]) -> Tuple[Optional[Dict[str, Any]], List[Dict[str, Any]], Optional[str]]:
        """
        Verify a non-arXiv reference using the local database
        
        Args:
            reference: Reference data dictionary
            
        Returns:
            Tuple of (verified_data, errors, url)
            - verified_data: Paper data from the database or None if not found
            - errors: List of error dictionaries
            - url: URL of the paper if found, None otherwise
        """
        errors = []
        
        # Extract reference data
        title = reference.get('title', '')
        authors = reference.get('authors', [])
        year = reference.get('year', 0)
        url = reference.get('url', '')
        raw_text = reference.get('raw_text', '')
        
        logger.debug(f"Local DB: Verifying reference - Title: '{title}', Authors: {authors}, Year: {year}")
        
        # Extract identifiers from the reference
        doi = None
        arxiv_id = None
        
        if 'doi' in reference and reference['doi']:
            doi = reference['doi']
        elif url:
            # Check if it's an arXiv URL first
            arxiv_id = extract_arxiv_id_from_url(url)
            if not arxiv_id:
                # If not arXiv, try extracting DOI
                doi = extract_doi_from_url(url)
        
        # Reject truncated/partial DOIs (e.g., "10.1016/j")
        from refchecker.utils.doi_utils import is_valid_doi_format
        if doi and not is_valid_doi_format(doi):
            logger.debug(f"Local DB: Rejecting invalid DOI format: {doi}")
            doi = None
        
        paper_data = None
        arxiv_tried = False
        
        # If the reference has an arXiv URL, try arXiv ID first and validate
        # the title matches.  If the title doesn't match the arXiv result,
        # the URL is wrong — fall back to title/DOI lookup so we find the
        # real paper and can flag the incorrect arXiv URL.
        if arxiv_id:
            arxiv_tried = True
            logger.debug(f"Local DB: Searching by arXiv ID first: {arxiv_id}")
            arxiv_paper = self.get_paper_by_arxiv_id(arxiv_id)
            
            if arxiv_paper:
                arxiv_title = arxiv_paper.get('title', '')
                if title and arxiv_title:
                    title_sim = compare_titles_with_latex_cleaning(title, arxiv_title)
                    if title_sim >= SIMILARITY_THRESHOLD:
                        logger.debug(f"ArXiv ID lookup matched title (similarity={title_sim:.2f}), using arXiv result")
                        paper_data = arxiv_paper
                    else:
                        logger.debug(f"ArXiv ID lookup title mismatch (similarity={title_sim:.2f}): "
                                     f"cited '{title}' vs arXiv '{arxiv_title}' — falling back to title/DOI lookup")
                else:
                    # No title to compare — accept the arXiv result
                    logger.debug(f"Found paper by arXiv ID (no title to cross-check)")
                    paper_data = arxiv_paper
            else:
                logger.debug(f"Could not find paper with arXiv ID: {arxiv_id}")
        
        # Try title/author search (primary fallback, or first if no arXiv URL)
        if not paper_data and (title or authors):
            logger.debug(f"Local DB: Searching by title/authors - Title: '{title}', Authors: {authors}, Year: {year}")
            paper_data = self.find_best_match(title, authors, year)
            
            if paper_data:
                logger.debug(f"Found paper by title/author search")
            else:
                logger.debug(f"Could not find matching paper by title/authors")
        
        # Try DOI if title search didn't find it
        if not paper_data and doi:
            logger.debug(f"Local DB: Searching by DOI: {doi}")
            paper_data = self.get_paper_by_doi(doi)
            
            if paper_data:
                logger.debug(f"Found paper by DOI: {doi}")
            else:
                logger.debug(f"Could not find paper with DOI: {doi}")
        
        # Try arXiv ID as last resort (only if we haven't tried it above)
        if not paper_data and arxiv_id and not arxiv_tried:
            logger.debug(f"Local DB: Searching by arXiv ID: {arxiv_id}")
            paper_data = self.get_paper_by_arxiv_id(arxiv_id)
            
            if paper_data:
                logger.debug(f"Found paper by arXiv ID: {arxiv_id}")
            else:
                logger.debug(f"Could not find paper with arXiv ID: {arxiv_id}")
        
        # If we couldn't find the paper, return no errors (can't verify)
        if not paper_data:
            logger.debug("Local DB: No matching paper found - cannot verify reference")
            return None, [], None
        
        logger.debug(f"Local DB: Found matching paper - Title: '{paper_data.get('title', '')}', Year: {paper_data.get('year', '')}")
        
        # Check title mismatch using similarity function
        found_title = paper_data.get('title', '')
        if title and found_title:
            title_similarity = compare_titles_with_latex_cleaning(title, found_title)
            if title_similarity < SIMILARITY_THRESHOLD:
                clean_cited_title = strip_latex_commands(title)
                errors.append({
                    'error_type': 'title',
                    'error_details': format_title_mismatch(clean_cited_title, found_title),
                    'ref_title_correct': found_title
                })
        
        # Verify authors
        if authors:
            authors_match, author_error = compare_authors(authors, paper_data.get('authors', []))
            
            if not authors_match:
                logger.debug(f"Local DB: Author mismatch - {author_error}")
                errors.append(create_author_error(author_error, paper_data.get('authors', [])))
        
        # Verify year (with tolerance)
        paper_year = paper_data.get('year')
        # Get year tolerance from config (default to 1 if not available)
        year_tolerance = 1  # Default tolerance
        try:
            from refchecker.config.settings import get_config
            config = get_config()
            year_tolerance = config.get('text_processing', {}).get('year_tolerance', 1)
        except (ImportError, Exception):
            pass  # Use default if config not available
        
        from refchecker.utils.error_utils import validate_year
        year_warning = validate_year(
            cited_year=year,
            paper_year=paper_year,
            year_tolerance=year_tolerance
        )
        if year_warning:
            logger.debug(f"Local DB: Year issue - {year_warning.get('warning_details', '')}")
            errors.append(year_warning)
        
        # Verify DOI
        paper_doi = None
        external_ids = paper_data.get('externalIds', {})
        if external_ids and 'DOI' in external_ids:
            paper_doi = external_ids['DOI']
            
            # Compare DOIs using utility function
            if doi and paper_doi and not compare_dois(doi, paper_doi):
                logger.debug(f"Local DB: DOI mismatch - cited: {doi}, actual: {paper_doi}")
                doi_error = create_doi_error(doi, paper_doi)
                if doi_error:  # Only add if there's actually a mismatch after cleaning
                    errors.append(doi_error)
        
        # Verify venue
        cited_venue = reference.get('journal', '') or reference.get('venue', '')
        paper_venue = paper_data.get('venue', '')
        
        if cited_venue and paper_venue:
            if are_venues_substantially_different(cited_venue, paper_venue):
                errors.append(create_venue_warning(cited_venue, paper_venue))
        elif not cited_venue and paper_venue:
            # Reference has no venue but paper has one
            # Skip generic/empty venues like 'arxiv'
            paper_venue_lower = paper_venue.lower().strip()
            if (paper_venue_lower and 
                paper_venue_lower not in ['arxiv', 'arxiv.org', 'preprint', ''] and
                not paper_venue_lower.startswith('arxiv')):
                errors.append({
                    'error_type': 'venue',
                    'error_details': f"Venue missing: should include '{paper_venue}'",
                    'ref_venue_correct': paper_venue
                })
        
        # Check for incorrect arXiv URL: if the reference has an arXiv URL,
        # verify it matches the paper we found (which was resolved by title).
        external_ids = paper_data.get('externalIds', {})
        paper_arxiv_id = external_ids.get('ArXiv') if external_ids else None
        if arxiv_id:
            # Reference has an arXiv URL — cross-check against the found paper
            if paper_arxiv_id and arxiv_id.lower() != paper_arxiv_id.lower():
                correct_arxiv_url = f"https://arxiv.org/abs/{paper_arxiv_id}"
                logger.debug(f"Local DB: ArXiv ID mismatch - cited: {arxiv_id}, actual: {paper_arxiv_id}")
                errors.append({
                    'error_type': 'arxiv_id',
                    'error_details': f"Incorrect ArXiv ID: cited {arxiv_id} should be {paper_arxiv_id}",
                    'ref_url_correct': correct_arxiv_url
                })
            elif not paper_arxiv_id:
                # Paper has no ArXiv ID in the DB but reference cites one —
                # the arXiv URL is spurious / points to another paper.
                logger.debug(f"Local DB: Reference cites arXiv ID {arxiv_id} but matched paper has no ArXiv ID")
                errors.append({
                    'error_type': 'arxiv_id',
                    'error_details': f"Incorrect ArXiv ID: paper '{paper_data.get('title', '')}' does not have ArXiv ID {arxiv_id}",
                })
        elif paper_arxiv_id:
            # No arXiv URL in reference but paper has one — suggest it
            arxiv_url = f"https://arxiv.org/abs/{paper_arxiv_id}"
            reference_url = reference.get('url', '')
            has_arxiv_url = arxiv_url in reference_url if reference_url else False
            arxiv_doi_url = f"https://doi.org/10.48550/arxiv.{paper_arxiv_id}"
            has_arxiv_doi = arxiv_doi_url.lower() in reference_url.lower() if reference_url else False
            if not (has_arxiv_url or has_arxiv_doi):
                errors.append({
                    'info_type': 'url',
                    'info_details': f"Reference could include arXiv URL: {arxiv_url}",
                    'ref_url_correct': arxiv_url
                })
        
        if errors:
            logger.debug(f"Local DB: Found {len(errors)} errors in reference verification")
        else:
            logger.debug("Local DB: Reference verification passed - no errors found")
        
        # Return the Semantic Scholar URL that was actually used for verification
        # since this is a Semantic Scholar database checker
        external_ids = paper_data.get('externalIds', {})
        
        # First try to get the Semantic Scholar URL using paperId
        if paper_data.get('paperId'):
            paper_url = construct_semantic_scholar_url(paper_data['paperId'])
            logger.debug(f"Using Semantic Scholar URL for verification: {paper_url}")
        else:
            # Fallback to best available URL if Semantic Scholar URL not available
            open_access_pdf = paper_data.get('openAccessPdf')
            paper_url = get_best_available_url(external_ids, open_access_pdf, paper_data.get('paperId'))
            if paper_url:
                logger.debug(f"Using fallback URL: {paper_url}")
        
        return paper_data, errors, paper_url
    
    def close(self):
        """Close the database connection"""
        self.conn.close()

if __name__ == "__main__":
    # Example usage
    import sys
    
    # Set up logging
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s - %(levelname)s - %(message)s',
        handlers=[
            logging.StreamHandler(sys.stdout)
        ]
    )
    
    # Check if database path is provided
    if len(sys.argv) > 1:
        db_path = sys.argv[1]
    else:
        db_path = "semantic_scholar_db/semantic_scholar.db"
    
    # Initialize the checker
    checker = LocalNonArxivReferenceChecker(db_path=db_path)
    
    # Example reference
    reference = {
        'title': 'Attention is All You Need',
        'authors': ['Ashish Vaswani', 'Noam Shazeer'],
        'year': 2017,
        'url': 'https://example.com/paper',
        'raw_text': 'Vaswani, A., Shazeer, N., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A. N., ... & Polosukhin, I. (2017). Attention is all you need. Advances in neural information processing systems, 30.'
    }
    
    # Verify the reference
    verified_data, errors = checker.verify_reference(reference)
    
    if verified_data:
        print(f"Found paper: {verified_data.get('title')}")
        
        if errors:
            print("Errors found:")
            for error in errors:
                print(f"  - {error['error_type']}: {error['error_details']}")
        else:
            print("No errors found")
    else:
        print("Could not find matching paper")
    
    # Close the database connection
    checker.close()
