"""
Advanced Multi-Modal Document Processing System for Toxo
Implements OCR, table recognition, formula parsing, and Gemini-powered knowledge extraction.
"""

import os
import json
import asyncio
import logging
from typing import Dict, List, Any, Optional, Tuple, Union
from dataclasses import dataclass, asdict
from pathlib import Path
import tempfile
import mimetypes
from io import BytesIO
import hashlib
import time

# Document processing libraries
import pypdf
import pymupdf  # PyMuPDF for advanced PDF processing
import pandas as pd
import openpyxl
from PIL import Image
import pytesseract
import camelot  # For table extraction
from pdfplumber import PDF
import docx
from xml.etree import ElementTree as ET

# ML/AI libraries for advanced processing
import numpy as np
from sklearn.cluster import DBSCAN
from sklearn.preprocessing import StandardScaler

# Gemini integration
from ..integrations.gemini_client import GeminiClient
from ..core.config import GeminiConfig
from ..utils.logger import get_logger
from ..utils.exceptions import ProcessingError, ValidationError


@dataclass
class DocumentStructure:
    """Represents the extracted structure of a document."""
    title: str
    sections: List[Dict[str, Any]]
    tables: List[Dict[str, Any]]
    images: List[Dict[str, Any]]
    formulas: List[Dict[str, str]]
    footnotes: List[Dict[str, str]]
    references: List[Dict[str, str]]
    metadata: Dict[str, Any]


@dataclass
class ProcessingResult:
    """Result of document processing."""
    document_id: str
    file_path: str
    structure: DocumentStructure
    extracted_text: str
    knowledge_entities: List[Dict[str, Any]]
    qa_pairs: List[Dict[str, str]]
    embeddings: Optional[np.ndarray] = None
    confidence_scores: Dict[str, float] = None
    gemini_insights: Optional[Dict[str, Any]] = None
    learned_patterns: Optional[Dict[str, Any]] = None


@dataclass
class GeminiDocumentInsight:
    """Insights generated by Gemini about a document."""
    document_type: str
    key_topics: List[str]
    main_concepts: List[Dict[str, Any]]
    relationships: List[Dict[str, Any]]
    summary: str
    importance_score: float
    complexity_level: str
    domain_classification: List[str]
    actionable_insights: List[str]
    data_quality_assessment: Dict[str, float]


class AdvancedDocumentProcessor:
    """
    Advanced multi-modal document processor with OCR, table recognition,
    and Gemini-powered intelligent knowledge extraction and learning.
    """
    
    def __init__(self, gemini_client: Optional[GeminiClient] = None):
        self.logger = get_logger(__name__)
        self.gemini_client = gemini_client
        self.temp_dir = Path(tempfile.gettempdir()) / "toxo_processing"
        self.temp_dir.mkdir(exist_ok=True)
        
        # Processing configuration
        self.ocr_languages = ['eng', 'fra', 'deu', 'spa']  # Multi-language support
        self.table_extraction_methods = ['camelot', 'pdfplumber', 'pymupdf']
        self.image_analysis_enabled = True
        self.formula_parsing_enabled = True
        
        # Quality thresholds
        self.min_text_confidence = 0.7
        self.min_table_confidence = 0.8
        self.min_entity_confidence = 0.6
        
        # Gemini-powered learning
        self.document_learning_enabled = True
        self.pattern_memory: Dict[str, Any] = {}
        self.domain_expertise: Dict[str, List[Dict[str, Any]]] = {}
        
        self.logger.info("Advanced Document Processor initialized with Gemini intelligence")
    
    async def process_document(
        self,
        file_path: Union[str, Path],
        document_id: Optional[str] = None,
        processing_options: Optional[Dict[str, Any]] = None,
        learn_from_document: bool = True
    ) -> ProcessingResult:
        """
        Main entry point for document processing with Gemini-enhanced intelligence.
        
        Args:
            file_path: Path to the document
            document_id: Optional document identifier
            processing_options: Custom processing configuration
            learn_from_document: Whether to learn patterns from this document
            
        Returns:
            ProcessingResult with extracted content, knowledge, and Gemini insights
        """
        try:
            file_path = Path(file_path)
            if not file_path.exists():
                raise ProcessingError(f"Document not found: {file_path}")
            
            document_id = document_id or file_path.stem
            self.logger.info(f"Processing document with Gemini intelligence: {file_path}")
            
            # Determine file type and select appropriate processor
            mime_type, _ = mimetypes.guess_type(str(file_path))
            processor = self._get_processor_for_type(mime_type, file_path.suffix)
            
            # Extract basic structure and content
            structure = await processor(file_path, processing_options or {})
            
            # Extract text content
            extracted_text = self._compile_extracted_text(structure)
            
            # Gemini-powered intelligent analysis
            gemini_insights = None
            if self.gemini_client:
                self.logger.info("Analyzing document with Gemini AI...")
                gemini_insights = await self._analyze_document_with_gemini(
                    extracted_text, structure, file_path
                )
            
            # Enhanced knowledge extraction using Gemini insights
            knowledge_entities = await self._extract_enhanced_knowledge_entities(
                extracted_text, gemini_insights
            )
            
            # Generate intelligent Q&A pairs using Gemini
            qa_pairs = await self._generate_intelligent_qa_pairs(
                extracted_text, structure, gemini_insights
            )
            
            # Learn patterns from document if enabled
            learned_patterns = None
            if learn_from_document and self.gemini_client:
                learned_patterns = await self._learn_document_patterns(
                    extracted_text, structure, gemini_insights
                )
            
            # Generate embeddings if Gemini client available
            embeddings = None
            if self.gemini_client:
                embeddings = await self._generate_embeddings(extracted_text)
            
            # Calculate enhanced confidence scores
            confidence_scores = self._calculate_enhanced_confidence_scores(
                structure, knowledge_entities, gemini_insights
            )
            
            result = ProcessingResult(
                document_id=document_id,
                file_path=str(file_path),
                structure=structure,
                extracted_text=extracted_text,
                knowledge_entities=knowledge_entities,
                qa_pairs=qa_pairs,
                embeddings=embeddings,
                confidence_scores=confidence_scores,
                gemini_insights=gemini_insights,
                learned_patterns=learned_patterns
            )
            
            self.logger.info(f"Successfully processed {len(extracted_text)} characters with Gemini enhancement")
            return result
            
        except Exception as e:
            self.logger.error(f"Error processing document {file_path}: {str(e)}")
            raise ProcessingError(f"Document processing failed: {str(e)}") from e
    
    async def _analyze_document_with_gemini(
        self, 
        text: str, 
        structure: DocumentStructure, 
        file_path: Path
    ) -> GeminiDocumentInsight:
        """
        Use Gemini to analyze document and extract intelligent insights.
        """
        try:
            # Prepare comprehensive analysis prompt
            analysis_prompt = f"""
            Analyze this document comprehensively and provide detailed insights:
            
            DOCUMENT METADATA:
            - Title: {structure.title}
            - File: {file_path.name}
            - Sections: {len(structure.sections)}
            - Tables: {len(structure.tables)}
            - Images: {len(structure.images)}
            
            DOCUMENT CONTENT:
            {text[:8000]}  # First 8000 characters
            
            Please provide a JSON response with the following analysis:
            {{
                "document_type": "academic_paper|business_report|legal_document|technical_manual|financial_statement|other",
                "key_topics": ["topic1", "topic2", "topic3"],
                "main_concepts": [
                    {{"concept": "concept_name", "definition": "definition", "importance": 0.9}}
                ],
                "relationships": [
                    {{"entity1": "A", "relationship": "causes", "entity2": "B", "confidence": 0.8}}
                ],
                "summary": "Brief comprehensive summary",
                "importance_score": 0.85,
                "complexity_level": "beginner|intermediate|advanced|expert",
                "domain_classification": ["domain1", "domain2"],
                "actionable_insights": ["insight1", "insight2"],
                "data_quality_assessment": {{
                    "completeness": 0.9,
                    "accuracy": 0.85,
                    "consistency": 0.8,
                    "relevance": 0.9
                }}
            }}
            """
            
            response = await self.gemini_client.generate(analysis_prompt)
            
            # Parse JSON response
            analysis_data = self._extract_json_from_response(response)
            
            if analysis_data:
                return GeminiDocumentInsight(
                    document_type=analysis_data.get("document_type", "other"),
                    key_topics=analysis_data.get("key_topics", []),
                    main_concepts=analysis_data.get("main_concepts", []),
                    relationships=analysis_data.get("relationships", []),
                    summary=analysis_data.get("summary", ""),
                    importance_score=analysis_data.get("importance_score", 0.5),
                    complexity_level=analysis_data.get("complexity_level", "intermediate"),
                    domain_classification=analysis_data.get("domain_classification", []),
                    actionable_insights=analysis_data.get("actionable_insights", []),
                    data_quality_assessment=analysis_data.get("data_quality_assessment", {})
                )
            else:
                self.logger.warning("Failed to parse Gemini document analysis response")
                return self._create_default_insight()
                
        except Exception as e:
            self.logger.error(f"Error in Gemini document analysis: {str(e)}")
            return self._create_default_insight()
    
    async def _extract_enhanced_knowledge_entities(
        self, 
        text: str, 
        insights: Optional[GeminiDocumentInsight]
    ) -> List[Dict[str, Any]]:
        """
        Extract knowledge entities using Gemini intelligence and document insights.
        """
        if not self.gemini_client:
            return await self._extract_knowledge_entities(text)
        
        try:
            # Use insights to guide entity extraction
            context = ""
            if insights:
                context = f"""
                DOCUMENT CONTEXT:
                - Type: {insights.document_type}
                - Key Topics: {', '.join(insights.key_topics)}
                - Domain: {', '.join(insights.domain_classification)}
                - Complexity: {insights.complexity_level}
                """
            
            entity_prompt = f"""
            {context}
            
            Extract detailed knowledge entities from this text:
            
            {text[:6000]}
            
            Provide a JSON list of entities with the following structure:
            [
                {{
                    "entity": "entity_name",
                    "type": "person|organization|concept|location|date|number|process|tool",
                    "description": "detailed description",
                    "context": "how it appears in document",
                    "importance": 0.85,
                    "relationships": ["related_entity1", "related_entity2"],
                    "attributes": {{"attribute1": "value1", "attribute2": "value2"}}
                }}
            ]
            
            Focus on the most important and relevant entities for this document type.
            """
            
            response = await self.gemini_client.generate(entity_prompt)
            entities = self._extract_json_from_response(response)
            
            if isinstance(entities, list):
                self.logger.info(f"Extracted {len(entities)} enhanced knowledge entities")
                return entities
            else:
                self.logger.warning("Invalid entity format from Gemini, falling back to basic extraction")
                return await self._extract_knowledge_entities(text)
                
        except Exception as e:
            self.logger.error(f"Error in enhanced entity extraction: {str(e)}")
            return await self._extract_knowledge_entities(text)
    
    async def _generate_intelligent_qa_pairs(
        self, 
        text: str, 
        structure: DocumentStructure, 
        insights: Optional[GeminiDocumentInsight]
    ) -> List[Dict[str, str]]:
        """
        Generate intelligent Q&A pairs using Gemini's understanding of the document.
        """
        if not self.gemini_client:
            return await self._generate_qa_pairs(text, structure)
        
        try:
            # Create context from insights
            context = ""
            if insights:
                context = f"""
                DOCUMENT INSIGHTS:
                - Type: {insights.document_type}
                - Summary: {insights.summary}
                - Key Topics: {', '.join(insights.key_topics)}
                - Complexity: {insights.complexity_level}
                """
            
            qa_prompt = f"""
            {context}
            
            Generate intelligent question-answer pairs from this document content:
            
            {text[:7000]}
            
            Create questions that:
            1. Test understanding of key concepts
            2. Explore relationships between ideas
            3. Apply knowledge to new scenarios
            4. Evaluate critical thinking
            5. Cover different difficulty levels
            
            Provide a JSON list with this structure:
            [
                {{
                    "question": "What is the relationship between X and Y in this context?",
                    "answer": "Detailed answer with explanation",
                    "difficulty": "easy|medium|hard",
                    "category": "factual|conceptual|analytical|application",
                    "key_concepts": ["concept1", "concept2"]
                }}
            ]
            
            Generate 8-12 high-quality questions covering the document comprehensively.
            """
            
            response = await self.gemini_client.generate(qa_prompt)
            qa_pairs = self._extract_json_from_response(response)
            
            if isinstance(qa_pairs, list):
                self.logger.info(f"Generated {len(qa_pairs)} intelligent Q&A pairs")
                return qa_pairs
            else:
                self.logger.warning("Invalid Q&A format from Gemini, falling back to basic generation")
                return await self._generate_qa_pairs(text, structure)
                
        except Exception as e:
            self.logger.error(f"Error in intelligent Q&A generation: {str(e)}")
            return await self._generate_qa_pairs(text, structure)
    
    async def _learn_document_patterns(
        self, 
        text: str, 
        structure: DocumentStructure, 
        insights: Optional[GeminiDocumentInsight]
    ) -> Dict[str, Any]:
        """
        Learn patterns from the document to improve future processing.
        """
        if not self.gemini_client or not insights:
            return {}
        
        try:
            learning_prompt = f"""
            Analyze this document to learn patterns for improving future document processing:
            
            DOCUMENT TYPE: {insights.document_type}
            DOMAIN: {', '.join(insights.domain_classification)}
            
            STRUCTURE ANALYSIS:
            - Sections: {len(structure.sections)}
            - Tables: {len(structure.tables)} 
            - Images: {len(structure.images)}
            - Title: {structure.title}
            
            CONTENT SAMPLE:
            {text[:5000]}
            
            Learn and provide patterns in JSON format:
            {{
                "document_patterns": {{
                    "typical_structure": ["section1", "section2", "section3"],
                    "common_terminology": ["term1", "term2"],
                    "data_formats": ["format1", "format2"],
                    "key_indicators": ["indicator1", "indicator2"]
                }},
                "extraction_hints": {{
                    "important_sections": ["section_type1", "section_type2"],
                    "critical_tables": ["table_type1", "table_type2"],
                    "key_figures": ["figure_type1", "figure_type2"]
                }},
                "quality_indicators": {{
                    "completeness_markers": ["marker1", "marker2"],
                    "accuracy_signals": ["signal1", "signal2"]
                }},
                "domain_specific_rules": [
                    "rule1: explanation",
                    "rule2: explanation"
                ]
            }}
            """
            
            response = await self.gemini_client.generate(learning_prompt)
            patterns = self._extract_json_from_response(response)
            
            if patterns:
                # Store learned patterns in memory
                domain_key = insights.document_type
                if domain_key not in self.domain_expertise:
                    self.domain_expertise[domain_key] = []
                
                self.domain_expertise[domain_key].append({
                    "timestamp": time.time(),
                    "patterns": patterns,
                    "source_document": structure.title
                })
                
                self.logger.info(f"Learned patterns for domain: {domain_key}")
                return patterns
            
        except Exception as e:
            self.logger.error(f"Error learning document patterns: {str(e)}")
        
        return {}
    
    async def get_intelligent_data_extraction_suggestions(
        self, 
        text: str, 
        target_fields: List[str]
    ) -> Dict[str, Any]:
        """
        Get Gemini-powered suggestions for extracting specific data fields.
        """
        if not self.gemini_client:
            return {}
        
        try:
            suggestion_prompt = f"""
            Help extract specific data fields from this document:
            
            TARGET FIELDS TO EXTRACT:
            {', '.join(target_fields)}
            
            DOCUMENT CONTENT:
            {text[:6000]}
            
            Provide extraction suggestions in JSON format:
            {{
                "field_extractions": {{
                    "field_name": {{
                        "value": "extracted_value",
                        "confidence": 0.9,
                        "location": "where found in document",
                        "extraction_method": "regex|keyword|context",
                        "alternative_values": ["alt1", "alt2"]
                    }}
                }},
                "extraction_patterns": {{
                    "field_name": "regex_pattern_or_description"
                }},
                "validation_suggestions": [
                    "Check that dates are in valid format",
                    "Verify numbers are within expected range"
                ]
            }}
            """
            
            response = await self.gemini_client.generate(suggestion_prompt)
            suggestions = self._extract_json_from_response(response)
            
            if suggestions:
                self.logger.info(f"Generated extraction suggestions for {len(target_fields)} fields")
                return suggestions
            
        except Exception as e:
            self.logger.error(f"Error generating extraction suggestions: {str(e)}")
        
        return {}
    
    async def learn_from_user_feedback(
        self, 
        document_id: str, 
        feedback: Dict[str, Any]
    ) -> None:
        """
        Learn from user feedback to improve future processing.
        """
        if not self.gemini_client:
            return
        
        try:
            learning_prompt = f"""
            Learn from user feedback about document processing:
            
            DOCUMENT ID: {document_id}
            
            USER FEEDBACK:
            {json.dumps(feedback, indent=2)}
            
            Analyze this feedback and suggest improvements:
            {{
                "identified_issues": ["issue1", "issue2"],
                "improvement_suggestions": ["suggestion1", "suggestion2"],
                "pattern_updates": {{
                    "pattern_type": "updated_pattern"
                }},
                "confidence_adjustments": {{
                    "metric": "adjustment_direction"
                }}
            }}
            """
            
            response = await self.gemini_client.generate(learning_prompt)
            improvements = self._extract_json_from_response(response)
            
            if improvements:
                # Apply improvements to pattern memory
                self.pattern_memory[f"feedback_{document_id}"] = {
                    "timestamp": time.time(),
                    "improvements": improvements,
                    "original_feedback": feedback
                }
                
                self.logger.info(f"Learned from feedback for document: {document_id}")
            
        except Exception as e:
            self.logger.error(f"Error learning from feedback: {str(e)}")
    
    def _extract_json_from_response(self, response: str) -> Optional[Dict[str, Any]]:
        """
        Extract JSON from Gemini response handling various formats.
        """
        try:
            # First try direct JSON parsing
            return json.loads(response)
        except json.JSONDecodeError:
            # Try to find JSON within the response
            import re
            json_match = re.search(r'\{.*\}', response, re.DOTALL)
            if json_match:
                try:
                    return json.loads(json_match.group())
                except json.JSONDecodeError:
                    pass
            
            # Try to find JSON array
            array_match = re.search(r'\[.*\]', response, re.DOTALL)
            if array_match:
                try:
                    return json.loads(array_match.group())
                except json.JSONDecodeError:
                    pass
        
        self.logger.warning("Could not extract JSON from Gemini response")
        return None
    
    def _create_default_insight(self) -> GeminiDocumentInsight:
        """Create a default insight when Gemini analysis fails."""
        return GeminiDocumentInsight(
            document_type="other",
            key_topics=[],
            main_concepts=[],
            relationships=[],
            summary="Document analysis unavailable",
            importance_score=0.5,
            complexity_level="intermediate",
            domain_classification=[],
            actionable_insights=[],
            data_quality_assessment={}
        )
    
    def _calculate_enhanced_confidence_scores(
        self, 
        structure: DocumentStructure, 
        entities: List[Dict[str, Any]],
        insights: Optional[GeminiDocumentInsight]
    ) -> Dict[str, float]:
        """
        Calculate enhanced confidence scores using Gemini insights.
        """
        base_scores = self._calculate_confidence_scores(structure, entities)
        
        if insights and insights.data_quality_assessment:
            # Enhance with Gemini's quality assessment
            gemini_scores = insights.data_quality_assessment
            
            # Weighted combination of base and Gemini scores
            enhanced_scores = {}
            for key, base_score in base_scores.items():
                gemini_key = key.replace('_confidence', '').replace('_score', '')
                gemini_score = gemini_scores.get(gemini_key, base_score)
                
                # 70% base processing, 30% Gemini assessment
                enhanced_scores[key] = 0.7 * base_score + 0.3 * gemini_score
            
            # Add Gemini-specific scores
            enhanced_scores['gemini_analysis_confidence'] = insights.importance_score
            enhanced_scores['overall_quality'] = np.mean(list(gemini_scores.values())) if gemini_scores else 0.5
            
            return enhanced_scores
        
        return base_scores
    
    def _get_processor_for_type(self, mime_type: str, suffix: str) -> callable:
        """Select appropriate processor based on file type."""
        if mime_type and mime_type.startswith('application/pdf') or suffix == '.pdf':
            return self._process_pdf
        elif mime_type and 'spreadsheet' in mime_type or suffix in ['.xlsx', '.xls', '.csv']:
            return self._process_spreadsheet
        elif mime_type and 'word' in mime_type or suffix in ['.docx', '.doc']:
            return self._process_word_document
        elif mime_type and mime_type.startswith('image/'):
            return self._process_image
        elif suffix in ['.json', '.xml', '.yaml', '.yml']:
            return self._process_structured_data
        else:
            return self._process_generic_text
    
    async def _process_pdf(self, file_path: Path, options: Dict[str, Any]) -> DocumentStructure:
        """Advanced PDF processing with OCR and table extraction."""
        self.logger.info(f"Processing PDF: {file_path}")
        
        # Use PyMuPDF for comprehensive analysis
        doc = pymupdf.open(str(file_path))
        
        title = self._extract_pdf_title(doc)
        sections = []
        tables = []
        images = []
        formulas = []
        footnotes = []
        references = []
        
        for page_num in range(len(doc)):
            page = doc[page_num]
            
            # Extract text with position information
            page_text = page.get_text("dict")
            section_content = self._analyze_pdf_page_structure(page_text, page_num)
            sections.extend(section_content["sections"])
            
            # Extract tables using multiple methods
            page_tables = await self._extract_pdf_tables(file_path, page_num + 1)
            tables.extend(page_tables)
            
            # Extract images and analyze them
            if self.image_analysis_enabled:
                page_images = await self._extract_pdf_images(page, page_num)
                images.extend(page_images)
            
            # Extract mathematical formulas
            if self.formula_parsing_enabled:
                page_formulas = self._extract_formulas(page_text)
                formulas.extend(page_formulas)
            
            # Extract footnotes and references
            page_footnotes, page_refs = self._extract_footnotes_and_references(page_text)
            footnotes.extend(page_footnotes)
            references.extend(page_refs)
        
        doc.close()
        
        # Compile metadata
        metadata = {
            "total_pages": len(doc),
            "extraction_method": "PyMuPDF + OCR",
            "processing_timestamp": self._get_timestamp(),
            "file_size": file_path.stat().st_size,
            "mime_type": "application/pdf"
        }
        
        return DocumentStructure(
            title=title,
            sections=sections,
            tables=tables,
            images=images,
            formulas=formulas,
            footnotes=footnotes,
            references=references,
            metadata=metadata
        )
    
    async def _extract_pdf_tables(self, file_path: Path, page_num: int) -> List[Dict[str, Any]]:
        """Extract tables using multiple methods for best results."""
        tables = []
        
        # Method 1: Camelot (best for structured tables)
        try:
            camelot_tables = camelot.read_pdf(str(file_path), pages=str(page_num), flavor='lattice')
            for i, table in enumerate(camelot_tables):
                if table.accuracy > self.min_table_confidence * 100:
                    tables.append({
                        "table_id": f"page_{page_num}_table_{i}_camelot",
                        "method": "camelot",
                        "data": table.df.to_dict('records'),
                        "accuracy": table.accuracy / 100,
                        "bbox": table._bbox if hasattr(table, '_bbox') else None,
                        "page": page_num
                    })
        except Exception as e:
            self.logger.warning(f"Camelot table extraction failed for page {page_num}: {e}")
        
        # Method 2: pdfplumber (good for complex layouts)
        try:
            import pdfplumber
            with pdfplumber.open(file_path) as pdf:
                if page_num - 1 < len(pdf.pages):
                    page = pdf.pages[page_num - 1]
                    plumber_tables = page.extract_tables()
                    for i, table in enumerate(plumber_tables):
                        if table and len(table) > 1:  # At least header + one row
                            df = pd.DataFrame(table[1:], columns=table[0])
                            tables.append({
                                "table_id": f"page_{page_num}_table_{i}_plumber",
                                "method": "pdfplumber",
                                "data": df.to_dict('records'),
                                "accuracy": 0.85,  # Estimated
                                "bbox": None,
                                "page": page_num
                            })
        except Exception as e:
            self.logger.warning(f"pdfplumber table extraction failed for page {page_num}: {e}")
        
        return tables
    
    async def _extract_pdf_images(self, page, page_num: int) -> List[Dict[str, Any]]:
        """Extract and analyze images from PDF page."""
        images = []
        image_list = page.get_images()
        
        for img_index, img in enumerate(image_list):
            try:
                # Extract image
                xref = img[0]
                pix = pymupdf.Pixmap(page.parent, xref)
                
                # Convert to PIL Image
                img_data = pix.tobytes("png")
                pil_image = Image.open(BytesIO(img_data))
                
                # Perform OCR on image if it contains text
                ocr_text = ""
                if self._image_likely_contains_text(pil_image):
                    ocr_text = pytesseract.image_to_string(pil_image, lang='+'.join(self.ocr_languages))
                
                # Analyze image with Gemini Vision if available
                image_analysis = ""
                if self.gemini_client and hasattr(self.gemini_client, 'analyze_image'):
                    image_analysis = await self.gemini_client.analyze_image(img_data)
                
                images.append({
                    "image_id": f"page_{page_num}_img_{img_index}",
                    "page": page_num,
                    "bbox": page.get_image_bbox(img),
                    "ocr_text": ocr_text.strip(),
                    "ai_analysis": image_analysis,
                    "dimensions": pix.size,
                    "format": "PNG"
                })
                
                pix = None  # Free memory
                
            except Exception as e:
                self.logger.warning(f"Failed to process image {img_index} on page {page_num}: {e}")
        
        return images
    
    async def _process_spreadsheet(self, file_path: Path, options: Dict[str, Any]) -> DocumentStructure:
        """Advanced spreadsheet processing with schema inference."""
        self.logger.info(f"Processing spreadsheet: {file_path}")
        
        if file_path.suffix.lower() == '.csv':
            df = pd.read_csv(file_path)
            sheets = {"Sheet1": df}
        else:
            sheets = pd.read_excel(file_path, sheet_name=None)
        
        title = file_path.stem
        sections = []
        tables = []
        
        for sheet_name, df in sheets.items():
            # Infer schema and data types
            schema = self._infer_spreadsheet_schema(df)
            
            # Detect data patterns and relationships
            patterns = self._detect_data_patterns(df)
            
            # Extract meaningful tables
            table_data = {
                "table_id": f"sheet_{sheet_name}",
                "sheet_name": sheet_name,
                "data": df.to_dict('records'),
                "schema": schema,
                "patterns": patterns,
                "rows": len(df),
                "columns": len(df.columns),
                "accuracy": 1.0  # Spreadsheets are typically clean
            }
            tables.append(table_data)
            
            # Create section for this sheet
            section = {
                "section_id": f"sheet_{sheet_name}",
                "title": sheet_name,
                "content": f"Spreadsheet data with {len(df)} rows and {len(df.columns)} columns",
                "level": 1,
                "page": 1
            }
            sections.append(section)
        
        metadata = {
            "total_sheets": len(sheets),
            "extraction_method": "pandas",
            "processing_timestamp": self._get_timestamp(),
            "file_size": file_path.stat().st_size
        }
        
        return DocumentStructure(
            title=title,
            sections=sections,
            tables=tables,
            images=[],
            formulas=[],
            footnotes=[],
            references=[],
            metadata=metadata
        )
    
    async def _extract_knowledge_entities(self, text: str) -> List[Dict[str, Any]]:
        """Extract knowledge entities using AI analysis."""
        if not self.gemini_client or len(text.strip()) < 50:
            return []
        
        try:
            extraction_prompt = f"""
            Analyze the following text and extract key knowledge entities including:
            1. Important concepts and definitions
            2. Key facts and figures
            3. Relationships between entities
            4. Domain-specific terminology
            5. Actionable insights
            
            Text to analyze:
            {text[:4000]}  # Limit text length for API
            
            Return the results as a JSON array with objects containing:
            - entity: the identified entity/concept
            - type: the type of entity (concept, fact, relationship, term, insight)
            - description: detailed description
            - confidence: confidence score (0.0 to 1.0)
            - context: relevant context from the text
            """
            
            response = await self.gemini_client.generate(extraction_prompt)
            
            # Parse JSON response
            try:
                entities = json.loads(response)
                # Filter by confidence threshold
                return [e for e in entities if e.get('confidence', 0) >= self.min_entity_confidence]
            except json.JSONDecodeError:
                self.logger.warning("Failed to parse knowledge entities JSON response")
                return []
            
        except Exception as e:
            self.logger.error(f"Knowledge entity extraction failed: {e}")
            return []
    
    async def _generate_qa_pairs(self, text: str, structure: DocumentStructure) -> List[Dict[str, str]]:
        """Generate question-answer pairs for training purposes."""
        if not self.gemini_client or len(text.strip()) < 100:
            return []
        
        try:
            qa_prompt = f"""
            Based on the following document content, generate diverse question-answer pairs that would be useful for training an AI system:
            
            Document title: {structure.title}
            Content: {text[:3000]}  # Limit for API
            
            Generate 10-15 question-answer pairs covering:
            1. Factual questions about specific information
            2. Conceptual questions about key ideas
            3. Application questions about how to use the information
            4. Analysis questions requiring understanding
            5. Comparison questions between different concepts
            
            Return as JSON array with objects containing 'question' and 'answer' fields.
            """
            
            response = await self.gemini_client.generate(qa_prompt)
            
            try:
                qa_pairs = json.loads(response)
                return qa_pairs if isinstance(qa_pairs, list) else []
            except json.JSONDecodeError:
                self.logger.warning("Failed to parse QA pairs JSON response")
                return []
            
        except Exception as e:
            self.logger.error(f"QA pair generation failed: {e}")
            return []
    
    def _infer_spreadsheet_schema(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Infer schema and patterns from spreadsheet data."""
        schema = {
            "columns": {},
            "constraints": {},
            "relationships": []
        }
        
        for column in df.columns:
            col_data = df[column].dropna()
            if len(col_data) == 0:
                continue
            
            # Infer data type
            if pd.api.types.is_numeric_dtype(col_data):
                data_type = "numeric"
                stats = {
                    "min": float(col_data.min()),
                    "max": float(col_data.max()),
                    "mean": float(col_data.mean()),
                    "std": float(col_data.std())
                }
            elif pd.api.types.is_datetime64_any_dtype(col_data):
                data_type = "datetime"
                stats = {
                    "min": str(col_data.min()),
                    "max": str(col_data.max())
                }
            else:
                data_type = "text"
                stats = {
                    "unique_values": int(col_data.nunique()),
                    "avg_length": float(col_data.astype(str).str.len().mean())
                }
            
            schema["columns"][column] = {
                "type": data_type,
                "nullable": df[column].isnull().any(),
                "stats": stats
            }
        
        return schema
    
    def _detect_data_patterns(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Detect patterns and relationships in spreadsheet data."""
        patterns = {
            "trends": [],
            "correlations": [],
            "anomalies": [],
            "periodicity": []
        }
        
        # Detect correlations between numeric columns
        numeric_cols = df.select_dtypes(include=[np.number]).columns
        if len(numeric_cols) > 1:
            corr_matrix = df[numeric_cols].corr()
            high_corr = corr_matrix[(abs(corr_matrix) > 0.7) & (abs(corr_matrix) < 1.0)]
            
            for i in range(len(high_corr.columns)):
                for j in range(i + 1, len(high_corr.columns)):
                    col1, col2 = high_corr.columns[i], high_corr.columns[j]
                    if not pd.isna(high_corr.iloc[i, j]):
                        patterns["correlations"].append({
                            "column1": col1,
                            "column2": col2,
                            "correlation": float(high_corr.iloc[i, j])
                        })
        
        return patterns
    
    def _compile_extracted_text(self, structure: DocumentStructure) -> str:
        """Compile all extracted text from document structure."""
        text_parts = []
        
        # Add title
        if structure.title:
            text_parts.append(f"Title: {structure.title}")
        
        # Add sections
        for section in structure.sections:
            if section.get("content"):
                text_parts.append(f"\nSection: {section.get('title', 'Untitled')}")
                text_parts.append(section["content"])
        
        # Add table summaries
        for table in structure.tables:
            if table.get("data"):
                text_parts.append(f"\nTable: {table.get('table_id', 'Unknown')}")
                # Add a summary of the table structure
                if isinstance(table["data"], list) and table["data"]:
                    text_parts.append(f"Table with {len(table['data'])} rows")
        
        # Add image OCR text
        for image in structure.images:
            if image.get("ocr_text"):
                text_parts.append(f"\nImage text: {image['ocr_text']}")
        
        return "\n".join(text_parts)
    
    def _calculate_confidence_scores(
        self, 
        structure: DocumentStructure, 
        entities: List[Dict[str, Any]]
    ) -> Dict[str, float]:
        """Calculate overall confidence scores for the processing."""
        scores = {}
        
        # Text extraction confidence
        total_sections = len(structure.sections)
        sections_with_content = sum(1 for s in structure.sections if s.get("content"))
        scores["text_extraction"] = sections_with_content / max(total_sections, 1)
        
        # Table extraction confidence
        if structure.tables:
            table_confidences = [t.get("accuracy", 0.5) for t in structure.tables]
            scores["table_extraction"] = sum(table_confidences) / len(table_confidences)
        else:
            scores["table_extraction"] = 1.0  # No tables to extract
        
        # Knowledge extraction confidence
        if entities:
            entity_confidences = [e.get("confidence", 0.5) for e in entities]
            scores["knowledge_extraction"] = sum(entity_confidences) / len(entity_confidences)
        else:
            scores["knowledge_extraction"] = 0.0
        
        # Overall confidence
        scores["overall"] = sum(scores.values()) / len(scores)
        
        return scores
    
    # Helper methods
    def _extract_pdf_title(self, doc) -> str:
        """Extract title from PDF metadata or first page."""
        # Try metadata first
        metadata = doc.metadata
        if metadata and metadata.get("title"):
            return metadata["title"]
        
        # Try first page
        if len(doc) > 0:
            first_page = doc[0]
            text_dict = first_page.get_text("dict")
            # Look for largest text block (likely title)
            blocks = text_dict.get("blocks", [])
            if blocks:
                # Find block with largest font size
                max_font_size = 0
                title_text = ""
                for block in blocks:
                    if "lines" in block:
                        for line in block["lines"]:
                            for span in line.get("spans", []):
                                if span.get("size", 0) > max_font_size:
                                    max_font_size = span["size"]
                                    title_text = span.get("text", "").strip()
                if title_text:
                    return title_text
        
        return "Untitled Document"
    
    def _analyze_pdf_page_structure(self, page_text: Dict, page_num: int) -> Dict[str, Any]:
        """Analyze the structure of a PDF page."""
        sections = []
        
        blocks = page_text.get("blocks", [])
        for i, block in enumerate(blocks):
            if "lines" in block:
                content = ""
                for line in block["lines"]:
                    for span in line.get("spans", []):
                        content += span.get("text", "") + " "
                
                content = content.strip()
                if content and len(content) > 10:  # Ignore very short content
                    sections.append({
                        "section_id": f"page_{page_num}_block_{i}",
                        "title": content[:50] + "..." if len(content) > 50 else content,
                        "content": content,
                        "level": 1,
                        "page": page_num,
                        "bbox": block.get("bbox")
                    })
        
        return {"sections": sections}
    
    def _extract_formulas(self, page_text: Dict) -> List[Dict[str, str]]:
        """Extract mathematical formulas from page text."""
        formulas = []
        # This is a simplified implementation
        # In production, you'd want more sophisticated formula detection
        
        blocks = page_text.get("blocks", [])
        for block in blocks:
            if "lines" in block:
                for line in block["lines"]:
                    for span in line.get("spans", []):
                        text = span.get("text", "")
                        # Simple heuristic for formula detection
                        if any(char in text for char in "∫∑∏√±∞≠≤≥αβγδ"):
                            formulas.append({
                                "formula_id": f"formula_{len(formulas)}",
                                "content": text,
                                "type": "mathematical"
                            })
        
        return formulas
    
    def _extract_footnotes_and_references(self, page_text: Dict) -> Tuple[List[Dict], List[Dict]]:
        """Extract footnotes and references from page text."""
        footnotes = []
        references = []
        
        # This is a simplified implementation
        # Production version would use more sophisticated NLP
        
        return footnotes, references
    
    def _image_likely_contains_text(self, image: Image.Image) -> bool:
        """Determine if an image likely contains text worth OCR processing."""
        # Simple heuristic based on image characteristics
        width, height = image.size
        aspect_ratio = width / height
        
        # Text-heavy images often have certain aspect ratios
        if 0.1 < aspect_ratio < 10:  # Not too narrow or wide
            return True
        
        return False
    
    async def _generate_embeddings(self, text: str) -> Optional[np.ndarray]:
        """Generate embeddings for the extracted text."""
        if not self.gemini_client:
            return None
        
        try:
            # Use Gemini's embedding capability
            embeddings = await self.gemini_client.embed_text(text[:8000])  # Limit text
            return np.array(embeddings) if embeddings else None
        except Exception as e:
            self.logger.warning(f"Failed to generate embeddings: {e}")
            return None
    
    def _get_timestamp(self) -> str:
        """Get current timestamp."""
        from datetime import datetime
        return datetime.now().isoformat()
    
    # Additional processors for other file types
    async def _process_word_document(self, file_path: Path, options: Dict[str, Any]) -> DocumentStructure:
        """Process Word documents."""
        # Implementation for DOCX files
        pass
    
    async def _process_image(self, file_path: Path, options: Dict[str, Any]) -> DocumentStructure:
        """Process image files with OCR."""
        # Implementation for image files
        pass
    
    async def _process_structured_data(self, file_path: Path, options: Dict[str, Any]) -> DocumentStructure:
        """Process JSON, XML, YAML files."""
        # Implementation for structured data
        pass
    
    async def _process_generic_text(self, file_path: Path, options: Dict[str, Any]) -> DocumentStructure:
        """Process generic text files."""
        try:
            # Read the text file
            content = file_path.read_text(encoding='utf-8')
            
            # Extract title from filename or first line
            title = file_path.stem
            if content:
                first_line = content.split('\n')[0].strip()
                if len(first_line) < 100 and not first_line.startswith(' '):
                    title = first_line
            
            # Create sections by splitting on double newlines or large gaps
            sections = []
            paragraphs = content.split('\n\n')
            
            for i, paragraph in enumerate(paragraphs):
                paragraph = paragraph.strip()
                if paragraph and len(paragraph) > 10:  # Skip very short paragraphs
                    # Use first few words as section title
                    section_title = ' '.join(paragraph.split()[:8])
                    if len(section_title) > 50:
                        section_title = section_title[:50] + "..."
                    
                    sections.append({
                        "section_id": f"paragraph_{i}",
                        "title": section_title,
                        "content": paragraph,
                        "level": 1
                    })
            
            # Create document structure
            structure = DocumentStructure(
                title=title,
                sections=sections,
                tables=[],
                images=[],
                formulas=[],
                footnotes=[],
                references=[],
                metadata={
                    "file_type": "text",
                    "file_size": len(content),
                    "encoding": "utf-8",
                    "paragraphs": len(paragraphs)
                }
            )
            
            return structure
            
        except Exception as e:
            self.logger.error(f"Error processing text file {file_path}: {str(e)}")
            # Return empty structure
            return DocumentStructure(
                title=file_path.stem,
                sections=[],
                tables=[],
                images=[],
                formulas=[],
                footnotes=[],
                references=[],
                metadata={"error": str(e)}
            )
    
    def get_state(self) -> Dict[str, Any]:
        """Get the current state for serialization."""
        return {
            "gemini_client_available": getattr(self, 'gemini_client', None) is not None,
            "config_available": getattr(self, 'config', None) is not None,
            "supported_formats_count": len(getattr(self, 'supported_formats', {})),
            "system_type": "AdvancedDocumentProcessor"
        } 