"""
Enhanced Multi-Modal Document Processing for Toxo - Phase 2 Implementation

This module implements advanced document processing capabilities including:
- Vision model integration for image and diagram analysis
- Advanced OCR with layout preservation
- Mathematical formula parsing and understanding
- Multi-column text flow understanding
- Cross-sheet relationship detection in spreadsheets
- Intelligent schema inference and validation
"""

import asyncio
import cv2
import numpy as np
import torch
from typing import Dict, List, Any, Optional, Tuple, Union
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
import tempfile
import json
import logging
import re
from PIL import Image, ImageDraw, ImageFont
import base64
from io import BytesIO

# Advanced document processing libraries
import pytesseract

# Optional advanced dependencies
try:
    import easyocr
    EASYOCR_AVAILABLE = True
except ImportError:
    easyocr = None
    EASYOCR_AVAILABLE = False

try:
    from transformers import (
        TrOCRProcessor, TrOCRForCausalLM,
        LayoutLMv3Processor, LayoutLMv3ForTokenClassification,
        DetrImageProcessor, DetrForObjectDetection
    )
    TRANSFORMERS_AVAILABLE = True
except ImportError:
    TRANSFORMERS_AVAILABLE = False

try:
    import layoutparser as lp
    LAYOUTPARSER_AVAILABLE = True
except ImportError:
    lp = None
    LAYOUTPARSER_AVAILABLE = False

try:
    from pdf2image import convert_from_path
    PDF2IMAGE_AVAILABLE = True
except ImportError:
    convert_from_path = None
    PDF2IMAGE_AVAILABLE = False

try:
    import sympy
    from sympy.parsing.latex import parse_latex
    SYMPY_AVAILABLE = True
except ImportError:
    sympy = None
    parse_latex = None
    SYMPY_AVAILABLE = False

# Standard document processing libraries
import fitz  # PyMuPDF
import pandas as pd
import openpyxl
from openpyxl.utils import get_column_letter

# Toxo imports
from ..integrations.gemini_client import GeminiClient
from ..utils.logger import get_logger
from ..utils.exceptions import ProcessingError, ValidationError
from ..core.config import ToxoConfig


@dataclass
class LayoutElement:
    """Represents a layout element in a document."""
    element_type: str  # text, image, table, formula, diagram
    bbox: Tuple[int, int, int, int]  # x1, y1, x2, y2
    content: str
    confidence: float
    page_number: int
    metadata: Dict[str, Any] = field(default_factory=dict)


@dataclass
class DocumentLayout:
    """Represents the complete layout of a document."""
    page_layouts: Dict[int, List[LayoutElement]]
    reading_order: List[str]  # Element IDs in reading order
    column_structure: Dict[int, List[Tuple[int, int]]]  # Page -> column boundaries
    metadata: Dict[str, Any] = field(default_factory=dict)


@dataclass
class FormulaAnalysis:
    """Analysis of mathematical formulas."""
    formula_text: str
    latex_representation: str
    sympy_expression: Optional[str]
    variables: List[str]
    formula_type: str  # equation, inequality, expression
    complexity_score: float
    context: str


@dataclass
class TableStructure:
    """Enhanced table structure with relationships."""
    headers: List[str]
    data: List[List[str]]
    data_types: Dict[str, str]
    relationships: List[Dict[str, Any]]
    merged_cells: List[Tuple[int, int, int, int]]
    table_type: str  # data, pivot, summary, etc.
    confidence: float


class VisionModelProcessor:
    """Handles vision model processing for images and diagrams."""
    
    def __init__(self):
        self.logger = get_logger(f"{__name__}.VisionProcessor")
        
        # Initialize models
        self.layout_model = None
        self.ocr_processor = None
        self.object_detector = None
        self.easyocr_reader = None
        
        self._initialize_models()
    
    def _initialize_models(self):
        """Initialize vision models."""
        try:
            # Layout analysis model
            if LAYOUTPARSER_AVAILABLE:
                self.layout_model = lp.Detectron2LayoutModel(
                    'lp://PubLayNet/faster_rcnn_R_50_FPN_3x/config',
                    extra_config=["MODEL.ROI_HEADS.SCORE_THRESH_TEST", 0.8],
                    label_map={0: "Text", 1: "Title", 2: "List", 3: "Table", 4: "Figure"}
                )
            else:
                self.logger.warning("LayoutParser not available - layout analysis will be limited")
            
            # TrOCR for text recognition
            if TRANSFORMERS_AVAILABLE:
                self.ocr_processor = TrOCRProcessor.from_pretrained('microsoft/trocr-base-printed')
                self.ocr_model = TrOCRForCausalLM.from_pretrained('microsoft/trocr-base-printed')
                
                # Object detection for diagrams
                self.object_processor = DetrImageProcessor.from_pretrained("facebook/detr-resnet-50")
                self.object_model = DetrForObjectDetection.from_pretrained("facebook/detr-resnet-50")
            else:
                self.logger.warning("Transformers not available - advanced OCR will be limited")
            
            # EasyOCR for multi-language support
            if EASYOCR_AVAILABLE:
                self.easyocr_reader = easyocr.Reader(['en', 'fr', 'de', 'es'])
            else:
                self.logger.warning("EasyOCR not available - will use Tesseract only")
            
            self.logger.info("Vision models initialized (some features may be limited due to missing dependencies)")
            
        except Exception as e:
            self.logger.warning(f"Failed to initialize some vision models: {str(e)}")
            # Continue with basic functionality
    
    async def analyze_document_layout(self, image: Image.Image) -> DocumentLayout:
        """Analyze document layout using vision models."""
        try:
            # Convert PIL to OpenCV format
            cv_image = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
            
            # Detect layout elements
            layout_elements = []
            if self.layout_model:
                layout_result = self.layout_model.detect(cv_image)
                
                for element in layout_result:
                    bbox = element.coordinates
                    element_type = element.type
                    confidence = element.score
                    
                    # Extract text from element
                    x1, y1, x2, y2 = int(bbox[0]), int(bbox[1]), int(bbox[2]), int(bbox[3])
                    element_image = image.crop((x1, y1, x2, y2))
                    
                    if element_type in ["Text", "Title", "List"]:
                        text_content = await self._extract_text_from_region(element_image)
                    else:
                        text_content = f"{element_type} element"
                    
                    layout_element = LayoutElement(
                        element_type=element_type.lower(),
                        bbox=(x1, y1, x2, y2),
                        content=text_content,
                        confidence=confidence,
                        page_number=1,
                        metadata={'detected_by': 'layoutlm'}
                    )
                    layout_elements.append(layout_element)
            
            # Determine reading order
            reading_order = await self._determine_reading_order(layout_elements)
            
            # Detect column structure
            column_structure = await self._detect_column_structure(layout_elements, image.size)
            
            return DocumentLayout(
                page_layouts={1: layout_elements},
                reading_order=reading_order,
                column_structure=column_structure,
                metadata={'analysis_method': 'vision_model'}
            )
            
        except Exception as e:
            self.logger.error(f"Error analyzing document layout: {str(e)}")
            raise ProcessingError(f"Failed to analyze layout: {str(e)}")
    
    async def _extract_text_from_region(self, image: Image.Image) -> str:
        """Extract text from image region using multiple OCR methods."""
        texts = []
        
        # TrOCR
        if TRANSFORMERS_AVAILABLE and self.ocr_processor and self.ocr_model:
            try:
                pixel_values = self.ocr_processor(image, return_tensors="pt").pixel_values
                generated_ids = self.ocr_model.generate(pixel_values)
                generated_text = self.ocr_processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
                texts.append(generated_text)
            except Exception as e:
                self.logger.debug(f"TrOCR failed: {str(e)}")
        
        # EasyOCR
        if EASYOCR_AVAILABLE and self.easyocr_reader:
            try:
                cv_image = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
                results = self.easyocr_reader.readtext(cv_image)
                easyocr_text = " ".join([result[1] for result in results])
                texts.append(easyocr_text)
            except Exception as e:
                self.logger.debug(f"EasyOCR failed: {str(e)}")
        
        # Tesseract fallback (always available)
        try:
            tesseract_text = pytesseract.image_to_string(image)
            texts.append(tesseract_text)
        except Exception as e:
            self.logger.debug(f"Tesseract failed: {str(e)}")
        
        # Return the longest text (usually most accurate)
        if texts:
            return max(texts, key=len).strip()
        return ""
    
    async def _determine_reading_order(self, elements: List[LayoutElement]) -> List[str]:
        """Determine reading order of layout elements."""
        # Sort by vertical position first, then horizontal
        sorted_elements = sorted(elements, key=lambda e: (e.bbox[1], e.bbox[0]))
        return [f"element_{i}" for i in range(len(sorted_elements))]
    
    async def _detect_column_structure(
        self,
        elements: List[LayoutElement],
        image_size: Tuple[int, int]
    ) -> Dict[int, List[Tuple[int, int]]]:
        """Detect column structure in the document."""
        width, height = image_size
        
        # Find vertical gaps that might indicate columns
        x_positions = []
        for element in elements:
            if element.element_type in ['text', 'title']:
                x_positions.extend([element.bbox[0], element.bbox[2]])
        
        if not x_positions:
            return {1: [(0, width)]}
        
        # Cluster x positions to find column boundaries
        x_positions = sorted(set(x_positions))
        gaps = []
        
        for i in range(len(x_positions) - 1):
            gap_size = x_positions[i + 1] - x_positions[i]
            if gap_size > width * 0.05:  # Significant gap
                gaps.append((x_positions[i], x_positions[i + 1]))
        
        # Determine columns
        if len(gaps) >= 1:
            # Multi-column layout
            columns = []
            start = 0
            for gap_start, gap_end in gaps:
                if gap_start - start > width * 0.1:
                    columns.append((start, gap_start))
                start = gap_end
            
            if width - start > width * 0.1:
                columns.append((start, width))
            
            return {1: columns}
        else:
            # Single column
            return {1: [(0, width)]}


class FormulaProcessor:
    """Processes mathematical formulas and equations."""
    
    def __init__(self):
        self.logger = get_logger(f"{__name__}.FormulaProcessor")
    
    async def extract_and_analyze_formulas(self, text: str) -> List[FormulaAnalysis]:
        """Extract and analyze mathematical formulas from text."""
        formulas = []
        
        # Patterns for different formula formats
        latex_pattern = r'\$([^$]+)\$|\\\[([^\]]+)\\\]|\\\(([^\)]+)\\\)'
        equation_pattern = r'([a-zA-Z]\s*[=<>≤≥≠]\s*[^.!?]*)'
        
        # Find LaTeX formulas
        latex_matches = re.finditer(latex_pattern, text)
        for match in latex_matches:
            formula_text = match.group(1) or match.group(2) or match.group(3)
            if formula_text:
                analysis = await self._analyze_formula(formula_text, 'latex')
                if analysis:
                    formulas.append(analysis)
        
        # Find equation-like patterns
        equation_matches = re.finditer(equation_pattern, text)
        for match in equation_matches:
            formula_text = match.group(1)
            if formula_text and len(formula_text) < 200:  # Reasonable length
                analysis = await self._analyze_formula(formula_text, 'equation')
                if analysis:
                    formulas.append(analysis)
        
        return formulas
    
    async def _analyze_formula(self, formula_text: str, format_type: str) -> Optional[FormulaAnalysis]:
        """Analyze a single formula."""
        try:
            # Clean the formula
            cleaned_formula = formula_text.strip()
            
            # Try to parse with SymPy if available
            sympy_expr = None
            variables = []
            
            if SYMPY_AVAILABLE:
                try:
                    if format_type == 'latex':
                        sympy_expr = parse_latex(cleaned_formula)
                    else:
                        # Try to parse as regular expression
                        sympy_expr = sympy.sympify(cleaned_formula)
                    
                    variables = [str(var) for var in sympy_expr.free_symbols]
                except Exception as e:
                    self.logger.debug(f"SymPy parsing failed for '{cleaned_formula}': {str(e)}")
            else:
                # Basic variable extraction without SymPy
                variables = list(set(re.findall(r'\b[a-zA-Z]\b', cleaned_formula)))
            
            # Determine formula type
            formula_type = self._classify_formula(cleaned_formula)
            
            # Calculate complexity score
            complexity = self._calculate_complexity(cleaned_formula)
            
            return FormulaAnalysis(
                formula_text=cleaned_formula,
                latex_representation=cleaned_formula if format_type == 'latex' else '',
                sympy_expression=str(sympy_expr) if sympy_expr else None,
                variables=variables,
                formula_type=formula_type,
                complexity_score=complexity,
                context=''  # Could be enhanced with surrounding text
            )
            
        except Exception as e:
            self.logger.debug(f"Formula analysis failed: {str(e)}")
            return None
    
    def _classify_formula(self, formula: str) -> str:
        """Classify the type of formula."""
        if '=' in formula:
            return 'equation'
        elif any(op in formula for op in ['<', '>', '≤', '≥', '≠']):
            return 'inequality'
        elif any(func in formula for func in ['sin', 'cos', 'tan', 'log', 'exp']):
            return 'function'
        elif any(op in formula for op in ['+', '-', '*', '/', '^']):
            return 'expression'
        else:
            return 'unknown'
    
    def _calculate_complexity(self, formula: str) -> float:
        """Calculate complexity score for formula."""
        complexity = 0.0
        
        # Base complexity
        complexity += len(formula) * 0.01
        
        # Operators
        operators = ['+', '-', '*', '/', '^', '=', '<', '>']
        complexity += sum(formula.count(op) for op in operators) * 0.1
        
        # Functions
        functions = ['sin', 'cos', 'tan', 'log', 'exp', 'sqrt', 'int', 'sum']
        complexity += sum(formula.lower().count(func) for func in functions) * 0.2
        
        # Parentheses (nesting)
        complexity += formula.count('(') * 0.15
        
        # Greek letters and special symbols
        greek_letters = ['alpha', 'beta', 'gamma', 'delta', 'theta', 'lambda', 'mu', 'sigma']
        complexity += sum(formula.lower().count(letter) for letter in greek_letters) * 0.1
        
        return min(1.0, complexity)


class EnhancedSpreadsheetProcessor:
    """Enhanced spreadsheet processing with relationship detection."""
    
    def __init__(self):
        self.logger = get_logger(f"{__name__}.SpreadsheetProcessor")
    
    async def process_spreadsheet_advanced(self, file_path: Path) -> Dict[str, Any]:
        """Process spreadsheet with advanced analysis."""
        try:
            workbook = openpyxl.load_workbook(file_path, data_only=True)
            analysis_results = {
                'sheets': {},
                'cross_sheet_relationships': [],
                'data_flow': {},
                'schema_analysis': {}
            }
            
            # Process each sheet
            for sheet_name in workbook.sheetnames:
                sheet = workbook[sheet_name]
                sheet_analysis = await self._analyze_sheet_advanced(sheet, sheet_name)
                analysis_results['sheets'][sheet_name] = sheet_analysis
            
            # Analyze cross-sheet relationships
            cross_relationships = await self._analyze_cross_sheet_relationships(workbook)
            analysis_results['cross_sheet_relationships'] = cross_relationships
            
            # Analyze data flow
            data_flow = await self._analyze_data_flow(workbook)
            analysis_results['data_flow'] = data_flow
            
            return analysis_results
            
        except Exception as e:
            self.logger.error(f"Error processing spreadsheet: {str(e)}")
            raise ProcessingError(f"Failed to process spreadsheet: {str(e)}")
    
    async def _analyze_sheet_advanced(self, sheet, sheet_name: str) -> Dict[str, Any]:
        """Advanced analysis of a single sheet."""
        # Convert to DataFrame for analysis
        data = []
        for row in sheet.iter_rows(values_only=True):
            data.append(row)
        
        if not data:
            return {'type': 'empty', 'analysis': {}}
        
        df = pd.DataFrame(data)
        
        # Detect sheet type
        sheet_type = await self._detect_sheet_type(df, sheet)
        
        # Analyze data structure
        structure_analysis = await self._analyze_data_structure(df)
        
        # Detect formulas and dependencies
        formula_analysis = await self._analyze_formulas(sheet)
        
        # Detect merged cells and formatting
        formatting_analysis = await self._analyze_formatting(sheet)
        
        return {
            'type': sheet_type,
            'structure': structure_analysis,
            'formulas': formula_analysis,
            'formatting': formatting_analysis,
            'row_count': len(data),
            'column_count': len(data[0]) if data else 0
        }
    
    async def _detect_sheet_type(self, df: pd.DataFrame, sheet) -> str:
        """Detect the type of spreadsheet (data, pivot, dashboard, etc.)."""
        # Check for pivot table characteristics
        if self._has_pivot_characteristics(df):
            return 'pivot_table'
        
        # Check for dashboard characteristics
        if self._has_dashboard_characteristics(sheet):
            return 'dashboard'
        
        # Check for data table characteristics
        if self._has_data_table_characteristics(df):
            return 'data_table'
        
        # Check for calculation sheet
        if self._has_calculation_characteristics(sheet):
            return 'calculation_sheet'
        
        return 'general'
    
    def _has_pivot_characteristics(self, df: pd.DataFrame) -> bool:
        """Check if sheet has pivot table characteristics."""
        # Look for grouped headers, subtotals, etc.
        if df.empty:
            return False
        
        # Check for "Total" or "Grand Total" rows/columns
        text_data = df.astype(str).values.flatten()
        pivot_keywords = ['total', 'grand total', 'subtotal', 'sum of', 'count of']
        
        return any(keyword in ' '.join(text_data).lower() for keyword in pivot_keywords)
    
    def _has_dashboard_characteristics(self, sheet) -> bool:
        """Check if sheet has dashboard characteristics."""
        # Look for charts, multiple data regions, etc.
        # This is a simplified check
        merged_cells = len(list(sheet.merged_cells.ranges))
        return merged_cells > 5  # Dashboards often have many merged cells
    
    def _has_data_table_characteristics(self, df: pd.DataFrame) -> bool:
        """Check if sheet is a data table."""
        if df.empty or len(df) < 2:
            return False
        
        # Check if first row looks like headers
        first_row = df.iloc[0].astype(str)
        second_row = df.iloc[1].astype(str)
        
        # Headers are usually text, data is mixed
        first_row_text = sum(1 for val in first_row if not val.replace('.', '').isdigit())
        second_row_text = sum(1 for val in second_row if not val.replace('.', '').isdigit())
        
        return first_row_text > second_row_text
    
    def _has_calculation_characteristics(self, sheet) -> bool:
        """Check if sheet is primarily for calculations."""
        formula_count = 0
        total_cells = 0
        
        for row in sheet.iter_rows():
            for cell in row:
                total_cells += 1
                if cell.value and isinstance(cell.value, str) and cell.value.startswith('='):
                    formula_count += 1
        
        return total_cells > 0 and (formula_count / total_cells) > 0.3
    
    async def _analyze_data_structure(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Analyze the data structure of the sheet."""
        if df.empty:
            return {'structure_type': 'empty'}
        
        structure = {
            'structure_type': 'tabular',
            'has_headers': False,
            'data_types': {},
            'null_percentages': {},
            'unique_counts': {},
            'patterns': []
        }
        
        # Detect headers
        if len(df) > 1:
            first_row_types = [type(val).__name__ for val in df.iloc[0]]
            second_row_types = [type(val).__name__ for val in df.iloc[1]]
            
            # If first row is mostly strings and second row is mixed, likely headers
            if first_row_types.count('str') > len(first_row_types) * 0.7:
                structure['has_headers'] = True
        
        # Analyze data types and patterns
        for col_idx in range(len(df.columns)):
            col_data = df.iloc[:, col_idx].dropna()
            
            if len(col_data) > 0:
                # Data type analysis
                numeric_count = sum(1 for val in col_data if isinstance(val, (int, float)))
                date_count = sum(1 for val in col_data if self._is_date_like(str(val)))
                
                if numeric_count > len(col_data) * 0.8:
                    data_type = 'numeric'
                elif date_count > len(col_data) * 0.8:
                    data_type = 'date'
                else:
                    data_type = 'text'
                
                structure['data_types'][f'column_{col_idx}'] = data_type
                structure['null_percentages'][f'column_{col_idx}'] = (len(df) - len(col_data)) / len(df)
                structure['unique_counts'][f'column_{col_idx}'] = len(col_data.unique())
        
        return structure
    
    def _is_date_like(self, value: str) -> bool:
        """Check if a string value looks like a date."""
        date_patterns = [
            r'\d{1,2}/\d{1,2}/\d{4}',
            r'\d{4}-\d{1,2}-\d{1,2}',
            r'\d{1,2}-\d{1,2}-\d{4}',
            r'\w+ \d{1,2}, \d{4}'
        ]
        
        return any(re.match(pattern, value) for pattern in date_patterns)
    
    async def _analyze_formulas(self, sheet) -> Dict[str, Any]:
        """Analyze formulas in the sheet."""
        formulas = []
        dependencies = {}
        
        for row in sheet.iter_rows():
            for cell in row:
                if cell.value and isinstance(cell.value, str) and cell.value.startswith('='):
                    formula_info = {
                        'cell': cell.coordinate,
                        'formula': cell.value,
                        'references': self._extract_cell_references(cell.value)
                    }
                    formulas.append(formula_info)
                    
                    # Build dependency graph
                    dependencies[cell.coordinate] = formula_info['references']
        
        return {
            'formula_count': len(formulas),
            'formulas': formulas,
            'dependencies': dependencies,
            'complexity_score': self._calculate_formula_complexity(formulas)
        }
    
    def _extract_cell_references(self, formula: str) -> List[str]:
        """Extract cell references from formula."""
        # Pattern for cell references like A1, B2, $A$1, etc.
        pattern = r'\$?[A-Z]+\$?\d+'
        return re.findall(pattern, formula)
    
    def _calculate_formula_complexity(self, formulas: List[Dict]) -> float:
        """Calculate overall formula complexity."""
        if not formulas:
            return 0.0
        
        total_complexity = 0.0
        for formula_info in formulas:
            formula = formula_info['formula']
            complexity = len(formula) * 0.01
            complexity += len(formula_info['references']) * 0.1
            complexity += formula.count('(') * 0.05
            total_complexity += complexity
        
        return total_complexity / len(formulas)
    
    async def _analyze_formatting(self, sheet) -> Dict[str, Any]:
        """Analyze formatting and merged cells."""
        merged_ranges = list(sheet.merged_cells.ranges)
        
        formatting_info = {
            'merged_cell_count': len(merged_ranges),
            'merged_ranges': [str(range_) for range_ in merged_ranges],
            'conditional_formatting': len(sheet.conditional_formatting),
            'has_charts': False  # Would need additional analysis
        }
        
        return formatting_info
    
    async def _analyze_cross_sheet_relationships(self, workbook) -> List[Dict[str, Any]]:
        """Analyze relationships between sheets."""
        relationships = []
        
        for sheet_name in workbook.sheetnames:
            sheet = workbook[sheet_name]
            
            for row in sheet.iter_rows():
                for cell in row:
                    if cell.value and isinstance(cell.value, str) and cell.value.startswith('='):
                        # Look for references to other sheets
                        other_sheet_refs = re.findall(r"'?([^'!]+)'?!", cell.value)
                        
                        for ref_sheet in other_sheet_refs:
                            if ref_sheet != sheet_name and ref_sheet in workbook.sheetnames:
                                relationships.append({
                                    'source_sheet': sheet_name,
                                    'target_sheet': ref_sheet,
                                    'source_cell': cell.coordinate,
                                    'formula': cell.value,
                                    'relationship_type': 'formula_reference'
                                })
        
        return relationships
    
    async def _analyze_data_flow(self, workbook) -> Dict[str, Any]:
        """Analyze data flow through the workbook."""
        data_flow = {
            'input_sheets': [],
            'calculation_sheets': [],
            'output_sheets': [],
            'flow_diagram': {}
        }
        
        # Simplified analysis - categorize sheets by their characteristics
        for sheet_name in workbook.sheetnames:
            sheet = workbook[sheet_name]
            
            formula_count = 0
            data_count = 0
            
            for row in sheet.iter_rows():
                for cell in row:
                    if cell.value:
                        if isinstance(cell.value, str) and cell.value.startswith('='):
                            formula_count += 1
                        else:
                            data_count += 1
            
            total_cells = formula_count + data_count
            if total_cells > 0:
                formula_ratio = formula_count / total_cells
                
                if formula_ratio > 0.5:
                    data_flow['calculation_sheets'].append(sheet_name)
                elif formula_ratio < 0.1:
                    data_flow['input_sheets'].append(sheet_name)
                else:
                    data_flow['output_sheets'].append(sheet_name)
        
        return data_flow


class EnhancedMultiModalProcessor:
    """
    Enhanced multi-modal document processor with advanced capabilities.
    """
    
    def __init__(self, gemini_client: Optional[GeminiClient] = None, config: Optional[ToxoConfig] = None):
        self.config = config or ToxoConfig()
        self.logger = get_logger(__name__)
        self.gemini_client = gemini_client
        
        # Initialize processors
        self.vision_processor = VisionModelProcessor()
        self.formula_processor = FormulaProcessor()
        self.spreadsheet_processor = EnhancedSpreadsheetProcessor()
        
        # Processing capabilities
        self.supported_formats = {
            '.pdf': self._process_pdf_enhanced,
            '.docx': self._process_docx_enhanced,
            '.xlsx': self._process_xlsx_enhanced,
            '.xls': self._process_xlsx_enhanced,
            '.png': self._process_image_enhanced,
            '.jpg': self._process_image_enhanced,
            '.jpeg': self._process_image_enhanced,
            '.tiff': self._process_image_enhanced
        }
        
        self.logger.info("Enhanced Multi-Modal Processor initialized")
    
    def get_state(self) -> Dict[str, Any]:
        """Get the current state of the processor for serialization."""
        return {
            "supported_formats": list(self.supported_formats.keys()),
            "has_vision_processor": hasattr(self, 'vision_processor') and self.vision_processor is not None,
            "has_formula_processor": hasattr(self, 'formula_processor') and self.formula_processor is not None,
            "has_spreadsheet_processor": hasattr(self, 'spreadsheet_processor') and self.spreadsheet_processor is not None,
            "config_available": self.config is not None,
            "gemini_client_available": self.gemini_client is not None
        }
    
    async def process_document_enhanced(
        self,
        file_path: Path,
        processing_options: Optional[Dict[str, Any]] = None
    ) -> Dict[str, Any]:
        """
        Enhanced document processing with advanced multi-modal capabilities.
        
        Args:
            file_path: Path to document
            processing_options: Processing configuration
            
        Returns:
            Enhanced processing results
        """
        try:
            file_path = Path(file_path)
            if not file_path.exists():
                raise ProcessingError(f"File not found: {file_path}")
            
            self.logger.info(f"Processing document with enhanced capabilities: {file_path}")
            
            # Determine processor
            suffix = file_path.suffix.lower()
            if suffix not in self.supported_formats:
                raise ProcessingError(f"Unsupported file format: {suffix}")
            
            processor = self.supported_formats[suffix]
            
            # Process document
            processing_result = await processor(file_path, processing_options or {})
            
            # Add enhanced analysis
            if self.gemini_client:
                enhanced_insights = await self._generate_enhanced_insights(processing_result)
                processing_result['enhanced_insights'] = enhanced_insights
            
            return processing_result
            
        except Exception as e:
            self.logger.error(f"Error in enhanced processing: {str(e)}")
            raise ProcessingError(f"Enhanced processing failed: {str(e)}")
    
    async def _process_pdf_enhanced(self, file_path: Path, options: Dict[str, Any]) -> Dict[str, Any]:
        """Enhanced PDF processing with layout analysis and formula detection."""
        results = {
            'document_type': 'pdf',
            'pages': [],
            'layout_analysis': {},
            'formulas': [],
            'tables': [],
            'images': [],
            'text_content': '',
            'metadata': {}
        }
        
        # Convert PDF to images for vision processing
        if PDF2IMAGE_AVAILABLE:
            try:
                images = convert_from_path(file_path, dpi=300)
                
                # Process each page
                for page_num, image in enumerate(images, 1):
                    self.logger.info(f"Processing page {page_num}")
                    
                    # Layout analysis
                    layout = await self.vision_processor.analyze_document_layout(image)
                    
                    # Extract text content
                    page_text = ""
                    for element in layout.page_layouts.get(page_num, []):
                        if element.element_type in ['text', 'title']:
                            page_text += element.content + "\n"
                    
                    # Formula detection
                    page_formulas = await self.formula_processor.extract_and_analyze_formulas(page_text)
                    
                    page_result = {
                        'page_number': page_num,
                        'layout': layout,
                        'text_content': page_text,
                        'formulas': page_formulas,
                        'image_size': image.size
                    }
                    
                    results['pages'].append(page_result)
                    results['text_content'] += page_text + "\n"
                    results['formulas'].extend(page_formulas)
                    
            except Exception as e:
                self.logger.warning(f"Failed to convert PDF to images: {str(e)}")
                # Fall back to basic text extraction
                return await self._process_pdf_basic(file_path, results)
        else:
            self.logger.warning("pdf2image not available - using basic PDF processing")
            return await self._process_pdf_basic(file_path, results)
        
        return results
    
    async def _process_pdf_basic(self, file_path: Path, results: Dict[str, Any]) -> Dict[str, Any]:
        """Basic PDF processing fallback."""
        try:
            import fitz  # PyMuPDF
            doc = fitz.open(file_path)
            
            for page_num in range(len(doc)):
                page = doc.load_page(page_num)
                text = page.get_text()
                results['text_content'] += text + "\n"
                
                # Basic formula detection on text
                page_formulas = await self.formula_processor.extract_and_analyze_formulas(text)
                results['formulas'].extend(page_formulas)
            
            doc.close()
            
        except ImportError:
            self.logger.warning("PyMuPDF not available - cannot process PDF")
        except Exception as e:
            self.logger.error(f"Error processing PDF: {str(e)}")
        
        return results
    
    async def _process_image_enhanced(self, file_path: Path, options: Dict[str, Any]) -> Dict[str, Any]:
        """Enhanced image processing with vision models."""
        image = Image.open(file_path)
        
        # Layout analysis
        layout = await self.vision_processor.analyze_document_layout(image)
        
        # Extract text
        text_content = ""
        for elements in layout.page_layouts.values():
            for element in elements:
                if element.element_type in ['text', 'title']:
                    text_content += element.content + "\n"
        
        # Formula analysis
        formulas = await self.formula_processor.extract_and_analyze_formulas(text_content)
        
        return {
            'document_type': 'image',
            'layout_analysis': layout,
            'text_content': text_content,
            'formulas': formulas,
            'image_size': image.size,
            'metadata': {'format': image.format, 'mode': image.mode}
        }
    
    async def _process_xlsx_enhanced(self, file_path: Path, options: Dict[str, Any]) -> Dict[str, Any]:
        """Enhanced spreadsheet processing."""
        analysis = await self.spreadsheet_processor.process_spreadsheet_advanced(file_path)
        
        return {
            'document_type': 'spreadsheet',
            'spreadsheet_analysis': analysis,
            'metadata': {'file_format': 'xlsx'}
        }
    
    async def _process_docx_enhanced(self, file_path: Path, options: Dict[str, Any]) -> Dict[str, Any]:
        """Enhanced Word document processing."""
        # Basic implementation - could be enhanced with python-docx
        return {
            'document_type': 'word_document',
            'text_content': '',
            'metadata': {'file_format': 'docx'}
        }
    
    async def _generate_enhanced_insights(self, processing_result: Dict[str, Any]) -> Dict[str, Any]:
        """Generate enhanced insights using Gemini."""
        if not self.gemini_client:
            return {}
        
        # Create analysis prompt
        content_summary = processing_result.get('text_content', '')[:2000]  # Limit length
        
        prompt = f"""
        Analyze this document processing result and provide enhanced insights:
        
        Document Type: {processing_result.get('document_type', 'unknown')}
        Content Sample: {content_summary}
        
        Please provide:
        1. Document purpose and domain
        2. Key information extraction opportunities
        3. Potential automation workflows
        4. Data quality assessment
        5. Recommended next steps for processing
        
        Format as JSON with clear categories.
        """
        
        try:
            response = await self.gemini_client.generate(prompt)
            return {'gemini_analysis': response, 'analysis_timestamp': datetime.now().isoformat()}
        except Exception as e:
            self.logger.warning(f"Failed to generate enhanced insights: {str(e)}")
            return {}


# Factory function
def create_enhanced_multimodal_processor(
    gemini_client: Optional[GeminiClient] = None,
    config: Optional[ToxoConfig] = None
) -> EnhancedMultiModalProcessor:
    """Create and return an Enhanced Multi-Modal Processor instance."""
    return EnhancedMultiModalProcessor(gemini_client, config) 