"""
Gemini-Powered Document Intelligence System for Toxo

This module implements industry-level document processing using Gemini's multimodal capabilities:
- Advanced PDF processing with vision-based analysis
- Intelligent image understanding and OCR
- Comprehensive dataset analysis and insights
- Multi-modal document understanding
- Structured data extraction from any document type
- Real-time document classification and routing
"""

import asyncio
import json
import time
from typing import Dict, List, Any, Optional, Union, Tuple
from dataclasses import dataclass, field
from pathlib import Path
from datetime import datetime
import tempfile
import hashlib
import mimetypes
from io import BytesIO
import base64

# Document processing libraries
from PIL import Image, ImageDraw, ImageFont
import pandas as pd
import numpy as np

# Optional dependencies
try:
    from pdf2image import convert_from_path
    PDF2IMAGE_AVAILABLE = True
except ImportError:
    convert_from_path = None
    PDF2IMAGE_AVAILABLE = False

try:
    import fitz  # PyMuPDF
    PYMUPDF_AVAILABLE = True
except ImportError:
    fitz = None
    PYMUPDF_AVAILABLE = False

import openpyxl
from openpyxl.utils import get_column_letter

# Gemini integration
from ..integrations.gemini_client import GeminiClient
from ..core.config import ToxoConfig
from ..utils.logger import get_logger
from ..utils.exceptions import ProcessingError, ValidationError


@dataclass
class DocumentInsights:
    """Comprehensive document insights from Gemini analysis."""
    document_type: str
    confidence: float
    key_entities: List[Dict[str, Any]]
    extracted_data: Dict[str, Any]
    business_context: Dict[str, Any]
    actionable_insights: List[str]
    quality_assessment: Dict[str, Any]
    processing_recommendations: List[str]
    metadata: Dict[str, Any] = field(default_factory=dict)


@dataclass
class ProcessingResult:
    """Comprehensive processing result with all extracted information."""
    file_path: str
    document_type: str
    processing_timestamp: datetime
    raw_content: Dict[str, Any]
    structured_data: Dict[str, Any]
    gemini_insights: DocumentInsights
    performance_metrics: Dict[str, Any]
    error_log: List[str] = field(default_factory=list)


class GeminiDocumentProcessor:
    """
    Industry-level document processor powered by Gemini's multimodal capabilities.
    
    Features:
    - Vision-based document analysis
    - Intelligent table extraction
    - Multi-modal content understanding
    - Real-time insights generation
    - Structured data extraction
    - Business context analysis
    - Quality assessment and recommendations
    """
    
    def __init__(self, gemini_client: GeminiClient, config: Optional[ToxoConfig] = None):
        """
        Initialize Gemini document processor.
        
        Args:
            gemini_client: Configured Gemini client
            config: Toxo configuration
        """
        self.gemini_client = gemini_client
        self.config = config or ToxoConfig()
        self.logger = get_logger(__name__)
        
        # Processing capabilities
        self.supported_formats = {
            '.pdf': self._process_pdf_with_vision,
            '.docx': self._process_docx_with_analysis,
            '.xlsx': self._process_excel_with_insights,
            '.xls': self._process_excel_with_insights,
            '.csv': self._process_csv_with_analysis,
            '.png': self._process_image_comprehensive,
            '.jpg': self._process_image_comprehensive,
            '.jpeg': self._process_image_comprehensive,
            '.tiff': self._process_image_comprehensive,
            '.bmp': self._process_image_comprehensive,
            '.json': self._process_json_with_insights,
            '.txt': self._process_text_with_analysis
        }
        
        # Document classification patterns
        self.document_patterns = {
            'invoice': ['invoice', 'bill', 'payment', 'due', 'amount'],
            'contract': ['agreement', 'terms', 'conditions', 'signature'],
            'report': ['summary', 'analysis', 'conclusion', 'findings'],
            'form': ['form', 'application', 'field', 'checkbox'],
            'financial': ['revenue', 'profit', 'balance', 'financial'],
            'technical': ['specification', 'manual', 'documentation', 'technical'],
            'legal': ['legal', 'law', 'regulation', 'compliance'],
            'medical': ['patient', 'medical', 'diagnosis', 'treatment']
        }
        
        # Processing cache for performance
        self._processing_cache = {}
        self._performance_metrics = {
            'total_processed': 0,
            'avg_processing_time': 0,
            'success_rate': 0
        }
        
        # Temporary directory for processing
        self.temp_dir = Path(tempfile.gettempdir()) / "toxo_gemini_processing"
        self.temp_dir.mkdir(exist_ok=True)
        
        self.logger.info("Gemini Document Processor initialized with full multimodal capabilities")
    
    async def process_document(
        self,
        file_path: Union[str, Path],
        processing_options: Optional[Dict[str, Any]] = None
    ) -> ProcessingResult:
        """
        Process any document with comprehensive Gemini analysis.
        
        Args:
            file_path: Path to document
            processing_options: Processing configuration
            
        Returns:
            Comprehensive processing result
        """
        file_path = Path(file_path)
        start_time = time.time()
        
        if not file_path.exists():
            raise ProcessingError(f"File not found: {file_path}")
        
        try:
            self.logger.info(f"Processing document with Gemini intelligence: {file_path.name}")
            
            # Check cache first
            file_hash = self._compute_file_hash(file_path)
            if file_hash in self._processing_cache:
                self.logger.info("Returning cached result")
                return self._processing_cache[file_hash]
            
            # Determine document type and processor
            file_extension = file_path.suffix.lower()
            if file_extension not in self.supported_formats:
                raise ProcessingError(f"Unsupported file format: {file_extension}")
            
            processor = self.supported_formats[file_extension]
            
            # Process document
            raw_content = await processor(file_path, processing_options or {})
            
            # Generate comprehensive insights with Gemini
            gemini_insights = await self._generate_comprehensive_insights(raw_content, file_path)
            
            # Extract structured data
            structured_data = await self._extract_structured_data(raw_content, gemini_insights)
            
            # Calculate performance metrics
            processing_time = time.time() - start_time
            performance_metrics = {
                'processing_time': processing_time,
                'file_size': file_path.stat().st_size,
                'tokens_processed': len(str(raw_content)),
                'insights_generated': len(gemini_insights.actionable_insights),
                'structured_fields': len(structured_data),
                'processing_rate': file_path.stat().st_size / processing_time
            }
            
            # Create comprehensive result
            result = ProcessingResult(
                file_path=str(file_path),
                document_type=gemini_insights.document_type,
                processing_timestamp=datetime.now(),
                raw_content=raw_content,
                structured_data=structured_data,
                gemini_insights=gemini_insights,
                performance_metrics=performance_metrics,
                error_log=[]
            )
            
            # Cache result
            self._processing_cache[file_hash] = result
            
            # Update metrics
            self._update_performance_metrics(processing_time, True)
            
            self.logger.info(f"Document processed successfully in {processing_time:.2f}s")
            return result
            
        except Exception as e:
            processing_time = time.time() - start_time
            self._update_performance_metrics(processing_time, False)
            self.logger.error(f"Document processing failed: {str(e)}")
            raise ProcessingError(f"Processing failed: {str(e)}")
    
    async def batch_process_documents(
        self,
        file_paths: List[Union[str, Path]],
        processing_options: Optional[Dict[str, Any]] = None,
        max_concurrent: int = 3,  # Reduced from 5 to prevent overload
        chunk_size: int = 10,     # Process in chunks
        enable_smart_batching: bool = True
    ) -> List[ProcessingResult]:
        """
        Process multiple documents concurrently with intelligent batching.
        
        Args:
            file_paths: List of file paths to process
            processing_options: Processing configuration
            max_concurrent: Maximum concurrent processing tasks
            chunk_size: Size of processing chunks
            enable_smart_batching: Enable intelligent batching based on file types
            
        Returns:
            List of processing results
        """
        self.logger.info(f"Starting batch processing of {len(file_paths)} documents")
        
        if enable_smart_batching:
            # Group files by type for better batching
            file_groups = self._group_files_by_complexity(file_paths)
            all_results = []
            
            for group_name, group_files in file_groups.items():
                if not group_files:
                    continue
                    
                self.logger.info(f"Processing {len(group_files)} {group_name} files")
                
                # Adjust concurrency based on file complexity
                if group_name == "high_complexity":
                    group_max_concurrent = min(2, max_concurrent)
                    group_chunk_size = min(5, chunk_size)
                elif group_name == "medium_complexity":
                    group_max_concurrent = min(3, max_concurrent)
                    group_chunk_size = min(8, chunk_size)
                else:  # low_complexity
                    group_max_concurrent = max_concurrent
                    group_chunk_size = chunk_size
                
                group_results = await self._process_file_group(
                    group_files, 
                    processing_options, 
                    group_max_concurrent, 
                    group_chunk_size
                )
                all_results.extend(group_results)
            
            successful_results = [r for r in all_results if r is not None]
        else:
            # Original processing with chunking
            successful_results = await self._process_file_group(
                file_paths, processing_options, max_concurrent, chunk_size
            )
        
        success_count = len(successful_results)
        total_count = len(file_paths)
        
        self.logger.info(f"Batch processing completed: {success_count}/{total_count} successful")
        return successful_results

    def _group_files_by_complexity(self, file_paths: List[Union[str, Path]]) -> Dict[str, List[Path]]:
        """Group files by processing complexity to optimize batching."""
        groups = {
            "low_complexity": [],      # txt, json, small images
            "medium_complexity": [],   # csv, small pdf, medium images  
            "high_complexity": []      # xlsx, large pdf, complex documents
        }
        
        for file_path in file_paths:
            path = Path(file_path)
            file_extension = path.suffix.lower()
            file_size = path.stat().st_size if path.exists() else 0
            
            if file_extension in ['.txt', '.json'] or (file_extension in ['.png', '.jpg', '.jpeg'] and file_size < 1024*1024):
                groups["low_complexity"].append(path)
            elif file_extension in ['.csv'] or (file_extension == '.pdf' and file_size < 5*1024*1024):
                groups["medium_complexity"].append(path)
            else:  # xlsx, large files, etc.
                groups["high_complexity"].append(path)
                
        return groups

    async def _process_file_group(
        self,
        file_paths: List[Union[str, Path]],
        processing_options: Optional[Dict[str, Any]],
        max_concurrent: int,
        chunk_size: int
    ) -> List[ProcessingResult]:
        """Process a group of files with chunking and concurrency control."""
        all_results = []
        
        # Process files in chunks to prevent memory overload
        for i in range(0, len(file_paths), chunk_size):
            chunk = file_paths[i:i + chunk_size]
            self.logger.info(f"Processing chunk {i//chunk_size + 1}/{(len(file_paths) + chunk_size - 1)//chunk_size} ({len(chunk)} files)")
            
            # Create semaphore for concurrency control
            semaphore = asyncio.Semaphore(max_concurrent)
            
            async def process_single_with_semaphore(file_path):
                async with semaphore:
                    try:
                        return await self.process_document(file_path, processing_options)
                    except Exception as e:
                        self.logger.error(f"Failed to process {file_path}: {str(e)}")
                        return None
            
            # Process chunk concurrently
            tasks = [process_single_with_semaphore(fp) for fp in chunk]
            chunk_results = await asyncio.gather(*tasks, return_exceptions=True)
            
            # Filter out None results and exceptions
            successful_chunk_results = [
                r for r in chunk_results 
                if r is not None and not isinstance(r, Exception)
            ]
            
            all_results.extend(successful_chunk_results)
            
            # Small delay between chunks to prevent API overload
            if i + chunk_size < len(file_paths):
                await asyncio.sleep(0.5)
        
        return all_results
    
    async def analyze_document_collection(
        self,
        file_paths: List[Union[str, Path]],
        analysis_type: str = "comprehensive"
    ) -> Dict[str, Any]:
        """
        Analyze a collection of documents to find patterns and insights.
        
        Args:
            file_paths: List of documents to analyze
            analysis_type: Type of collection analysis
            
        Returns:
            Collection analysis results
        """
        self.logger.info(f"Analyzing document collection ({len(file_paths)} documents)")
        
        # Process all documents
        results = await self.batch_process_documents(file_paths)
        
        # Aggregate insights
        collection_insights = await self._analyze_document_collection(results, analysis_type)
        
        return collection_insights
    
    # Format-specific processors
    async def _process_pdf_with_vision(self, file_path: Path, options: Dict[str, Any]) -> Dict[str, Any]:
        """Process PDF using Gemini Vision for comprehensive analysis."""
        if not PDF2IMAGE_AVAILABLE:
            self.logger.warning("pdf2image not available - falling back to text extraction")
            if str(sys.platform).startswith("darwin"):
                self.logger.warning("To enable full PDF page rendering on macOS: `brew install poppler` (then re-run).")
            self.logger.warning("To install Python deps for PDFs/images: `python -m toxo.cli install vision --run`")
            return await self._process_pdf_fallback(file_path)
        
        try:
            # Convert PDF to images for vision analysis
            images = convert_from_path(file_path, dpi=300, first_page=1, last_page=None)
            
            # Analyze all pages with Gemini Vision
            pages_analysis = []
            for i, image in enumerate(images):
                self.logger.info(f"Analyzing PDF page {i+1}/{len(images)}")
                
                # Convert PIL Image to bytes for Gemini
                img_bytes = BytesIO()
                image.save(img_bytes, format='PNG')
                img_bytes = img_bytes.getvalue()
                
                # Analyze page with Gemini Vision
                page_analysis = await self.gemini_client.analyze_document(
                    img_bytes,
                    document_type="pdf_page",
                    extract_tables=True,
                    extract_charts=True,
                    extract_text=True
                )
                
                page_analysis['page_number'] = i + 1
                pages_analysis.append(page_analysis)
            
            # Combine all pages analysis
            combined_analysis = {
                'document_type': 'pdf',
                'total_pages': len(images),
                'pages': pages_analysis,
                'combined_content': self._combine_pdf_content(pages_analysis),
                'metadata': {
                    'file_size': file_path.stat().st_size,
                    'processing_method': 'gemini_vision',
                    'dpi': 300
                }
            }
            
            return combined_analysis
            
        except Exception as e:
            self.logger.error(f"PDF processing failed: {str(e)}")
            if "poppler" in str(e).lower() or "page count" in str(e).lower():
                if str(sys.platform).startswith("darwin"):
                    self.logger.warning("It looks like Poppler is missing. On macOS run: `brew install poppler`.")
                self.logger.warning("If Python deps are missing, run: `python -m toxo.cli install vision --run`.")
            # Fallback to text-based processing
            return await self._process_pdf_fallback(file_path)
    
    async def _process_image_comprehensive(self, file_path: Path, options: Dict[str, Any]) -> Dict[str, Any]:
        """Process image with comprehensive Gemini Vision analysis."""
        try:
            # Load image
            image = Image.open(file_path)
            
            # Perform comprehensive analysis
            comprehensive_analysis = await self.gemini_client.analyze_image(
                image,
                prompt="Provide comprehensive analysis of this image including text, tables, charts, and any structured data.",
                analysis_type="comprehensive"
            )
            
            # Extract structured data if present
            structured_extraction = await self.gemini_client.extract_structured_data(
                image,
                data_type="auto",
                output_format="json"
            )
            
            return {
                'document_type': 'image',
                'image_analysis': comprehensive_analysis,
                'structured_extraction': structured_extraction,
                'image_metadata': {
                    'size': image.size,
                    'format': image.format,
                    'mode': image.mode,
                    'file_size': file_path.stat().st_size
                }
            }
            
        except Exception as e:
            self.logger.error(f"Image processing failed: {str(e)}")
            raise ProcessingError(f"Image processing failed: {str(e)}")
    
    async def _process_excel_with_insights(self, file_path: Path, options: Dict[str, Any]) -> Dict[str, Any]:
        """Process Excel file with Gemini-powered insights and improved error handling."""
        try:
            # Read Excel file with error handling
            try:
                xl_file = pd.ExcelFile(file_path)
            except Exception as e:
                self.logger.error(f"Failed to read Excel file {file_path}: {e}")
                raise ProcessingError(f"Excel file read failed: {str(e)}")
            
            sheets_data = {}
            
            for sheet_name in xl_file.sheet_names:
                try:
                    # Read sheet with safe options
                    df = pd.read_excel(file_path, sheet_name=sheet_name, nrows=1000)  # Limit rows to prevent timeout
                    
                    # Handle empty sheets
                    if df.empty:
                        self.logger.warning(f"Sheet '{sheet_name}' is empty, skipping")
                        continue
                    
                    # Clean and prepare data for JSON serialization
                    df_cleaned = df.copy()
                    
                    # Handle mixed-type columns by converting to string
                    for col in df_cleaned.columns:
                        if df_cleaned[col].dtype == 'object':
                            df_cleaned[col] = df_cleaned[col].astype(str)
                    
                    # Convert to structured format with comprehensive JSON-serializable data
                    sheet_info = {
                        'name': sheet_name,
                        'shape': list(df.shape),  # Convert tuple to list
                        'columns': list(df.columns),
                        'data_types': {col: str(dtype) for col, dtype in df_cleaned.dtypes.items()},
                        'sample_data': self._make_json_serializable(df_cleaned.head(5).to_dict('records')),
                        'null_counts': self._make_json_serializable({col: int(df[col].isnull().sum()) for col in df.columns}),
                        'summary_stats': {}
                    }
                    
                    # Add summary statistics for numeric columns only
                    numeric_df = df.select_dtypes(include=[np.number])
                    if not numeric_df.empty:
                        summary_stats = numeric_df.describe().to_dict()
                        sheet_info['summary_stats'] = self._make_json_serializable(summary_stats)
                
                    sheets_data[sheet_name] = sheet_info
                
                except Exception as sheet_error:
                    self.logger.warning(f"Failed to process sheet '{sheet_name}': {sheet_error}")
                    sheets_data[sheet_name] = {
                        'name': sheet_name,
                        'error': str(sheet_error),
                        'status': 'failed'
                    }
            
            if not sheets_data:
                raise ProcessingError("No sheets could be processed from Excel file")
            
            # Prepare dataset info for Gemini analysis (limit size)
            dataset_summary = {
                    'file_type': 'excel',
                    'sheets': list(sheets_data.keys()),
                    'total_sheets': len(sheets_data),
                'sample_sheet_details': {k: v for k, v in list(sheets_data.items())[:3]}  # Limit to 3 sheets
            }
            
            # Analyze with Gemini using the improved client
            try:
                dataset_analysis = await self.gemini_client.analyze_dataset(
                    dataset_info=dataset_summary,
                    sample_data=json.dumps(dataset_summary, default=str, indent=2)[:3000],  # Limit size
                    analysis_goals=['business_insights', 'data_quality', 'relationships'],
                    timeout=180  # 3 minute timeout
                )
            except Exception as analysis_error:
                self.logger.warning(f"Gemini analysis failed for Excel file: {analysis_error}")
                dataset_analysis = {
                    'analysis_status': 'failed',
                    'error': str(analysis_error),
                    'fallback_insights': [
                        f"Excel file contains {len(sheets_data)} sheets",
                        f"Total columns across all sheets: {sum(len(sheet.get('columns', [])) for sheet in sheets_data.values() if isinstance(sheet, dict) and 'columns' in sheet)}"
                    ]
                }
            
            return {
                'document_type': 'excel',
                'sheets_data': sheets_data,
                'dataset_analysis': dataset_analysis,
                'metadata': {
                    'file_size': file_path.stat().st_size,
                    'total_sheets': len(sheets_data),
                    'processing_method': 'pandas_gemini_improved',
                    'processing_timestamp': time.time()
                }
            }
            
        except Exception as e:
            self.logger.error(f"Excel processing failed: {str(e)}")
            raise ProcessingError(f"Excel processing failed: {str(e)}")
    
    async def _process_csv_with_analysis(self, file_path: Path, options: Dict[str, Any]) -> Dict[str, Any]:
        """Process CSV file with Gemini-powered analysis and improved error handling."""
        try:
            # Read CSV with safe options and row limit
            try:
                df = pd.read_csv(file_path, nrows=1000, encoding='utf-8')  # Limit rows
            except UnicodeDecodeError:
                try:
                    df = pd.read_csv(file_path, nrows=1000, encoding='latin1')
                except Exception:
                    df = pd.read_csv(file_path, nrows=1000, encoding='utf-8', errors='ignore')
            
            # Handle empty CSV
            if df.empty:
                raise ProcessingError("CSV file is empty")
            
            # Clean data for JSON serialization
            df_cleaned = df.copy()
            
            # Handle mixed-type columns
            for col in df_cleaned.columns:
                if df_cleaned[col].dtype == 'object':
                    df_cleaned[col] = df_cleaned[col].astype(str)
            
            # Prepare dataset information with comprehensive JSON-serializable data
            dataset_info = {
                'file_type': 'csv',
                'shape': list(df.shape),  # Convert tuple to list
                'columns': list(df.columns),
                'data_types': {col: str(dtype) for col, dtype in df_cleaned.dtypes.items()},
                'null_counts': self._make_json_serializable({col: int(df[col].isnull().sum()) for col in df.columns}),
                'sample_data': self._make_json_serializable(df_cleaned.head(10).to_dict('records')),
                'summary_stats': {}
            }
            
            # Add summary statistics for numeric columns
            numeric_df = df.select_dtypes(include=[np.number])
            if not numeric_df.empty:
                summary_stats = numeric_df.describe().to_dict()
                dataset_info['summary_stats'] = self._make_json_serializable(summary_stats)
            
            # Analyze with Gemini using improved error handling
            try:
                dataset_analysis = await self.gemini_client.analyze_dataset(
                    dataset_info=dataset_info,
                    sample_data=df_cleaned.head(20).to_csv(index=False)[:2000],  # Limit size
                    analysis_goals=['patterns', 'quality', 'insights', 'recommendations'],
                    timeout=120  # 2 minute timeout
                )
            except Exception as analysis_error:
                self.logger.warning(f"Gemini analysis failed for CSV file: {analysis_error}")
                dataset_analysis = {
                    'analysis_status': 'failed',
                    'error': str(analysis_error),
                    'fallback_insights': [
                        f"CSV contains {df.shape[0]} rows and {df.shape[1]} columns",
                        f"Columns: {', '.join(df.columns[:5])}{'...' if len(df.columns) > 5 else ''}",
                        f"Data types: {len(df.select_dtypes(include=[np.number]).columns)} numeric, {len(df.select_dtypes(include=['object']).columns)} text"
                    ]
                }
            
            return {
                'document_type': 'csv',
                'dataset_info': dataset_info,
                'dataset_analysis': dataset_analysis,
                'metadata': {
                    'file_size': file_path.stat().st_size,
                    'rows': len(df),
                    'columns': len(df.columns),
                    'processing_method': 'pandas_gemini_improved',
                    'processing_timestamp': time.time()
                }
            }
            
        except Exception as e:
            self.logger.error(f"CSV processing failed: {str(e)}")
            raise ProcessingError(f"CSV processing failed: {str(e)}")
    
    async def _process_json_with_insights(self, file_path: Path, options: Dict[str, Any]) -> Dict[str, Any]:
        """Process JSON file with Gemini-powered insights."""
        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                json_data = json.load(f)
            
            # Analyze JSON structure
            structure_analysis = self._analyze_json_structure(json_data)
            
            # Get Gemini insights
            gemini_analysis = await self.gemini_client.analyze_content(
                json.dumps(json_data, indent=2)[:3000],  # Limit size
                analysis_type="classification"
            )
            
            return {
                'document_type': 'json',
                'json_data': json_data,
                'structure_analysis': structure_analysis,
                'gemini_analysis': gemini_analysis,
                'metadata': {
                    'file_size': file_path.stat().st_size,
                    'keys_count': len(json_data) if isinstance(json_data, dict) else 0,
                    'data_type': type(json_data).__name__
                }
            }
            
        except Exception as e:
            self.logger.error(f"JSON processing failed: {str(e)}")
            raise ProcessingError(f"JSON processing failed: {str(e)}")
    
    async def _process_text_with_analysis(self, file_path: Path, options: Dict[str, Any]) -> Dict[str, Any]:
        """Process text file with Gemini analysis."""
        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                text_content = f.read()
            
            # Analyze with Gemini
            content_analysis = await self.gemini_client.analyze_content(
                text_content,
                analysis_type="comprehensive"
            )
            
            # Extract keywords and summary
            keywords_analysis = await self.gemini_client.analyze_content(
                text_content,
                analysis_type="keywords"
            )
            
            summary_analysis = await self.gemini_client.analyze_content(
                text_content,
                analysis_type="summary"
            )
            
            return {
                'document_type': 'text',
                'text_content': text_content,
                'content_analysis': content_analysis,
                'keywords_analysis': keywords_analysis,
                'summary_analysis': summary_analysis,
                'metadata': {
                    'file_size': file_path.stat().st_size,
                    'character_count': len(text_content),
                    'word_count': len(text_content.split()),
                    'line_count': len(text_content.split('\n'))
                }
            }
            
        except Exception as e:
            self.logger.error(f"Text processing failed: {str(e)}")
            raise ProcessingError(f"Text processing failed: {str(e)}")
    
    async def _process_docx_with_analysis(self, file_path: Path, options: Dict[str, Any]) -> Dict[str, Any]:
        """Process DOCX file with Gemini analysis."""
        try:
            # For now, convert to text and analyze
            # In production, you'd want to use python-docx for better structure extraction
            text_content = f"DOCX file: {file_path.name} - Content extraction would require python-docx library"
            
            # Analyze with Gemini
            content_analysis = await self.gemini_client.analyze_content(
                text_content,
                analysis_type="comprehensive"
            )
            
            return {
                'document_type': 'docx',
                'text_content': text_content,
                'content_analysis': content_analysis,
                'metadata': {
                    'file_size': file_path.stat().st_size,
                    'note': 'Full DOCX processing requires python-docx library'
                }
            }
            
        except Exception as e:
            self.logger.error(f"DOCX processing failed: {str(e)}")
            raise ProcessingError(f"DOCX processing failed: {str(e)}")
    
    # Helper methods
    async def _generate_comprehensive_insights(
        self,
        raw_content: Dict[str, Any],
        file_path: Path
    ) -> DocumentInsights:
        """Generate comprehensive insights using Gemini."""
        try:
            # Prepare content for analysis
            content_summary = self._prepare_content_for_analysis(raw_content)
            
            # Create comprehensive analysis prompt
            analysis_prompt = f"""
            Analyze this document and provide comprehensive business insights:
            
            File: {file_path.name}
            Content Type: {raw_content.get('document_type', 'unknown')}
            
            Content Summary:
            {content_summary}
            
            Please provide a comprehensive analysis in the following JSON format:
            {{
                "document_type": "classified document type",
                "confidence": 0.0-1.0,
                "key_entities": [
                    {{"type": "entity_type", "value": "entity_value", "confidence": 0.0-1.0}}
                ],
                "extracted_data": {{
                    "key_information": "important data points"
                }},
                "business_context": {{
                    "industry": "relevant industry",
                    "use_case": "primary use case",
                    "business_value": "value proposition"
                }},
                "actionable_insights": [
                    "specific actionable insight 1",
                    "specific actionable insight 2"
                ],
                "quality_assessment": {{
                    "completeness": 0.0-1.0,
                    "accuracy": 0.0-1.0,
                    "readability": 0.0-1.0,
                    "issues": ["any quality issues"]
                }},
                "processing_recommendations": [
                    "recommendation 1",
                    "recommendation 2"
                ]
            }}
            
            Provide detailed, specific, and actionable insights.
            """
            
            # Get analysis from Gemini
            analysis_response = await self.gemini_client.generate(
                analysis_prompt,
                temperature=0.3,
                max_tokens=2000
            )
            
            # Parse response
            insights_data = self._parse_insights_response(analysis_response)
            
            # Create DocumentInsights object
            insights = DocumentInsights(
                document_type=insights_data.get('document_type', 'unknown'),
                confidence=insights_data.get('confidence', 0.0),
                key_entities=insights_data.get('key_entities', []),
                extracted_data=insights_data.get('extracted_data', {}),
                business_context=insights_data.get('business_context', {}),
                actionable_insights=insights_data.get('actionable_insights', []),
                quality_assessment=insights_data.get('quality_assessment', {}),
                processing_recommendations=insights_data.get('processing_recommendations', []),
                metadata={
                    'analysis_timestamp': datetime.now().isoformat(),
                    'model_used': 'gemini',
                    'content_length': len(content_summary)
                }
            )
            
            return insights
            
        except Exception as e:
            self.logger.error(f"Insights generation failed: {str(e)}")
            # Return basic insights
            return DocumentInsights(
                document_type=raw_content.get('document_type', 'unknown'),
                confidence=0.5,
                key_entities=[],
                extracted_data={},
                business_context={},
                actionable_insights=[],
                quality_assessment={},
                processing_recommendations=[],
                metadata={'error': str(e)}
            )
    
    async def _extract_structured_data(
        self,
        raw_content: Dict[str, Any],
        insights: DocumentInsights
    ) -> Dict[str, Any]:
        """Extract structured data based on document type and insights."""
        structured_data = {}
        
        try:
            # Extract based on document type
            if insights.document_type == 'invoice':
                structured_data = await self._extract_invoice_data(raw_content)
            elif insights.document_type == 'contract':
                structured_data = await self._extract_contract_data(raw_content)
            elif insights.document_type == 'report':
                structured_data = await self._extract_report_data(raw_content)
            elif insights.document_type in ['excel', 'csv']:
                structured_data = await self._extract_tabular_data(raw_content)
            else:
                structured_data = await self._extract_generic_data(raw_content)
            
            # Add metadata
            structured_data['extraction_metadata'] = {
                'timestamp': datetime.now().isoformat(),
                'method': 'gemini_extraction',
                'confidence': insights.confidence
            }
            
            return structured_data
            
        except Exception as e:
            self.logger.error(f"Structured data extraction failed: {str(e)}")
            return {'error': str(e)}
    
    # Utility methods
    def _compute_file_hash(self, file_path: Path) -> str:
        """Compute hash of file for caching."""
        hasher = hashlib.md5()
        with open(file_path, 'rb') as f:
            for chunk in iter(lambda: f.read(4096), b""):
                hasher.update(chunk)
        return hasher.hexdigest()
    
    def _prepare_content_for_analysis(self, raw_content: Dict[str, Any]) -> str:
        """Prepare content summary for Gemini analysis."""
        # Extract key text content
        content_parts = []
        
        if 'text_content' in raw_content:
            content_parts.append(f"Text: {raw_content['text_content'][:1000]}")
        
        if 'dataset_info' in raw_content:
            content_parts.append(f"Dataset: {raw_content['dataset_info']}")
        
        if 'image_analysis' in raw_content:
            content_parts.append(f"Image Analysis: {raw_content['image_analysis']}")
        
        if 'combined_content' in raw_content:
            content_parts.append(f"Combined Content: {raw_content['combined_content']}")
        
        return "\n\n".join(content_parts)[:2000]  # Limit length
    
    def _parse_insights_response(self, response: str) -> Dict[str, Any]:
        """Parse insights response from Gemini."""
        try:
            # Try to extract JSON from response
            if '{' in response and '}' in response:
                start = response.find('{')
                end = response.rfind('}') + 1
                json_str = response[start:end]
                return json.loads(json_str)
            else:
                # Fallback parsing
                return {
                    'document_type': 'unknown',
                    'confidence': 0.5,
                    'key_entities': [],
                    'extracted_data': {'raw_response': response},
                    'business_context': {},
                    'actionable_insights': [response[:200] + "..."],
                    'quality_assessment': {},
                    'processing_recommendations': []
                }
        except Exception as e:
            self.logger.debug(f"Failed to parse insights response: {e}")
            return {
                'document_type': 'unknown',
                'confidence': 0.0,
                'key_entities': [],
                'extracted_data': {},
                'business_context': {},
                'actionable_insights': [],
                'quality_assessment': {},
                'processing_recommendations': []
            }
    
    def _combine_pdf_content(self, pages_analysis: List[Dict[str, Any]]) -> Dict[str, Any]:
        """Combine content from multiple PDF pages."""
        combined = {
            'all_text': '',
            'all_tables': [],
            'all_charts': [],
            'page_summaries': []
        }
        
        for page in pages_analysis:
            page_num = page.get('page_number', 0)
            
            # Add page text
            if page.get('raw_response'):
                combined['all_text'] += f"\n--- Page {page_num} ---\n"
                combined['all_text'] += page['raw_response']
            
            # Add page summary
            combined['page_summaries'].append({
                'page': page_num,
                'summary': page.get('raw_response', '')[:200] + "..."
            })
        
        return combined
    
    def _analyze_json_structure(self, json_data: Any) -> Dict[str, Any]:
        """Analyze JSON structure."""
        if isinstance(json_data, dict):
            return {
                'type': 'object',
                'keys': list(json_data.keys()),
                'depth': self._get_json_depth(json_data),
                'size': len(json_data)
            }
        elif isinstance(json_data, list):
            return {
                'type': 'array',
                'length': len(json_data),
                'item_types': list(set(type(item).__name__ for item in json_data[:10]))
            }
        else:
            return {
                'type': type(json_data).__name__,
                'value': str(json_data)[:100]
            }
    
    def _get_json_depth(self, obj: Any, depth: int = 0) -> int:
        """Get maximum depth of JSON object."""
        if isinstance(obj, dict):
            return max([self._get_json_depth(v, depth + 1) for v in obj.values()], default=depth)
        elif isinstance(obj, list):
            return max([self._get_json_depth(item, depth + 1) for item in obj], default=depth)
        else:
            return depth
    
    def _update_performance_metrics(self, processing_time: float, success: bool):
        """Update performance metrics."""
        self._performance_metrics['total_processed'] += 1
        
        # Update average processing time
        total = self._performance_metrics['total_processed']
        current_avg = self._performance_metrics['avg_processing_time']
        self._performance_metrics['avg_processing_time'] = (current_avg * (total - 1) + processing_time) / total
        
        # Update success rate
        if success:
            current_success_rate = self._performance_metrics['success_rate']
            self._performance_metrics['success_rate'] = (current_success_rate * (total - 1) + 1) / total
        else:
            current_success_rate = self._performance_metrics['success_rate']
            self._performance_metrics['success_rate'] = (current_success_rate * (total - 1)) / total
    
    # Specialized extraction methods
    async def _extract_invoice_data(self, raw_content: Dict[str, Any]) -> Dict[str, Any]:
        """Extract structured data from invoice."""
        # Implementation would depend on invoice format
        return {'type': 'invoice', 'status': 'extraction_needed'}
    
    async def _extract_contract_data(self, raw_content: Dict[str, Any]) -> Dict[str, Any]:
        """Extract structured data from contract."""
        return {'type': 'contract', 'status': 'extraction_needed'}
    
    async def _extract_report_data(self, raw_content: Dict[str, Any]) -> Dict[str, Any]:
        """Extract structured data from report."""
        return {'type': 'report', 'status': 'extraction_needed'}
    
    async def _extract_tabular_data(self, raw_content: Dict[str, Any]) -> Dict[str, Any]:
        """Extract structured data from tabular formats."""
        if 'dataset_info' in raw_content:
            return raw_content['dataset_info']
        return {'type': 'tabular', 'status': 'extraction_needed'}
    
    async def _extract_generic_data(self, raw_content: Dict[str, Any]) -> Dict[str, Any]:
        """Extract generic structured data."""
        return {'type': 'generic', 'content': raw_content}
    
    async def _process_pdf_fallback(self, file_path: Path) -> Dict[str, Any]:
        """Fallback PDF processing without vision."""
        if not PYMUPDF_AVAILABLE:
            self.logger.warning("PyMuPDF not available - PDF processing not supported")
            return {
                'document_type': 'pdf',
                'text_content': f"PDF file: {file_path.name} - Content extraction requires PyMuPDF library",
                'metadata': {
                    'processing_method': 'not_available',
                    'note': 'PDF processing requires PyMuPDF library'
                }
            }
        
        try:
            doc = fitz.open(file_path)
            text_content = ""
            
            for page in doc:
                text_content += page.get_text()
            
            doc.close()
            
            return {
                'document_type': 'pdf',
                'text_content': text_content,
                'metadata': {
                    'processing_method': 'fallback_text_extraction',
                    'note': 'Vision processing failed, using text extraction'
                }
            }
        except Exception as e:
            raise ProcessingError(f"Fallback PDF processing failed: {str(e)}")
    
    async def _analyze_document_collection(
        self,
        results: List[ProcessingResult],
        analysis_type: str
    ) -> Dict[str, Any]:
        """Analyze a collection of processed documents."""
        # Aggregate insights from all documents
        document_types = {}
        all_insights = []
        
        for result in results:
            doc_type = result.document_type
            document_types[doc_type] = document_types.get(doc_type, 0) + 1
            all_insights.extend(result.gemini_insights.actionable_insights)
        
        # Create collection summary
        collection_summary = {
            'total_documents': len(results),
            'document_types': document_types,
            'common_insights': list(set(all_insights)),
            'processing_performance': {
                'avg_processing_time': sum(r.performance_metrics['processing_time'] for r in results) / len(results),
                'total_size': sum(r.performance_metrics['file_size'] for r in results),
                'success_rate': 1.0  # All results are successful if we got here
            }
        }
        
        # Get collection insights from Gemini
        collection_prompt = f"""
        Analyze this collection of {len(results)} documents and provide insights:
        
        Document Types: {document_types}
        Sample Insights: {all_insights[:10]}
        
        Provide:
        1. Collection overview and patterns
        2. Cross-document relationships
        3. Business insights from the collection
        4. Recommendations for the document set
        
        Format as structured analysis.
        """
        
        try:
            collection_analysis = await self.gemini_client.generate(
                collection_prompt,
                temperature=0.3,
                max_tokens=1500
            )
            
            collection_summary['gemini_collection_analysis'] = collection_analysis
        except Exception as e:
            collection_summary['analysis_error'] = str(e)
        
        return collection_summary
    
    def get_performance_metrics(self) -> Dict[str, Any]:
        """Get current performance metrics."""
        return {
            **self._performance_metrics,
            'cache_size': len(self._processing_cache),
            'supported_formats': list(self.supported_formats.keys())
        }
    
    def clear_cache(self):
        """Clear processing cache."""
        self._processing_cache.clear()
        self.logger.info("Processing cache cleared")

    def _make_json_serializable(self, data: Any) -> Any:
        """Convert data to a JSON-serializable format."""
        import pandas as pd
        import numpy as np
        
        if isinstance(data, dict):
            return {k: self._make_json_serializable(v) for k, v in data.items()}
        elif isinstance(data, list):
            return [self._make_json_serializable(item) for item in data]
        elif isinstance(data, tuple):
            return [self._make_json_serializable(item) for item in data]
        elif isinstance(data, set):
            return [self._make_json_serializable(item) for item in data]
        elif isinstance(data, (np.integer, np.int64, np.int32, np.int16, np.int8)):
            return int(data)
        elif isinstance(data, (np.floating, np.float64, np.float32, np.float16)):
            return float(data)
        elif isinstance(data, np.bool_):
            return bool(data)
        elif isinstance(data, np.ndarray):
            return data.tolist()
        elif pd.isna(data):
            return None
        elif hasattr(data, 'isoformat'):  # Handle datetime objects
            return data.isoformat()
        elif hasattr(data, 'item'):  # Handle numpy scalars
            try:
                return data.item()
            except (ValueError, AttributeError):
                return str(data)
        elif isinstance(data, pd.Timestamp):
            return data.isoformat()
        elif isinstance(data, pd.Timedelta):
            return str(data)
        elif hasattr(data, 'dtype'):  # Handle pandas objects with dtype
            try:
                if data.dtype == 'object':
                    return str(data)
                else:
                    return data.item() if hasattr(data, 'item') else str(data)
            except (ValueError, AttributeError):
                return str(data)
        elif isinstance(data, str):
            return data
        elif isinstance(data, (int, float, bool)) or data is None:
            return data
        else:
            # Fallback: convert to string for unsupported types
            try:
                return str(data)
            except Exception:
                return f"<{type(data).__name__} object>"


# Factory function
def create_gemini_document_processor(
    gemini_client: GeminiClient,
    config: Optional[ToxoConfig] = None
) -> GeminiDocumentProcessor:
    """Create and return a Gemini Document Processor instance."""
    return GeminiDocumentProcessor(gemini_client, config) 