"""
Data Analyzer Module

Intelligent data analysis including feature type detection, 
missing value analysis, and data quality assessment.
"""

import pandas as pd
import numpy as np
from typing import Dict, List, Tuple, Any, Optional
from scipy import stats
import warnings


class DataAnalyzer:
    """
    Intelligent data analyzer that automatically detects feature types,
    analyzes data quality, and provides insights for preprocessing.
    """
    
    def __init__(self):
        self.analysis_results = {}
        self.feature_types = {}
        
    def analyze_dataset(self, df: pd.DataFrame, target_column: Optional[str] = None) -> Dict[str, Any]:
        """
        Comprehensive analysis of the dataset
        
        Args:
            df: Input DataFrame
            target_column: Name of target column (if applicable)
            
        Returns:
            Dictionary containing analysis results
        """
        analysis = {
            'basic_info': self._get_basic_info(df),
            'feature_types': self._detect_feature_types(df),
            'missing_values': self._analyze_missing_values(df),
            'outliers': self._detect_outliers(df),
            'correlations': self._analyze_correlations(df),
            'data_quality': self._assess_data_quality(df)
        }
        
        if target_column and target_column in df.columns:
            analysis['target_analysis'] = self._analyze_target(df[target_column])
            
        self.analysis_results = analysis
        return analysis
    
    def _get_basic_info(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Get basic dataset information"""
        return {
            'shape': df.shape,
            'columns': list(df.columns),
            'dtypes': df.dtypes.to_dict(),
            'memory_usage': df.memory_usage(deep=True).sum(),
            'duplicate_rows': df.duplicated().sum()
        }
    
    def _detect_feature_types(self, df: pd.DataFrame) -> Dict[str, str]:
        """
        Automatically detect feature types
        
        Returns:
            Dictionary mapping column names to feature types
        """
        feature_types = {}
        
        for col in df.columns:
            dtype = df[col].dtype
            
            if pd.api.types.is_numeric_dtype(dtype):
                if df[col].nunique() < 10 and df[col].min() >= 0:
                    feature_types[col] = 'ordinal'
                else:
                    feature_types[col] = 'numerical'
            elif pd.api.types.is_datetime64_any_dtype(dtype):
                feature_types[col] = 'datetime'
            elif df[col].dtype == 'object':
                # Check if it's categorical or text
                unique_ratio = df[col].nunique() / len(df)
                if unique_ratio < 0.1 or df[col].nunique() < 20:
                    feature_types[col] = 'categorical'
                else:
                    feature_types[col] = 'text'
            else:
                feature_types[col] = 'unknown'
        
        self.feature_types = feature_types
        return feature_types
    
    def _analyze_missing_values(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Analyze missing values patterns"""
        missing_counts = df.isnull().sum()
        missing_percentages = (missing_counts / len(df)) * 100
        
        return {
            'missing_counts': missing_counts.to_dict(),
            'missing_percentages': missing_percentages.to_dict(),
            'total_missing': missing_counts.sum(),
            'columns_with_missing': missing_counts[missing_counts > 0].index.tolist(),
            'missing_patterns': self._detect_missing_patterns(df)
        }
    
    def _detect_missing_patterns(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Detect patterns in missing data"""
        missing_matrix = df.isnull()
        
        # Find columns that are often missing together
        missing_correlations = missing_matrix.corr()
        
        # Identify rows with missing values
        rows_with_missing = missing_matrix.any(axis=1)
        
        return {
            'missing_correlations': missing_correlations.to_dict(),
            'rows_with_missing_count': rows_with_missing.sum(),
            'max_missing_in_row': missing_matrix.sum(axis=1).max()
        }
    
    def _detect_outliers(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Detect outliers in numerical columns"""
        outliers = {}
        
        for col in df.select_dtypes(include=[np.number]).columns:
            if self.feature_types.get(col) == 'numerical':
                # IQR method
                Q1 = df[col].quantile(0.25)
                Q3 = df[col].quantile(0.75)
                IQR = Q3 - Q1
                lower_bound = Q1 - 1.5 * IQR
                upper_bound = Q3 + 1.5 * IQR
                
                outlier_mask = (df[col] < lower_bound) | (df[col] > upper_bound)
                outliers[col] = {
                    'count': outlier_mask.sum(),
                    'percentage': (outlier_mask.sum() / len(df)) * 100,
                    'bounds': {'lower': lower_bound, 'upper': upper_bound}
                }
        
        return outliers
    
    def _analyze_correlations(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Analyze correlations between features"""
        numerical_cols = df.select_dtypes(include=[np.number]).columns
        
        if len(numerical_cols) > 1:
            corr_matrix = df[numerical_cols].corr()
            
            # Find highly correlated pairs
            high_corr_pairs = []
            for i in range(len(corr_matrix.columns)):
                for j in range(i+1, len(corr_matrix.columns)):
                    corr_val = corr_matrix.iloc[i, j]
                    if abs(corr_val) > 0.7:
                        high_corr_pairs.append({
                            'feature1': corr_matrix.columns[i],
                            'feature2': corr_matrix.columns[j],
                            'correlation': corr_val
                        })
            
            return {
                'correlation_matrix': corr_matrix.to_dict(),
                'high_correlations': high_corr_pairs
            }
        
        return {'correlation_matrix': {}, 'high_correlations': []}
    
    def _assess_data_quality(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Assess overall data quality"""
        quality_score = 100
        issues = []
        
        # Check missing values
        missing_percentage = (df.isnull().sum().sum() / (df.shape[0] * df.shape[1])) * 100
        if missing_percentage > 5:
            quality_score -= missing_percentage
            issues.append(f"High missing value percentage: {missing_percentage:.2f}%")
        
        # Check duplicates
        duplicate_percentage = (df.duplicated().sum() / len(df)) * 100
        if duplicate_percentage > 1:
            quality_score -= duplicate_percentage * 2
            issues.append(f"High duplicate percentage: {duplicate_percentage:.2f}%")
        
        # Check for constant columns
        constant_cols = [col for col in df.columns if df[col].nunique() == 1]
        if constant_cols:
            quality_score -= len(constant_cols) * 5
            issues.append(f"Constant columns found: {constant_cols}")
        
        return {
            'quality_score': max(0, quality_score),
            'issues': issues,
            'missing_percentage': missing_percentage,
            'duplicate_percentage': duplicate_percentage,
            'constant_columns': constant_cols
        }
    
    def _analyze_target(self, target_series: pd.Series) -> Dict[str, Any]:
        """Analyze target variable"""
        analysis = {
            'type': 'unknown',
            'distribution': {},
            'statistics': {}
        }
        
        if pd.api.types.is_numeric_dtype(target_series):
            if target_series.nunique() < 20:
                analysis['type'] = 'classification'
                analysis['distribution'] = target_series.value_counts().to_dict()
                analysis['class_balance'] = (target_series.value_counts() / len(target_series)).to_dict()
            else:
                analysis['type'] = 'regression'
                analysis['statistics'] = {
                    'mean': target_series.mean(),
                    'std': target_series.std(),
                    'min': target_series.min(),
                    'max': target_series.max(),
                    'skewness': stats.skew(target_series.dropna()),
                    'kurtosis': stats.kurtosis(target_series.dropna())
                }
        else:
            analysis['type'] = 'classification'
            analysis['distribution'] = target_series.value_counts().to_dict()
            analysis['class_balance'] = (target_series.value_counts() / len(target_series)).to_dict()
        
        return analysis
    
    def get_preprocessing_recommendations(self) -> List[str]:
        """Get AI-powered preprocessing recommendations"""
        recommendations = []
        
        if not self.analysis_results:
            return ["Please run analyze_dataset() first"]
        
        # Missing value recommendations
        missing_analysis = self.analysis_results.get('missing_values', {})
        for col, percentage in missing_analysis.get('missing_percentages', {}).items():
            if percentage > 0:
                if percentage < 5:
                    recommendations.append(f"Consider mean/median imputation for {col} ({percentage:.1f}% missing)")
                elif percentage < 30:
                    recommendations.append(f"Consider advanced imputation (KNN, MICE) for {col} ({percentage:.1f}% missing)")
                else:
                    recommendations.append(f"Consider dropping {col} or using indicator variables ({percentage:.1f}% missing)")
        
        # Outlier recommendations
        outliers = self.analysis_results.get('outliers', {})
        for col, outlier_info in outliers.items():
            if outlier_info['percentage'] > 5:
                recommendations.append(f"Consider outlier treatment for {col} ({outlier_info['percentage']:.1f}% outliers)")
        
        # Correlation recommendations
        correlations = self.analysis_results.get('correlations', {})
        for corr_pair in correlations.get('high_correlations', []):
            recommendations.append(f"High correlation detected between {corr_pair['feature1']} and {corr_pair['feature2']} (r={corr_pair['correlation']:.2f})")
        
        # Data quality recommendations
        quality = self.analysis_results.get('data_quality', {})
        if quality.get('quality_score', 100) < 80:
            recommendations.append("Data quality score is low. Consider data cleaning before modeling.")
        
        return recommendations
