"""
Preprocessing Recommender Module

AI-powered recommendations for preprocessing steps based on data characteristics
and best practices.
"""

import pandas as pd
import numpy as np
from typing import Dict, List, Any, Optional
from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler
import warnings


class PreprocessingRecommender:
    """
    AI-powered recommender system for preprocessing steps.
    Provides intelligent suggestions based on data characteristics.
    """
    
    def __init__(self):
        self.recommendations_history = []
        
    def recommend_preprocessing_pipeline(self, df: pd.DataFrame, 
                                        feature_types: Dict[str, str],
                                        target_column: Optional[str] = None) -> Dict[str, Any]:
        """
        Generate comprehensive preprocessing recommendations
        
        Args:
            df: Input DataFrame
            feature_types: Dictionary mapping column names to feature types
            target_column: Name of target column
            
        Returns:
            Dictionary containing recommendations
        """
        recommendations = {
            'missing_values': self._recommend_missing_value_strategy(df, feature_types),
            'scaling': self._recommend_scaling_strategy(df, feature_types),
            'encoding': self._recommend_encoding_strategy(df, feature_types),
            'feature_engineering': self._recommend_feature_engineering(df, feature_types),
            'outlier_detection': self._recommend_outlier_strategy(df, feature_types),
            'dimensionality_reduction': self._recommend_dimensionality_reduction(df),
            'data_quality': self._recommend_data_quality_improvements(df, feature_types)
        }
        
        if target_column and target_column in df.columns:
            recommendations['target_preprocessing'] = self._recommend_target_preprocessing(
                df[target_column])
        
        # Generate overall pipeline recommendation
        recommendations['pipeline_summary'] = self._generate_pipeline_summary(recommendations)
        
        self.recommendations_history.append(recommendations)
        return recommendations
    
    def _recommend_missing_value_strategy(self, df: pd.DataFrame, 
                                        feature_types: Dict[str, str]) -> List[Dict[str, Any]]:
        """Recommend missing value handling strategies"""
        recommendations = []
        
        for col in df.columns:
            if df[col].isnull().sum() > 0:
                missing_percentage = (df[col].isnull().sum() / len(df)) * 100
                feature_type = feature_types.get(col, 'unknown')
                
                if missing_percentage < 5:
                    if feature_type == 'numerical':
                        recommendations.append({
                            'column': col,
                            'strategy': 'simple_imputation',
                            'method': 'median',
                            'reason': f'Low missing percentage ({missing_percentage:.1f}%)'
                        })
                    else:
                        recommendations.append({
                            'column': col,
                            'strategy': 'simple_imputation',
                            'method': 'mode',
                            'reason': f'Low missing percentage ({missing_percentage:.1f}%)'
                        })
                        
                elif missing_percentage < 30:
                    if feature_type == 'numerical':
                        recommendations.append({
                            'column': col,
                            'strategy': 'advanced_imputation',
                            'method': 'KNN',
                            'reason': f'Moderate missing percentage ({missing_percentage:.1f}%)'
                        })
                    else:
                        recommendations.append({
                            'column': col,
                            'strategy': 'simple_imputation',
                            'method': 'mode',
                            'reason': f'Moderate missing percentage ({missing_percentage:.1f}%)'
                        })
                        
                else:
                    recommendations.append({
                        'column': col,
                        'strategy': 'drop_or_indicator',
                        'method': 'missing_indicator',
                        'reason': f'High missing percentage ({missing_percentage:.1f}%)'
                    })
        
        return recommendations
    
    def _recommend_scaling_strategy(self, df: pd.DataFrame, 
                                  feature_types: Dict[str, str]) -> List[Dict[str, Any]]:
        """Recommend scaling strategies for numerical features"""
        recommendations = []
        numerical_cols = [col for col in df.columns 
                         if feature_types.get(col) == 'numerical']
        
        if not numerical_cols:
            return recommendations
        
        # Analyze distributions
        for col in numerical_cols:
            skewness = df[col].skew()
            outlier_count = self._count_outliers(df[col])
            outlier_percentage = (outlier_count / len(df)) * 100
            
            if outlier_percentage > 5:
                recommendations.append({
                    'column': col,
                    'strategy': 'robust_scaling',
                    'scaler': 'RobustScaler',
                    'reason': f'High outlier percentage ({outlier_percentage:.1f}%)'
                })
            elif abs(skewness) > 2:
                recommendations.append({
                    'column': col,
                    'strategy': 'robust_scaling',
                    'scaler': 'RobustScaler',
                    'reason': f'Highly skewed distribution (skewness: {skewness:.2f})'
                })
            else:
                recommendations.append({
                    'column': col,
                    'strategy': 'standard_scaling',
                    'scaler': 'StandardScaler',
                    'reason': 'Normal distribution with few outliers'
                })
        
        return recommendations
    
    def _recommend_encoding_strategy(self, df: pd.DataFrame, 
                                   feature_types: Dict[str, str]) -> List[Dict[str, Any]]:
        """Recommend encoding strategies for categorical features"""
        recommendations = []
        categorical_cols = [col for col in df.columns 
                          if feature_types.get(col) == 'categorical']
        
        for col in categorical_cols:
            n_unique = df[col].nunique()
            cardinality_ratio = n_unique / len(df)
            
            if n_unique == 2:
                recommendations.append({
                    'column': col,
                    'strategy': 'label_encoding',
                    'encoder': 'LabelEncoder',
                    'reason': 'Binary categorical feature'
                })
            elif n_unique <= 10:
                recommendations.append({
                    'column': col,
                    'strategy': 'one_hot_encoding',
                    'encoder': 'OneHotEncoder',
                    'reason': f'Low cardinality ({n_unique} unique values)'
                })
            elif cardinality_ratio < 0.05:
                recommendations.append({
                    'column': col,
                    'strategy': 'one_hot_encoding',
                    'encoder': 'OneHotEncoder',
                    'reason': f'Moderate cardinality ({n_unique} unique values)'
                })
            else:
                recommendations.append({
                    'column': col,
                    'strategy': 'ordinal_encoding',
                    'encoder': 'OrdinalEncoder',
                    'reason': f'High cardinality ({n_unique} unique values)'
                })
        
        return recommendations
    
    def _recommend_feature_engineering(self, df: pd.DataFrame, 
                                    feature_types: Dict[str, str]) -> List[Dict[str, Any]]:
        """Recommend feature engineering strategies"""
        recommendations = []
        numerical_cols = [col for col in df.columns 
                         if feature_types.get(col) == 'numerical']
        
        # Polynomial features for numerical columns
        if len(numerical_cols) >= 2:
            correlations = df[numerical_cols].corr()
            high_corr_pairs = []
            
            for i in range(len(correlations.columns)):
                for j in range(i+1, len(correlations.columns)):
                    if abs(correlations.iloc[i, j]) > 0.7:
                        high_corr_pairs.append((correlations.columns[i], correlations.columns[j]))
            
            if high_corr_pairs:
                recommendations.append({
                    'strategy': 'polynomial_features',
                    'features': [pair[0] for pair in high_corr_pairs[:3]],
                    'reason': 'Highly correlated features may benefit from polynomial combinations'
                })
        
        # Interaction features
        if len(numerical_cols) >= 3:
            recommendations.append({
                'strategy': 'interaction_features',
                'features': numerical_cols[:5],
                'reason': 'Create interaction terms between numerical features'
            })
        
        # Binning for numerical features with many unique values
        for col in numerical_cols:
            if df[col].nunique() > 50:
                recommendations.append({
                    'strategy': 'binning',
                    'feature': col,
                    'method': 'quantile_binning',
                    'reason': f'High cardinality numerical feature ({df[col].nunique()} unique values)'
                })
                break
        
        return recommendations
    
    def _recommend_outlier_strategy(self, df: pd.DataFrame, 
                                  feature_types: Dict[str, str]) -> List[Dict[str, Any]]:
        """Recommend outlier detection and handling strategies"""
        recommendations = []
        numerical_cols = [col for col in df.columns 
                         if feature_types.get(col) == 'numerical']
        
        for col in numerical_cols:
            outlier_count = self._count_outliers(df[col])
            outlier_percentage = (outlier_count / len(df)) * 100
            
            if outlier_percentage > 0:
                if outlier_percentage < 1:
                    recommendations.append({
                        'column': col,
                        'strategy': 'keep_outliers',
                        'reason': f'Low outlier percentage ({outlier_percentage:.1f}%)'
                    })
                elif outlier_percentage < 5:
                    recommendations.append({
                        'column': col,
                        'strategy': 'winsorization',
                        'method': 'IQR',
                        'reason': f'Moderate outlier percentage ({outlier_percentage:.1f}%)'
                    })
                else:
                    recommendations.append({
                        'column': col,
                        'strategy': 'robust_methods',
                        'method': 'robust_scaling',
                        'reason': f'High outlier percentage ({outlier_percentage:.1f}%)'
                    })
        
        return recommendations
    
    def _recommend_dimensionality_reduction(self, df: pd.DataFrame) -> List[Dict[str, Any]]:
        """Recommend dimensionality reduction strategies"""
        recommendations = []
        
        if len(df.columns) > 50:
            recommendations.append({
                'strategy': 'feature_selection',
                'method': 'SelectKBest',
                'reason': f'High dimensionality ({len(df.columns)} features)'
            })
        
        if len(df.columns) > 100:
            recommendations.append({
                'strategy': 'pca',
                'method': 'PCA',
                'n_components': min(50, len(df.columns) // 2),
                'reason': f'Very high dimensionality ({len(df.columns)} features)'
            })
        
        return recommendations
    
    def _recommend_data_quality_improvements(self, df: pd.DataFrame, 
                                           feature_types: Dict[str, str]) -> List[Dict[str, Any]]:
        """Recommend data quality improvements"""
        recommendations = []
        
        # Check for duplicates
        duplicate_count = df.duplicated().sum()
        if duplicate_count > 0:
            recommendations.append({
                'issue': 'duplicate_rows',
                'count': duplicate_count,
                'recommendation': 'remove_duplicates',
                'reason': f'Found {duplicate_count} duplicate rows'
            })
        
        # Check for constant columns
        constant_cols = [col for col in df.columns if df[col].nunique() == 1]
        if constant_cols:
            recommendations.append({
                'issue': 'constant_columns',
                'columns': constant_cols,
                'recommendation': 'remove_columns',
                'reason': 'Constant columns provide no information'
            })
        
        # Check for highly correlated features
        numerical_cols = [col for col in df.columns 
                         if feature_types.get(col) == 'numerical']
        if len(numerical_cols) > 1:
            corr_matrix = df[numerical_cols].corr()
            high_corr_pairs = []
            
            for i in range(len(corr_matrix.columns)):
                for j in range(i+1, len(corr_matrix.columns)):
                    if abs(corr_matrix.iloc[i, j]) > 0.95:
                        high_corr_pairs.append((corr_matrix.columns[i], correlations.columns[j]))
            
            if high_corr_pairs:
                recommendations.append({
                    'issue': 'high_correlation',
                    'pairs': high_corr_pairs,
                    'recommendation': 'remove_one_feature',
                    'reason': 'Highly correlated features may cause multicollinearity'
                })
        
        return recommendations
    
    def _recommend_target_preprocessing(self, target_series: pd.Series) -> Dict[str, Any]:
        """Recommend preprocessing for target variable"""
        recommendations = {}
        
        if pd.api.types.is_numeric_dtype(target_series):
            if target_series.nunique() <= 20:
                # Classification task
                recommendations['task_type'] = 'classification'
                recommendations['encoding'] = 'label_encoding'
                
                # Check class balance
                class_counts = target_series.value_counts()
                min_class_ratio = class_counts.min() / class_counts.max()
                
                if min_class_ratio < 0.1:
                    recommendations['class_imbalance'] = {
                        'detected': True,
                        'severity': 'severe',
                        'recommendation': 'use_class_weights_or_sampling'
                    }
                elif min_class_ratio < 0.3:
                    recommendations['class_imbalance'] = {
                        'detected': True,
                        'severity': 'moderate',
                        'recommendation': 'consider_class_weights'
                    }
                else:
                    recommendations['class_imbalance'] = {
                        'detected': False,
                        'severity': 'balanced',
                        'recommendation': 'no_action_needed'
                    }
            else:
                # Regression task
                recommendations['task_type'] = 'regression'
                
                # Check distribution
                skewness = target_series.skew()
                if abs(skewness) > 2:
                    recommendations['transformation'] = {
                        'recommended': True,
                        'method': 'log_transform' if skewness > 0 else 'exp_transform',
                        'reason': f'Highly skewed distribution (skewness: {skewness:.2f})'
                    }
                else:
                    recommendations['transformation'] = {
                        'recommended': False,
                        'reason': 'Relatively normal distribution'
                    }
        else:
            recommendations['task_type'] = 'classification'
            recommendations['encoding'] = 'label_encoding'
        
        return recommendations
    
    def _generate_pipeline_summary(self, recommendations: Dict[str, Any]) -> List[str]:
        """Generate a summary of recommended preprocessing pipeline"""
        summary = []
        
        # Missing values
        if recommendations['missing_values']:
            summary.append("Handle missing values using recommended strategies")
        
        # Encoding
        if recommendations['encoding']:
            summary.append("Encode categorical features with appropriate methods")
        
        # Scaling
        if recommendations['scaling']:
            summary.append("Scale numerical features based on distribution")
        
        # Feature engineering
        if recommendations['feature_engineering']:
            summary.append("Apply feature engineering techniques")
        
        # Outlier handling
        if recommendations['outlier_detection']:
            summary.append("Handle outliers using robust methods")
        
        # Dimensionality reduction
        if recommendations['dimensionality_reduction']:
            summary.append("Apply dimensionality reduction if needed")
        
        return summary
    
    def _count_outliers(self, series: pd.Series) -> int:
        """Count outliers using IQR method"""
        Q1 = series.quantile(0.25)
        Q3 = series.quantile(0.75)
        IQR = Q3 - Q1
        lower_bound = Q1 - 1.5 * IQR
        upper_bound = Q3 + 1.5 * IQR
        
        outliers = ((series < lower_bound) | (series > upper_bound)).sum()
        return outliers
    
    def get_recommendation_explanation(self, recommendations: Dict[str, Any]) -> str:
        """Get detailed explanation of recommendations"""
        explanation = "Based on your data analysis, here are the recommended preprocessing steps:\n\n"
        
        for category, recs in recommendations.items():
            if recs and category != 'pipeline_summary':
                explanation += f"## {category.replace('_', ' ').title()}\n"
                
                if isinstance(recs, list):
                    for rec in recs:
                        if isinstance(rec, dict):
                            explanation += f"- {rec.get('reason', 'No reason provided')}\n"
                        else:
                            explanation += f"- {rec}\n"
                elif isinstance(recs, dict):
                    for key, value in recs.items():
                        explanation += f"- {key}: {value}\n"
                
                explanation += "\n"
        
        return explanation
