"""
Dataset Intelligence System (DIS)
==================================

Analyzes user datasets to understand characteristics, requirements,
and recommends optimal algorithms for the specific problem.

Features:
- Automatic dataset profiling and characteristic extraction
- Problem type classification
- Algorithm requirement analysis
- Intelligent algorithm recommendation based on dataset properties
- Data quality assessment
"""

import numpy as np
import pandas as pd
from typing import Dict, List, Tuple, Optional, Any, Union
from dataclasses import dataclass
from enum import Enum
import warnings


class ProblemType(Enum):
    """Classification of optimization problems"""
    NUMERICAL_CONTINUOUS = "numerical_continuous"
    NUMERICAL_DISCRETE = "numerical_discrete"
    COMBINATORIAL = "combinatorial"
    FEATURE_SELECTION = "feature_selection"
    PARAMETER_TUNING = "parameter_tuning"
    CLASSIFICATION = "classification"
    REGRESSION = "regression"
    CLUSTERING = "clustering"
    UNKNOWN = "unknown"


class DataCharacteristic(Enum):
    """Dataset characteristics"""
    MULTIMODAL = "multimodal"
    UNIMODAL = "unimodal"
    NOISY = "noisy"
    SPARSE = "sparse"
    DENSE = "dense"
    HIGH_DIMENSIONAL = "high_dimensional"
    LOW_DIMENSIONAL = "low_dimensional"
    IMBALANCED = "imbalanced"
    BALANCED = "balanced"
    DISCRETE = "discrete"
    CONTINUOUS = "continuous"
    MIXED = "mixed"


@dataclass
class DatasetProfile:
    """Comprehensive profile of a dataset"""
    name: str
    size: int
    dimensions: int
    num_samples: int
    feature_names: List[str]
    feature_types: Dict[str, str]  # name -> dtype
    statistics: Dict[str, Any]
    characteristics: List[DataCharacteristic]
    problem_type: ProblemType
    estimated_complexity: float  # 0-1
    dimensionality_ratio: float  # dimensions / num_samples
    data_quality_score: float  # 0-1
    requires_scaling: bool
    requires_encoding: bool
    recommended_algorithms: List[str]
    estimated_search_space_size: float
    diversity_score: float  # measure of data spread
    
    def to_dict(self) -> Dict:
        """Convert to dictionary"""
        return {
            'name': self.name,
            'size': self.size,
            'dimensions': self.dimensions,
            'num_samples': self.num_samples,
            'feature_names': self.feature_names,
            'feature_types': self.feature_types,
            'characteristics': [c.value for c in self.characteristics],
            'problem_type': self.problem_type.value,
            'estimated_complexity': float(self.estimated_complexity),
            'dimensionality_ratio': float(self.dimensionality_ratio),
            'data_quality_score': float(self.data_quality_score),
            'requires_scaling': self.requires_scaling,
            'requires_encoding': self.requires_encoding,
            'recommended_algorithms': self.recommended_algorithms,
            'estimated_search_space_size': float(self.estimated_search_space_size),
            'diversity_score': float(self.diversity_score)
        }


class DatasetIntelligence:
    """
    Intelligent system for analyzing datasets and understanding requirements.
    """
    
    def __init__(self):
        """Initialize the dataset intelligence system"""
        self.algorithm_requirements = self._initialize_algorithm_requirements()
        self.algorithm_suitability = self._initialize_algorithm_suitability()
    
    def _initialize_algorithm_requirements(self) -> Dict[str, Dict]:
        """
        Define requirements and constraints for each algorithm
        """
        return {
            'PSO': {
                'best_for': [ProblemType.NUMERICAL_CONTINUOUS, ProblemType.PARAMETER_TUNING],
                'characteristics': [DataCharacteristic.MULTIMODAL, DataCharacteristic.CONTINUOUS],
                'min_dimensions': 1,
                'max_dimensions': 5000,
                'scalability': 'excellent',
                'requires_scaling': False,
                'supports_constraints': True,
                'supports_discrete': False,
                'convergence_speed': 'fast',
                'population_flexibility': True,
                'best_complexity_range': (0.3, 0.9)
            },
            'GA': {
                'best_for': [ProblemType.COMBINATORIAL, ProblemType.FEATURE_SELECTION, ProblemType.NUMERICAL_DISCRETE],
                'characteristics': [DataCharacteristic.MULTIMODAL, DataCharacteristic.DISCRETE],
                'min_dimensions': 1,
                'max_dimensions': 10000,
                'scalability': 'very_good',
                'requires_scaling': False,
                'supports_constraints': True,
                'supports_discrete': True,
                'convergence_speed': 'medium',
                'population_flexibility': True,
                'best_complexity_range': (0.4, 1.0)
            },
            'DE': {
                'best_for': [ProblemType.NUMERICAL_CONTINUOUS, ProblemType.PARAMETER_TUNING],
                'characteristics': [DataCharacteristic.CONTINUOUS, DataCharacteristic.MULTIMODAL],
                'min_dimensions': 1,
                'max_dimensions': 1000,
                'scalability': 'very_good',
                'requires_scaling': True,
                'supports_constraints': True,
                'supports_discrete': False,
                'convergence_speed': 'very_fast',
                'population_flexibility': False,
                'best_complexity_range': (0.2, 0.8)
            },
            'GWO': {
                'best_for': [ProblemType.NUMERICAL_CONTINUOUS, ProblemType.FEATURE_SELECTION],
                'characteristics': [DataCharacteristic.CONTINUOUS, DataCharacteristic.MULTIMODAL],
                'min_dimensions': 1,
                'max_dimensions': 2000,
                'scalability': 'excellent',
                'requires_scaling': False,
                'supports_constraints': True,
                'supports_discrete': False,
                'convergence_speed': 'very_fast',
                'population_flexibility': True,
                'best_complexity_range': (0.3, 0.9)
            },
            'WOA': {
                'best_for': [ProblemType.NUMERICAL_CONTINUOUS, ProblemType.FEATURE_SELECTION],
                'characteristics': [DataCharacteristic.MULTIMODAL, DataCharacteristic.CONTINUOUS],
                'min_dimensions': 1,
                'max_dimensions': 3000,
                'scalability': 'excellent',
                'requires_scaling': False,
                'supports_constraints': True,
                'supports_discrete': False,
                'convergence_speed': 'fast',
                'population_flexibility': True,
                'best_complexity_range': (0.3, 0.9)
            },
            'ACO': {
                'best_for': [ProblemType.COMBINATORIAL],
                'characteristics': [DataCharacteristic.DISCRETE, DataCharacteristic.SPARSE],
                'min_dimensions': 1,
                'max_dimensions': 500,
                'scalability': 'good',
                'requires_scaling': False,
                'supports_constraints': True,
                'supports_discrete': True,
                'convergence_speed': 'medium',
                'population_flexibility': False,
                'best_complexity_range': (0.5, 1.0)
            },
            'ABC': {
                'best_for': [ProblemType.NUMERICAL_CONTINUOUS, ProblemType.PARAMETER_TUNING],
                'characteristics': [DataCharacteristic.MULTIMODAL, DataCharacteristic.CONTINUOUS],
                'min_dimensions': 1,
                'max_dimensions': 1000,
                'scalability': 'very_good',
                'requires_scaling': False,
                'supports_constraints': True,
                'supports_discrete': False,
                'convergence_speed': 'fast',
                'population_flexibility': True,
                'best_complexity_range': (0.2, 0.9)
            },
            'SA': {
                'best_for': [ProblemType.COMBINATORIAL, ProblemType.NUMERICAL_DISCRETE],
                'characteristics': [DataCharacteristic.DISCRETE, DataCharacteristic.MULTIMODAL],
                'min_dimensions': 1,
                'max_dimensions': 200,
                'scalability': 'poor',
                'requires_scaling': False,
                'supports_constraints': True,
                'supports_discrete': True,
                'convergence_speed': 'slow',
                'population_flexibility': False,
                'best_complexity_range': (0.5, 1.0)
            },
            'BA': {
                'best_for': [ProblemType.NUMERICAL_CONTINUOUS, ProblemType.PARAMETER_TUNING],
                'characteristics': [DataCharacteristic.MULTIMODAL, DataCharacteristic.CONTINUOUS],
                'min_dimensions': 1,
                'max_dimensions': 2000,
                'scalability': 'very_good',
                'requires_scaling': False,
                'supports_constraints': True,
                'supports_discrete': False,
                'convergence_speed': 'fast',
                'population_flexibility': True,
                'best_complexity_range': (0.3, 0.9)
            },
            'FSA': {
                'best_for': [ProblemType.NUMERICAL_CONTINUOUS, ProblemType.FEATURE_SELECTION],
                'characteristics': [DataCharacteristic.CONTINUOUS, DataCharacteristic.MULTIMODAL],
                'min_dimensions': 1,
                'max_dimensions': 1500,
                'scalability': 'very_good',
                'requires_scaling': False,
                'supports_constraints': True,
                'supports_discrete': False,
                'convergence_speed': 'very_fast',
                'population_flexibility': True,
                'best_complexity_range': (0.3, 0.9)
            }
        }
    
    def _initialize_algorithm_suitability(self) -> Dict[str, Dict]:
        """
        Define suitability scores for different characteristics
        """
        return {
            'PSO': {'multimodal': 0.9, 'unimodal': 0.7, 'noisy': 0.8, 'high_dimensional': 0.9, 'balanced': 0.85},
            'GA': {'multimodal': 0.95, 'unimodal': 0.6, 'noisy': 0.85, 'discrete': 0.95, 'balanced': 0.8},
            'DE': {'multimodal': 0.85, 'unimodal': 0.95, 'noisy': 0.7, 'high_dimensional': 0.85, 'continuous': 0.95},
            'GWO': {'multimodal': 0.85, 'unimodal': 0.9, 'noisy': 0.75, 'high_dimensional': 0.9, 'continuous': 0.9},
            'WOA': {'multimodal': 0.9, 'unimodal': 0.75, 'noisy': 0.8, 'high_dimensional': 0.9, 'balanced': 0.85},
            'ACO': {'combinatorial': 0.95, 'discrete': 0.9, 'sparse': 0.9, 'high_dimensional': 0.6, 'balanced': 0.75},
            'ABC': {'multimodal': 0.85, 'unimodal': 0.8, 'continuous': 0.9, 'balanced': 0.85, 'noisy': 0.8},
            'SA': {'combinatorial': 0.9, 'discrete': 0.85, 'sparse': 0.75, 'noisy': 0.85, 'balanced': 0.7},
            'BA': {'multimodal': 0.88, 'unimodal': 0.75, 'noisy': 0.82, 'high_dimensional': 0.85, 'balanced': 0.83},
            'FSA': {'multimodal': 0.87, 'unimodal': 0.82, 'continuous': 0.88, 'high_dimensional': 0.85, 'balanced': 0.84}
        }
    
    def analyze_dataset(self, data: Union[np.ndarray, pd.DataFrame, List],
                        target: Optional[Union[np.ndarray, pd.Series]] = None,
                        dataset_name: str = "dataset",
                        problem_type: Optional[str] = None) -> DatasetProfile:
        """
        Analyze a dataset and create a comprehensive profile.
        
        Parameters
        ----------
        data : array-like
            Input dataset (features)
        target : array-like, optional
            Target variable (for classification/regression)
        dataset_name : str
            Name of the dataset
        problem_type : str, optional
            Manually specified problem type
        
        Returns
        -------
        DatasetProfile with comprehensive analysis
        """
        
        # Convert to numpy array if needed
        if isinstance(data, pd.DataFrame):
            feature_names = list(data.columns)
            X = data.values
        elif isinstance(data, list):
            X = np.array(data)
            feature_names = [f"feature_{i}" for i in range(X.shape[1])]
        else:
            X = np.array(data)
            feature_names = [f"feature_{i}" for i in range(X.shape[1])]
        
        # Basic statistics
        num_samples, num_features = X.shape
        data_size = X.size
        
        # Feature type detection
        feature_types = self._detect_feature_types(X)
        
        # Calculate characteristics
        characteristics = self._identify_characteristics(X, target)
        
        # Estimate problem complexity
        complexity = self._estimate_complexity(X, target, characteristics)
        
        # Detect problem type
        if problem_type:
            detected_problem_type = self._parse_problem_type(problem_type)
        else:
            detected_problem_type = self._detect_problem_type(X, target, characteristics)
        
        # Calculate dimensionality ratio
        dimensionality_ratio = num_features / num_samples if num_samples > 0 else 1.0
        
        # Assess data quality
        quality_score = self._assess_data_quality(X, characteristics)
        
        # Calculate diversity
        diversity = self._calculate_diversity(X)
        
        # Determine preprocessing needs
        requires_scaling = self._needs_scaling(X)
        requires_encoding = any(ft == 'categorical' for ft in feature_types.values())
        
        # Estimate search space
        search_space = self._estimate_search_space(X, detected_problem_type)
        
        # Recommend algorithms
        recommended = self._recommend_algorithms(
            detected_problem_type, 
            characteristics, 
            num_features,
            complexity
        )
        
        return DatasetProfile(
            name=dataset_name,
            size=data_size,
            dimensions=num_features,
            num_samples=num_samples,
            feature_names=feature_names,
            feature_types=feature_types,
            statistics=self._compute_statistics(X),
            characteristics=characteristics,
            problem_type=detected_problem_type,
            estimated_complexity=complexity,
            dimensionality_ratio=dimensionality_ratio,
            data_quality_score=quality_score,
            requires_scaling=requires_scaling,
            requires_encoding=requires_encoding,
            recommended_algorithms=recommended,
            estimated_search_space_size=search_space,
            diversity_score=diversity
        )
    
    def _detect_feature_types(self, X: np.ndarray) -> Dict[str, str]:
        """Detect whether features are continuous or discrete"""
        feature_types = {}
        for i in range(X.shape[1]):
            col = X[:, i]
            # Check if all values are integers
            if np.all(np.equal(np.mod(col, 1), 0)):
                feature_types[f"feature_{i}"] = "integer"
            else:
                feature_types[f"feature_{i}"] = "continuous"
        return feature_types
    
    def _identify_characteristics(self, X: np.ndarray,
                                 target: Optional[np.ndarray] = None) -> List[DataCharacteristic]:
        """Identify dataset characteristics"""
        characteristics = []
        
        # Dimensionality
        if X.shape[1] > 50:
            characteristics.append(DataCharacteristic.HIGH_DIMENSIONAL)
        else:
            characteristics.append(DataCharacteristic.LOW_DIMENSIONAL)
        
        # Feature type
        has_discrete = any(np.all(np.equal(np.mod(X[:, i], 1), 0)) for i in range(X.shape[1]))
        has_continuous = any(not np.all(np.equal(np.mod(X[:, i], 1), 0)) for i in range(X.shape[1]))
        
        if has_discrete and has_continuous:
            characteristics.append(DataCharacteristic.MIXED)
        elif has_discrete:
            characteristics.append(DataCharacteristic.DISCRETE)
        else:
            characteristics.append(DataCharacteristic.CONTINUOUS)
        
        # Data density and sparsity
        sparsity = np.sum(X == 0) / X.size
        if sparsity > 0.7:
            characteristics.append(DataCharacteristic.SPARSE)
        else:
            characteristics.append(DataCharacteristic.DENSE)
        
        # Noise estimation (via variance analysis)
        if X.shape[0] > 10:
            variances = np.var(X, axis=0)
            mean_variance = np.mean(variances)
            if mean_variance > 10:
                characteristics.append(DataCharacteristic.NOISY)
        
        # Target-based characteristics
        if target is not None:
            target = np.array(target)
            if len(np.unique(target)) < len(target) / 10:
                characteristics.append(DataCharacteristic.IMBALANCED)
            else:
                characteristics.append(DataCharacteristic.BALANCED)
        
        return characteristics
    
    def _estimate_complexity(self, X: np.ndarray, target: Optional[np.ndarray],
                            characteristics: List[DataCharacteristic]) -> float:
        """Estimate problem complexity (0-1)"""
        complexity = 0.5  # Start at medium
        
        # Dimensionality increases complexity
        if DataCharacteristic.HIGH_DIMENSIONAL in characteristics:
            complexity += 0.3
        
        # Sparsity increases complexity
        if DataCharacteristic.SPARSE in characteristics:
            complexity += 0.1
        
        # Noise decreases effective complexity
        if DataCharacteristic.NOISY in characteristics:
            complexity -= 0.1
        
        # Class imbalance increases complexity
        if target is not None and DataCharacteristic.IMBALANCED in characteristics:
            complexity += 0.15
        
        return min(1.0, max(0.0, complexity))
    
    def _detect_problem_type(self, X: np.ndarray, target: Optional[np.ndarray],
                            characteristics: List[DataCharacteristic]) -> ProblemType:
        """Auto-detect problem type from data"""
        
        # If target exists, it's supervised learning
        if target is not None:
            # Could be classification, regression, feature selection
            unique_targets = len(np.unique(target))
            if unique_targets < 20 and unique_targets < len(target) / 10:
                return ProblemType.CLASSIFICATION
            else:
                return ProblemType.REGRESSION
        
        # Check if all discrete
        if DataCharacteristic.DISCRETE in characteristics:
            if X.shape[1] < 100:
                return ProblemType.NUMERICAL_DISCRETE
            else:
                return ProblemType.COMBINATORIAL
        
        # Default to continuous
        return ProblemType.NUMERICAL_CONTINUOUS
    
    def _parse_problem_type(self, problem_type: str) -> ProblemType:
        """Parse problem type string"""
        mapping = {
            'continuous': ProblemType.NUMERICAL_CONTINUOUS,
            'discrete': ProblemType.NUMERICAL_DISCRETE,
            'combinatorial': ProblemType.COMBINATORIAL,
            'feature_selection': ProblemType.FEATURE_SELECTION,
            'parameter_tuning': ProblemType.PARAMETER_TUNING,
            'classification': ProblemType.CLASSIFICATION,
            'regression': ProblemType.REGRESSION,
            'clustering': ProblemType.CLUSTERING,
        }
        return mapping.get(problem_type.lower(), ProblemType.UNKNOWN)
    
    def _assess_data_quality(self, X: np.ndarray,
                            characteristics: List[DataCharacteristic]) -> float:
        """Assess overall data quality"""
        quality = 1.0
        
        # Check for NaN or Inf
        if np.isnan(X).any() or np.isinf(X).any():
            quality -= 0.3
        
        # Check for constant columns
        if np.any(np.std(X, axis=0) == 0):
            quality -= 0.15
        
        # High sparsity reduces quality
        if DataCharacteristic.SPARSE in characteristics:
            quality -= 0.1
        
        # Imbalanced classes reduce quality
        if DataCharacteristic.IMBALANCED in characteristics:
            quality -= 0.05
        
        return max(0.0, quality)
    
    def _calculate_diversity(self, X: np.ndarray) -> float:
        """Calculate data diversity score"""
        if X.shape[0] < 2:
            return 0.5
        
        # Calculate pairwise distances
        from scipy.spatial.distance import pdist
        try:
            distances = pdist(X, metric='euclidean')
            # Normalize by expected distance range
            diversity = np.mean(distances) / (np.max(X) - np.min(X) + 1e-6)
            return min(1.0, max(0.0, diversity))
        except:
            return 0.5
    
    def _needs_scaling(self, X: np.ndarray) -> bool:
        """Check if data needs normalization/scaling"""
        # Check for large range differences
        ranges = np.max(X, axis=0) - np.min(X, axis=0)
        return np.max(ranges) / (np.min(ranges) + 1e-6) > 10
    
    def _estimate_search_space(self, X: np.ndarray, problem_type: ProblemType) -> float:
        """Estimate search space size"""
        if problem_type == ProblemType.NUMERICAL_DISCRETE:
            # Estimate based on range and number of features
            ranges = np.max(X, axis=0) - np.min(X, axis=0)
            space_size = np.prod(ranges[ranges > 0])
            return float(space_size)
        else:
            # Continuous space - estimate based on dimensionality and range
            return float(X.shape[1]) * 100
    
    def _compute_statistics(self, X: np.ndarray) -> Dict[str, Any]:
        """Compute descriptive statistics"""
        return {
            'mean': np.mean(X, axis=0).tolist(),
            'std': np.std(X, axis=0).tolist(),
            'min': np.min(X, axis=0).tolist(),
            'max': np.max(X, axis=0).tolist(),
            'median': np.median(X, axis=0).tolist(),
            'range': (np.max(X) - np.min(X)).item()
        }
    
    def _recommend_algorithms(self, problem_type: ProblemType,
                             characteristics: List[DataCharacteristic],
                             num_features: int,
                             complexity: float) -> List[str]:
        """Recommend algorithms based on dataset characteristics"""
        recommendations = []
        
        # Score each algorithm
        scores = {}
        for algo_name, req in self.algorithm_requirements.items():
            score = 0.0
            
            # Check problem type match
            if problem_type in req['best_for']:
                score += 0.3
            
            # Check characteristic match
            char_matches = sum(1 for c in characteristics if c.value.replace('_', ' ') in str(req['characteristics']).lower())
            score += (char_matches / max(len(characteristics), 1)) * 0.3
            
            # Check dimension suitability
            if req['min_dimensions'] <= num_features <= req['max_dimensions']:
                score += 0.2
            
            # Check complexity match
            if req['best_complexity_range'][0] <= complexity <= req['best_complexity_range'][1]:
                score += 0.15
            
            # Add characteristic-specific bonuses
            for char in characteristics:
                if algo_name in self.algorithm_suitability:
                    if char.value.replace('_', ' ') in self.algorithm_suitability[algo_name]:
                        score += self.algorithm_suitability[algo_name][char.value.replace('_', ' ')] * 0.05
            
            scores[algo_name] = score
        
        # Sort by score and return top 5
        sorted_algos = sorted(scores.items(), key=lambda x: x[1], reverse=True)
        recommendations = [algo for algo, _ in sorted_algos[:5]]
        
        return recommendations
    
    def get_intelligence_report(self, profile: DatasetProfile) -> Dict[str, Any]:
        """Generate an intelligence report for the dataset"""
        return {
            'profile': profile.to_dict(),
            'insights': {
                'problem_assessment': f"This appears to be a {profile.problem_type.value} problem",
                'complexity_assessment': f"Problem complexity: {'High' if profile.estimated_complexity > 0.7 else 'Medium' if profile.estimated_complexity > 0.4 else 'Low'}",
                'recommendations': profile.recommended_algorithms,
                'preprocessing_needs': {
                    'scaling': 'Required' if profile.requires_scaling else 'Not required',
                    'encoding': 'Required' if profile.requires_encoding else 'Not required'
                },
                'best_suited_algorithms': {
                    'primary': profile.recommended_algorithms[0] if profile.recommended_algorithms else 'None',
                    'alternatives': profile.recommended_algorithms[1:4] if len(profile.recommended_algorithms) > 1 else []
                }
            }
        }


# Export main classes
__all__ = ['DatasetIntelligence', 'DatasetProfile', 'ProblemType', 'DataCharacteristic']
