# createsonline/ai/services.py
"""
CREATESONLINE AI Services - COMPLETE ENHANCED VERSION

AI service implementations for OpenAI, Anthropic, local ML models,
and vector operations. Provides unified interface with internal fallback.
"""
import json
import os
import hashlib
import math
import random
import time
from typing import Dict, Any, List, Optional
from abc import ABC, abstractmethod
from datetime import datetime

# Internal imports
try:
    from ..http.client import HTTPClient, AsyncHTTPClient
    INTERNAL_HTTP_AVAILABLE = True
except ImportError:
    INTERNAL_HTTP_AVAILABLE = False

try:
    import numpy as np
    NUMPY_AVAILABLE = True
except ImportError:
    NUMPY_AVAILABLE = False

try:
    from ..data.dataframe import CreatesonlineDataFrame
    from ..data.series import CreatesonlineSeries
    INTERNAL_DATA_AVAILABLE = True
except ImportError:
    INTERNAL_DATA_AVAILABLE = False

# ========================================
# BASE AI SERVICE INTERFACE
# ========================================

class BaseAIService(ABC):
    """Base class for AI services with caching and configuration"""
    
    def __init__(self, config: Dict[str, Any]):
        """Initialize AI service with configuration"""
        self.config = config
        self.timeout = config.get("timeout", 30)
        self.enable_caching = config.get("enable_caching", True)
        self.cache_ttl = config.get("cache_ttl", 3600)
        self._cache = {}
        self._stats = {
            "requests": 0,
            "cache_hits": 0,
            "cache_misses": 0,
            "errors": 0,
            "total_tokens": 0,
            "avg_response_time": 0.0
        }
    
    def _get_cache_key(self, operation: str, **kwargs) -> str:
        """Generate cache key for operation"""
        # Remove sensitive data from cache key
        clean_kwargs = {k: v for k, v in kwargs.items() if 'key' not in k.lower() and 'token' not in k.lower()}
        cache_data = {
            "operation": operation,
            "params": clean_kwargs,
            "service": self.__class__.__name__
        }
        cache_str = json.dumps(cache_data, sort_keys=True, default=str)
        return hashlib.md5(cache_str.encode()).hexdigest()
    
    def _get_cached_result(self, cache_key: str) -> Optional[Any]:
        """Get cached result if available and valid"""
        if not self.enable_caching or cache_key not in self._cache:
            self._stats["cache_misses"] += 1
            return None
        
        cached_item = self._cache[cache_key]
        
        if (datetime.utcnow().timestamp() - cached_item["timestamp"]) < self.cache_ttl:
            self._stats["cache_hits"] += 1
            return cached_item["result"]
        else:
            # Remove expired cache entry
            del self._cache[cache_key]
            self._stats["cache_misses"] += 1
            return None
    
    def _set_cached_result(self, cache_key: str, result: Any):
        """Cache result with timestamp"""
        if self.enable_caching:
            self._cache[cache_key] = {
                "result": result,
                "timestamp": datetime.utcnow().timestamp()
            }
    
    def _update_stats(self, operation: str, response_time: float = 0.0, tokens: int = 0, error: bool = False):
        """Update service statistics"""
        self._stats["requests"] += 1
        if error:
            self._stats["errors"] += 1
        if tokens:
            self._stats["total_tokens"] += tokens
        if response_time:
            # Update average response time
            current_avg = self._stats["avg_response_time"]
            total_requests = self._stats["requests"]
            self._stats["avg_response_time"] = (current_avg * (total_requests - 1) + response_time) / total_requests
    
    def get_stats(self) -> Dict[str, Any]:
        """Get service statistics"""
        return {
            **self._stats,
            "cache_size": len(self._cache),
            "cache_hit_rate": self._stats["cache_hits"] / max(1, self._stats["cache_hits"] + self._stats["cache_misses"]),
            "error_rate": self._stats["errors"] / max(1, self._stats["requests"]),
            "uptime": "operational"
        }
    
    def clear_cache(self):
        """Clear service cache"""
        self._cache.clear()
    
    @abstractmethod
    async def generate_text(self, prompt: str, **kwargs) -> str:
        """Generate text using AI service"""
        pass
    
    @abstractmethod
    async def get_embedding(self, text: str, **kwargs) -> List[float]:
        """Get text embedding"""
        pass

# ========================================
# ENHANCED INTERNAL AI ENGINE
# ========================================

class EnhancedInternalAIEngine:
    """Enhanced pure Python AI engine with better algorithms"""
    
    def __init__(self):
        self.cache = {}
        self.models = {}
        self.vocabulary = set()
        self.patterns = {
            'positive': ['good', 'great', 'excellent', 'amazing', 'wonderful', 'fantastic', 'awesome', 'brilliant', 'perfect', 'outstanding'],
            'negative': ['bad', 'terrible', 'awful', 'horrible', 'disappointing', 'poor', 'worst', 'pathetic', 'disgusting', 'dreadful'],
            'technical': ['api', 'framework', 'algorithm', 'database', 'server', 'client', 'protocol', 'interface', 'implementation'],
            'business': ['revenue', 'profit', 'customer', 'market', 'sales', 'growth', 'strategy', 'roi', 'conversion', 'acquisition']
        }
        self._build_vocabulary()
    
    def _build_vocabulary(self):
        """Build internal vocabulary from patterns"""
        for category, words in self.patterns.items():
            self.vocabulary.update(words)
    
    def hash_text(self, text: str) -> str:
        """Generate consistent hash for text"""
        return hashlib.md5(text.encode()).hexdigest()
    
    def generate_embedding(self, text: str, dimensions: int = 768) -> List[float]:
        """Generate enhanced embedding from text using TF-IDF-like approach"""
        text_lower = text.lower()
        words = [word for word in text_lower.split() if word.isalpha()]
        
        # Calculate word frequencies
        word_freq = {}
        for word in words:
            word_freq[word] = word_freq.get(word, 0) + 1
        
        # Generate embedding based on semantic patterns
        embedding = [0.0] * dimensions
        
        # Use hash for base randomness but add semantic meaning
        hash_val = self.hash_text(text)
        
        for i in range(dimensions):
            # Base value from hash
            seed_char = hash_val[i % len(hash_val)]
            base_value = ord(seed_char) / 255.0
            
            # Add semantic components
            semantic_boost = 0.0
            
            # Check for pattern matches
            for category, pattern_words in self.patterns.items():
                category_score = sum(1 for word in words if word in pattern_words) / max(1, len(words))
                if category_score > 0:
                    # Add category-specific components to certain dimensions
                    if i % 4 == hash(category) % 4:
                        semantic_boost += category_score * 0.3
            
            # Calculate TF-IDF-like score for dimension
            if i < len(words):
                word = words[i % len(words)]
                tf = word_freq.get(word, 0) / len(words)
                # Simple IDF approximation
                idf = math.log(1000 / (10 + sum(1 for w in self.vocabulary if w == word)))
                tfidf_component = tf * idf * 0.2
            else:
                tfidf_component = 0.0
            
            # Combine components
            final_value = (base_value * 0.5 + semantic_boost + tfidf_component) - 0.5
            embedding[i] = max(-1.0, min(1.0, final_value))  # Clamp to [-1, 1]
        
        return embedding
    
    def similarity(self, vec1: List[float], vec2: List[float]) -> float:
        """Enhanced cosine similarity calculation"""
        if not vec1 or not vec2 or len(vec1) != len(vec2):
            return 0.0
        
        try:
            if NUMPY_AVAILABLE:
                v1 = np.array(vec1)
                v2 = np.array(vec2)
                return float(np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2)))
            else:
                # Manual calculation
                dot_product = sum(a * b for a, b in zip(vec1, vec2))
                norm1 = math.sqrt(sum(a * a for a in vec1))
                norm2 = math.sqrt(sum(b * b for b in vec2))
                
                if norm1 == 0 or norm2 == 0:
                    return 0.0
                
                return dot_product / (norm1 * norm2)
        except:
            return 0.0
    
    def generate_text(self, prompt: str, max_tokens: int = 100) -> str:
        """Enhanced text generation with better context understanding"""
        prompt_lower = prompt.lower()
        words = prompt_lower.split()
        
        # Analyze prompt intent
        intent = self._analyze_intent(prompt_lower, words)
        
        # Generate response based on intent
        if intent == 'summary':
            return self._generate_summary(prompt, max_tokens)
        elif intent == 'question':
            return self._generate_answer(prompt, max_tokens)
        elif intent == 'creative':
            return self._generate_creative(prompt, max_tokens)
        elif intent == 'technical':
            return self._generate_technical(prompt, max_tokens)
        else:
            return self._generate_general(prompt, max_tokens)
    
    def _analyze_intent(self, prompt_lower: str, words: List[str]) -> str:
        """Analyze prompt intent for better generation"""
        if any(word in prompt_lower for word in ['summarize', 'summary', 'sum up', 'brief']):
            return 'summary'
        elif any(word in prompt_lower for word in ['what', 'how', 'why', 'when', 'where', 'who', '?']):
            return 'question'
        elif any(word in prompt_lower for word in ['write', 'create', 'story', 'poem', 'creative']):
            return 'creative'
        elif any(word in prompt_lower for word in ['api', 'code', 'function', 'algorithm', 'technical', 'implement']):
            return 'technical'
        else:
            return 'general'
    
    def _generate_summary(self, prompt: str, max_tokens: int) -> str:
        """Generate summary-style response"""
        key_phrases = self._extract_key_phrases(prompt)
        return f"Summary: Key points include {', '.join(key_phrases[:3])}. {prompt[:100]}... (Generated by CREATESONLINE AI)"
    
    def _generate_answer(self, prompt: str, max_tokens: int) -> str:
        """Generate answer-style response"""
        if 'what is' in prompt.lower():
            subject = prompt.lower().split('what is')[1].strip().split()[0]
            return f"{subject.title()} is a concept/entity that relates to the context you've provided. Based on the CREATESONLINE AI analysis, this appears to be significant in your domain."
        elif 'how to' in prompt.lower():
            return f"To accomplish this task: 1) Analyze the requirements, 2) Plan the approach, 3) Implement systematically. CREATESONLINE recommends breaking down complex tasks into manageable steps."
        else:
            return f"Based on your question, the CREATESONLINE AI suggests considering multiple factors and approaches. The context indicates this is an important query that requires thoughtful analysis."
    
    def _generate_creative(self, prompt: str, max_tokens: int) -> str:
        """Generate creative content"""
        themes = self._extract_themes(prompt)
        return f"Creative Response: Inspired by {', '.join(themes)}, this creates an engaging narrative that captures the essence of your request. The CREATESONLINE AI weaves together elements to form a compelling piece."
    
    def _generate_technical(self, prompt: str, max_tokens: int) -> str:
        """Generate technical response"""
        tech_terms = [word for word in prompt.split() if word.lower() in self.patterns['technical']]
        return f"Technical Analysis: Regarding {', '.join(tech_terms)}, the CREATESONLINE framework recommends implementing best practices with consideration for scalability, maintainability, and performance optimization."
    
    def _generate_general(self, prompt: str, max_tokens: int) -> str:
        """Generate general response"""
        return f"AI Response: Based on your input '{prompt[:50]}...', the CREATESONLINE AI provides contextually relevant information and insights tailored to your specific needs and requirements."
    
    def _extract_key_phrases(self, text: str) -> List[str]:
        """Extract key phrases from text"""
        words = [word for word in text.lower().split() if word.isalpha() and len(word) > 3]
        # Simple frequency-based extraction
        word_freq = {}
        for word in words:
            word_freq[word] = word_freq.get(word, 0) + 1
        
        return sorted(word_freq.keys(), key=lambda x: word_freq[x], reverse=True)[:5]
    
    def _extract_themes(self, text: str) -> List[str]:
        """Extract themes from text"""
        themes = []
        for category, pattern_words in self.patterns.items():
            if any(word in text.lower() for word in pattern_words):
                themes.append(category)
        return themes or ['general']
    
    def classify_text(self, text: str, categories: List[str] = None) -> Dict[str, float]:
        """Enhanced text classification"""
        if not categories:
            categories = list(self.patterns.keys()) + ['neutral']
        
        text_lower = text.lower()
        words = text_lower.split()
        scores = {}
        
        for category in categories:
            if category in self.patterns:
                # Pattern-based scoring
                pattern_words = self.patterns[category]
                matches = sum(1 for word in words if word in pattern_words)
                scores[category] = min(1.0, matches / max(1, len(words)) * 2)
            elif category == 'neutral':
                # Neutral score is inverse of other categories
                other_scores = [scores.get(cat, 0) for cat in self.patterns.keys()]
                scores[category] = max(0.1, 1.0 - max(other_scores, default=0))
            else:
                scores[category] = 0.1  # Default low score
        
        # Normalize scores
        total = sum(scores.values())
        if total > 0:
            scores = {k: v / total for k, v in scores.items()}
        
        return scores
    
    def predict_numeric(self, features: Dict[str, Any]) -> float:
        """Enhanced numeric prediction with feature engineering"""
        feature_sum = 0
        feature_count = 0
        
        for key, value in features.items():
            if isinstance(value, (int, float)):
                # Apply feature-specific weights
                weight = 1.0
                if 'score' in key.lower() or 'rating' in key.lower():
                    weight = 1.5
                elif 'count' in key.lower() or 'number' in key.lower():
                    weight = 0.8
                elif 'time' in key.lower() or 'duration' in key.lower():
                    weight = 0.6
                
                feature_sum += value * weight
                feature_count += 1
            elif isinstance(value, str):
                # Text features
                sentiment_scores = self.classify_text(value, ['positive', 'negative'])
                feature_sum += sentiment_scores.get('positive', 0) * 0.3
                feature_count += 0.3
            elif isinstance(value, bool):
                feature_sum += 1.0 if value else 0.0
                feature_count += 1
        
        if feature_count == 0:
            return random.random()
        
        # Normalize and add some intelligent variation
        base_score = feature_sum / feature_count
        
        # Add deterministic but varied component based on feature hash
        feature_hash = self.hash_text(str(sorted(features.items())))
        hash_component = int(feature_hash[:8], 16) % 100 / 100.0
        
        # Combine with sigmoid function for better distribution
        final_score = 1 / (1 + math.exp(-(base_score - 0.5) * 3))
        final_score = (final_score * 0.7) + (hash_component * 0.3)
        
        return max(0.0, min(1.0, final_score))

# Global enhanced AI engine
_enhanced_ai_engine = EnhancedInternalAIEngine()

# ========================================
# OPENAI SERVICE (ENHANCED)
# ========================================

class OpenAIService(BaseAIService):
    """Enhanced OpenAI API service with better error handling and features.
    
    Supports multiple providers:
    - OpenAI (default): https://api.openai.com/v1
    - Azure OpenAI: https://{resource}.openai.azure.com/openai/deployments/{deployment}
    - GitHub Models: https://models.inference.ai.azure.com
    - Compatible APIs (Ollama, LM Studio, etc.)
    """
    
    def __init__(self, config: Dict[str, Any]):
        """Initialize OpenAI service with enhanced configuration"""
        super().__init__(config)
        self.api_key = config.get("openai_api_key") or os.getenv("OPENAI_API_KEY")
        self.base_url = config.get("base_url", "https://api.openai.com/v1")
        self.organization = config.get("organization") or os.getenv("OPENAI_ORG_ID")
        self.provider = config.get("provider", "openai")  # openai, azure, github, custom
        
        # Enhanced fallback mode
        if not self.api_key:
            self.api_key = "test-key-for-development"
            self._test_mode = True
        else:
            self._test_mode = False
        
        # Correct OpenAI headers
        self.headers = {
            "Authorization": f"Bearer {self.api_key}",
            "Content-Type": "application/json",
        }
        
        # Add organization header if provided
        if self.organization:
            self.headers["OpenAI-Organization"] = self.organization
        
        # Azure-specific headers
        if self.provider == "azure":
            self.headers = {
                "api-key": self.api_key,
                "Content-Type": "application/json",
            }
            self.api_version = config.get("api_version", "2024-02-15-preview")
        
        # Model configurations for OpenAI
        self.model_configs = {
            "gpt-4o": {"max_tokens": 16384, "cost_per_token": 0.000005},
            "gpt-4o-mini": {"max_tokens": 16384, "cost_per_token": 0.00000015},
            "gpt-4-turbo": {"max_tokens": 4096, "cost_per_token": 0.00001},
            "gpt-4": {"max_tokens": 8192, "cost_per_token": 0.00003},
            "gpt-3.5-turbo": {"max_tokens": 4096, "cost_per_token": 0.000002},
            "text-embedding-ada-002": {"dimensions": 1536, "cost_per_token": 0.0000001},
            "text-embedding-3-small": {"dimensions": 1536, "cost_per_token": 0.00000002},
            "text-embedding-3-large": {"dimensions": 3072, "cost_per_token": 0.00000013},
        }
    
    async def generate_text(
        self,
        prompt: str,
        model: str = None,
        max_tokens: int = None,
        temperature: float = None,
        system_prompt: str = None,
        **kwargs
    ) -> str:
        """Enhanced OpenAI text generation with Chat Completions API"""
        
        start_time = time.time()
        model = model or self.config.get("default_llm_model", "gpt-4o-mini")
        max_tokens = max_tokens or self.config.get("max_tokens", 500)
        temperature = temperature or self.config.get("temperature", 0.7)
        
        # Enhanced test mode
        if self._test_mode or not INTERNAL_HTTP_AVAILABLE:
            response_time = time.time() - start_time
            result = f"OpenAI Response (CREATESONLINE): {_enhanced_ai_engine.generate_text(prompt, max_tokens)}"
            self._update_stats("generate_text", response_time, len(result.split()))
            return result
        
        # Check cache
        cache_key = self._get_cache_key(
            "generate_text",
            prompt=prompt,
            model=model,
            max_tokens=max_tokens,
            temperature=temperature,
            system_prompt=system_prompt
        )
        
        cached_result = self._get_cached_result(cache_key)
        if cached_result:
            return cached_result
        
        # Prepare messages in OpenAI Chat Completions format
        messages = []
        if system_prompt:
            messages.append({"role": "system", "content": system_prompt})
        messages.append({"role": "user", "content": prompt})
        
        request_data = {
            "model": model,
            "messages": messages,
            "max_tokens": max_tokens,
            "temperature": temperature
        }
        
        # Build endpoint URL based on provider
        if self.provider == "azure":
            endpoint = f"{self.base_url}?api-version={self.api_version}"
        else:
            endpoint = f"{self.base_url}/chat/completions"
        
        try:
            # Use internal HTTP client
            client = AsyncHTTPClient()
            response = await client.post(
                url=endpoint,
                headers=self.headers,
                json=request_data,
                timeout=self.timeout
            )
            
            if response.status_code >= 400:
                raise ValueError(f"OpenAI API error: {response.status_code}")
                
            result = response.json()
            generated_text = result["choices"][0]["message"]["content"]
            
            # Track usage
            usage = result.get("usage", {})
            total_tokens = usage.get("total_tokens", 0)
            
            response_time = time.time() - start_time
            self._update_stats("generate_text", response_time, total_tokens)
            
            # Cache result
            self._set_cached_result(cache_key, generated_text)
            
            return generated_text
                
        except Exception as e:
            self._update_stats("generate_text", time.time() - start_time, 0, True)
            # Fallback to enhanced internal engine
            return f"OpenAI (Enhanced): {_enhanced_ai_engine.generate_text(prompt, max_tokens)}"
    
    async def get_embedding(self, text: str, model: str = None, **kwargs) -> List[float]:
        """Generate embeddings using OpenAI Embeddings API"""
        if self._test_mode or not INTERNAL_HTTP_AVAILABLE:
            return _enhanced_ai_engine.generate_embedding(text, kwargs.get("dimensions", 1536))
        
        model = model or self.config.get("embedding_model", "text-embedding-3-small")
        
        # Build endpoint URL based on provider
        if self.provider == "azure":
            endpoint = f"{self.base_url}?api-version={self.api_version}"
        else:
            endpoint = f"{self.base_url}/embeddings"
        
        request_data = {
            "model": model,
            "input": text
        }
        
        # Add dimensions if supported by model
        dimensions = kwargs.get("dimensions")
        if dimensions and model in ["text-embedding-3-small", "text-embedding-3-large"]:
            request_data["dimensions"] = dimensions
        
        try:
            client = AsyncHTTPClient()
            response = await client.post(
                url=endpoint,
                headers=self.headers,
                json=request_data,
                timeout=self.timeout
            )
            
            if response.status_code >= 400:
                raise ValueError(f"OpenAI Embeddings API error: {response.status_code}")
            
            result = response.json()
            return result["data"][0]["embedding"]
            
        except Exception as e:
            # Fallback to internal engine
            return _enhanced_ai_engine.generate_embedding(text, kwargs.get("dimensions", 1536))
    
    async def analyze_sentiment(self, text: str) -> Dict[str, Any]:
        """Analyze sentiment using OpenAI or enhanced internal engine"""
        if self._test_mode:
            return _enhanced_ai_engine.classify_text(text, ["positive", "negative", "neutral"])
        
        prompt = f"Analyze the sentiment of this text and return only a JSON object with 'sentiment' (positive/negative/neutral), 'confidence' (0-1): {text}"
        
        try:
            response = await self.generate_text(
                prompt=prompt,
                max_tokens=100,
                temperature=0.1
            )
            # Try to parse JSON response
            return json.loads(response)
        except:
            # Fallback to enhanced internal analysis
            sentiment_scores = _enhanced_ai_engine.classify_text(text, ["positive", "negative", "neutral"])
            best_sentiment = max(sentiment_scores.items(), key=lambda x: x[1])
            return {
                "sentiment": best_sentiment[0],
                "confidence": best_sentiment[1]
            }

# ========================================
# ENHANCED LOCAL ML SERVICE
# ========================================

class LocalMLService(BaseAIService):
    """Enhanced local machine learning service"""
    
    def __init__(self, config: Dict[str, Any]):
        """Initialize enhanced local ML service"""
        super().__init__(config)
        self.models = {}
        self.model_cache_dir = config.get("model_cache_dir", "./models")
        self.vector_store = {}
        
        # Create model directory if it doesn't exist
        import os
        os.makedirs(self.model_cache_dir, exist_ok=True)
        
        # Enhanced preprocessing pipelines
        self.preprocessors = {
            "text": self._preprocess_text,
            "numeric": self._preprocess_numeric,
            "categorical": self._preprocess_categorical
        }
    
    async def generate_text(self, prompt: str, model: str = None, **kwargs) -> str:
        """Generate text using local models or enhanced internal engine"""
        return _enhanced_ai_engine.generate_text(prompt, kwargs.get("max_tokens", 100))
    
    async def get_embedding(self, text: str, model: str = None, **kwargs) -> List[float]:
        """Generate embeddings using local models"""
        dimensions = kwargs.get("dimensions", 768)
        return _enhanced_ai_engine.generate_embedding(text, dimensions)
    
    async def predict(
        self,
        data: Dict[str, Any],
        model_name: str,
        prediction_type: str = "classification",
        **kwargs
    ) -> Dict[str, Any]:
        """Enhanced prediction with better feature processing"""
        
        start_time = time.time()
        
        # Check cache
        cache_key = self._get_cache_key(
            "predict",
            data=data,
            model_name=model_name,
            prediction_type=prediction_type
        )
        
        cached_result = self._get_cached_result(cache_key)
        if cached_result:
            return cached_result
        
        try:
            # Load or create model
            model = await self._get_or_create_model(model_name, prediction_type)
            
            # Enhanced feature preparation
            features = await self._prepare_enhanced_features(data)
            
            # Make prediction
            if prediction_type == "classification":
                prediction = await self._classify(model, features)
            elif prediction_type == "regression":
                prediction = await self._regress(model, features)
            elif prediction_type == "clustering":
                prediction = await self._cluster(model, features)
            else:
                prediction = await self._custom_predict(model, features, prediction_type)
            
            result = {
                "prediction": prediction["value"],
                "confidence": prediction["confidence"],
                "model": model_name,
                "prediction_type": prediction_type,
                "feature_importance": prediction.get("feature_importance", {}),
                "explanation": prediction.get("explanation", "")
            }
            
            response_time = time.time() - start_time
            self._update_stats("predict", response_time, len(str(data)))
            
            # Cache result
            self._set_cached_result(cache_key, result)
            
            return result
            
        except Exception as e:
            self._update_stats("predict", time.time() - start_time, 0, True)
            # Enhanced fallback prediction
            return await self._enhanced_fallback_prediction(data, prediction_type)
    
    async def _get_or_create_model(self, model_name: str, prediction_type: str):
        """Get existing model or create new one"""
        if model_name not in self.models:
            try:
                # Try to load from cache
                import os
                model_path = os.path.join(self.model_cache_dir, f"{model_name}.pkl")
                
                if os.path.exists(model_path):
                    import joblib
                    self.models[model_name] = joblib.load(model_path)
                else:
                    # Create new model
                    self.models[model_name] = await self._create_model(prediction_type)
                    
            except ImportError:
                # Create enhanced mock model
                self.models[model_name] = {
                    "type": prediction_type,
                    "created": datetime.utcnow(),
                    "training_data": [],
                    "enhanced": True
                }
        
        return self.models[model_name]
    
    async def _create_model(self, prediction_type: str):
        """Create new ML model"""
        try:
            if prediction_type == "classification":
                from sklearn.ensemble import RandomForestClassifier
                model = RandomForestClassifier(n_estimators=100, random_state=42)
            elif prediction_type == "regression":
                from sklearn.ensemble import RandomForestRegressor
                model = RandomForestRegressor(n_estimators=100, random_state=42)
            elif prediction_type == "clustering":
                from sklearn.cluster import KMeans
                model = KMeans(n_clusters=3, random_state=42)
            else:
                # Default to classification
                from sklearn.ensemble import RandomForestClassifier
                model = RandomForestClassifier(n_estimators=100, random_state=42)
            
            # Train with dummy data
            import numpy as np
            X_dummy = np.random.rand(100, 10)
            if prediction_type == "clustering":
                model.fit(X_dummy)
            else:
                y_dummy = np.random.randint(0, 2, 100) if prediction_type == "classification" else np.random.rand(100)
                model.fit(X_dummy, y_dummy)
            
            return model
            
        except ImportError:
            # Return enhanced mock model
            return {
                "type": prediction_type,
                "enhanced": True,
                "weights": [random.random() for _ in range(10)]
            }
    
    async def _prepare_enhanced_features(self, data: Dict[str, Any]) -> List[float]:
        """Enhanced feature preparation with multiple data types"""
        features = []
        
        for key, value in data.items():
            if isinstance(value, (int, float)):
                # Numeric features with normalization
                normalized_value = self._normalize_numeric(value)
                features.append(normalized_value)
                
            elif isinstance(value, str):
                # Text features with enhanced processing
                text_features = await self._extract_text_features(value)
                features.extend(text_features)
                
            elif isinstance(value, bool):
                features.append(float(value))
                
            elif isinstance(value, list):
                # List features - statistical summary
                if value and all(isinstance(x, (int, float)) for x in value):
                    features.extend([
                        sum(value) / len(value),  # mean
                        max(value) - min(value),  # range
                        len(value)  # count
                    ])
                else:
                    features.append(len(value))
            
            elif isinstance(value, dict):
                # Dict features - extract numeric values
                numeric_values = [v for v in value.values() if isinstance(v, (int, float))]
                if numeric_values:
                    features.append(sum(numeric_values) / len(numeric_values))
                else:
                    features.append(0.0)
        
        # Ensure we have at least 10 features (pad if necessary)
        while len(features) < 10:
            features.append(0.0)
        
        return features[:10]  # Limit to 10 features for consistency
    
    async def _extract_text_features(self, text: str) -> List[float]:
        """Extract enhanced features from text"""
        text_lower = text.lower()
        
        features = [
            len(text),  # Length
            len(text.split()),  # Word count
            len(set(text.split())) / max(1, len(text.split())),  # Unique word ratio
            text.count('.') + text.count('!') + text.count('?'),  # Sentence count
            sum(1 for char in text if char.isupper()) / max(1, len(text))  # Uppercase ratio
        ]
        
        # Sentiment features
        sentiment_scores = _enhanced_ai_engine.classify_text(text, ["positive", "negative"])
        features.extend([
            sentiment_scores.get("positive", 0),
            sentiment_scores.get("negative", 0)
        ])
        
        return features
    
    def _normalize_numeric(self, value: float) -> float:
        """Normalize numeric value"""
        # Simple min-max normalization to [0, 1]
        # In real implementation, this would use dataset statistics
        if value < 0:
            return 0.0
        elif value > 100:
            return 1.0
        else:
            return value / 100.0
    
    async def _classify(self, model, features: List[float]) -> Dict[str, Any]:
        """Enhanced classification"""
        try:
            if hasattr(model, 'predict') and hasattr(model, 'predict_proba'):
                # Real scikit-learn model
                import numpy as np
                X = np.array([features])
                prediction = model.predict(X)[0]
                probabilities = model.predict_proba(X)[0]
                confidence = float(max(probabilities))
                
                return {
                    "value": prediction,
                    "confidence": confidence,
                    "probabilities": probabilities.tolist(),
                    "explanation": f"Classified with {confidence:.2%} confidence"
                }
            else:
                # Enhanced mock classification
                return await self._enhanced_mock_classification(features)
                
        except Exception:
            return await self._enhanced_mock_classification(features)
    
    async def _enhanced_mock_classification(self, features: List[float]) -> Dict[str, Any]:
        """Enhanced mock classification with feature analysis"""
        # Analyze features for more intelligent prediction
        feature_sum = sum(features)
        feature_variance = sum((x - feature_sum/len(features))**2 for x in features) / len(features)
        
        # Generate prediction based on feature analysis
        if feature_sum > 5.0:
            prediction = "high_value"
            confidence = 0.85 + min(0.1, feature_variance * 0.1)
        elif feature_sum > 2.5:
            prediction = "medium_value"
            confidence = 0.75 + min(0.1, feature_variance * 0.05)
        else:
            prediction = "low_value"
            confidence = 0.65 + min(0.15, feature_variance * 0.15)
        
        return {
            "value": prediction,
            "confidence": min(0.95, confidence),
            "feature_importance": {f"feature_{i}": abs(f) for i, f in enumerate(features[:5])},
            "explanation": f"Classification based on feature analysis (sum: {feature_sum:.2f})"
        }
    
    async def _regress(self, model, features: List[float]) -> Dict[str, Any]:
        """Enhanced regression"""
        try:
            if hasattr(model, 'predict'):
                # Real scikit-learn model
                import numpy as np
                X = np.array([features])
                prediction = float(model.predict(X)[0])
                
                return {
                    "value": prediction,
                    "confidence": 0.8,  # Would be calculated from model variance
                    "explanation": f"Regression prediction: {prediction:.3f}"
                }
            else:
                return await self._enhanced_mock_regression(features)
                
        except Exception:
            return await self._enhanced_mock_regression(features)
    
    async def _enhanced_mock_regression(self, features: List[float]) -> Dict[str, Any]:
        """Enhanced mock regression"""
        # Weighted combination of features
        weights = [0.3, 0.2, 0.15, 0.1, 0.08, 0.05, 0.04, 0.03, 0.03, 0.02]
        prediction = sum(f * w for f, w in zip(features, weights))
        
        # Add some non-linearity
        prediction = 1 / (1 + math.exp(-prediction + 2.5))  # Sigmoid
        
        return {
            "value": prediction,
            "confidence": 0.82,
            "feature_importance": {f"feature_{i}": w for i, w in enumerate(weights)},
            "explanation": f"Weighted regression prediction: {prediction:.3f}"
        }
    
    async def _cluster(self, model, features: List[float]) -> Dict[str, Any]:
        """Enhanced clustering"""
        try:
            if hasattr(model, 'predict'):
                import numpy as np
                X = np.array([features])
                cluster = int(model.predict(X)[0])
                
                return {
                    "value": cluster,
                    "confidence": 0.7,
                    "explanation": f"Assigned to cluster {cluster}"
                }
            else:
                return await self._enhanced_mock_clustering(features)
                
        except Exception:
            return await self._enhanced_mock_clustering(features)
    
    async def _enhanced_mock_clustering(self, features: List[float]) -> Dict[str, Any]:
        """Enhanced mock clustering"""
        # Simple distance-based clustering
        feature_sum = sum(features)
        
        if feature_sum < 2.0:
            cluster = 0
        elif feature_sum < 4.0:
            cluster = 1
        else:
            cluster = 2
        
        return {
            "value": cluster,
            "confidence": 0.75,
            "explanation": f"Distance-based clustering (sum: {feature_sum:.2f})"
        }
    
    async def _custom_predict(self, model, features: List[float], prediction_type: str) -> Dict[str, Any]:
        """Custom prediction for unknown types"""
        return {
            "value": _enhanced_ai_engine.predict_numeric({"features": features}),
            "confidence": 0.6,
            "explanation": f"Custom prediction for {prediction_type}"
        }
    
    async def _enhanced_fallback_prediction(self, data: Dict[str, Any], prediction_type: str) -> Dict[str, Any]:
        """Enhanced fallback prediction when models fail"""
        if prediction_type == "classification":
            return {
                "prediction": "unknown",
                "confidence": 0.5,
                "model": "fallback_classifier",
                "prediction_type": prediction_type,
                "explanation": "Fallback classification due to model unavailability"
            }
        elif prediction_type == "regression":
            value = _enhanced_ai_engine.predict_numeric(data)
            return {
                "prediction": value,
                "confidence": 0.6,
                "model": "fallback_regressor",
                "prediction_type": prediction_type,
                "explanation": f"Fallback regression prediction: {value:.3f}"
            }
        else:
            return {
                "prediction": 0,
                "confidence": 0.4,
                "model": "fallback_generic",
                "prediction_type": prediction_type,
                "explanation": "Generic fallback prediction"
            }
    
    def _preprocess_text(self, text: str) -> Dict[str, Any]:
        """Enhanced text preprocessing"""
        return {
            "length": len(text),
            "word_count": len(text.split()),
            "sentiment": _enhanced_ai_engine.classify_text(text),
            "keywords": _enhanced_ai_engine._extract_key_phrases(text)[:5]
        }
    
    def _preprocess_numeric(self, value: float) -> Dict[str, Any]:
        """Enhanced numeric preprocessing"""
        return {
            "value": value,
            "normalized": self._normalize_numeric(value),
            "log_value": math.log(max(0.01, abs(value))),
            "category": "high" if value > 10 else "medium" if value > 1 else "low"
        }
    
    def _preprocess_categorical(self, value: str) -> Dict[str, Any]:
        """Enhanced categorical preprocessing"""
        return {
            "value": value,
            "hash": abs(hash(value)) % 1000,
            "length": len(value),
            "category_type": "text"
        }

# ========================================
# ENHANCED VECTOR SERVICE
# ========================================

class VectorService(BaseAIService):
    """Enhanced vector operations and similarity search service"""
    
    def __init__(self, config: Dict[str, Any]):
        """Initialize enhanced vector service"""
        super().__init__(config)
        self.vector_store = {}  # In-memory vector store
        self.indices = {}  # Vector indices for faster search
        self.index_name = "default"
        self.distance_metrics = {
            "cosine": self._cosine_similarity,
            "euclidean": self._euclidean_distance,
            "dot_product": self._dot_product,
            "manhattan": self._manhattan_distance
        }
    
    async def generate_text(self, prompt: str, **kwargs) -> str:
        """Vector service doesn't generate text, fallback to internal engine"""
        return _enhanced_ai_engine.generate_text(prompt, kwargs.get("max_tokens", 100))
    
    async def get_embedding(self, text: str, **kwargs) -> List[float]:
        """Generate embedding for vector operations"""
        dimensions = kwargs.get("dimensions", 768)
        return _enhanced_ai_engine.generate_embedding(text, dimensions)
    
    async def similarity_search(
        self,
        query_vector: List[float],
        documents: List[Dict[str, Any]] = None,
        top_k: int = 5,
        threshold: float = 0.8,
        metric: str = "cosine",
        **kwargs
    ) -> List[Dict[str, Any]]:
        """Enhanced similarity search with multiple metrics and optimization"""
        
        start_time = time.time()
        
        if documents is None:
            documents = self.vector_store.get(self.index_name, [])
        
        if not documents:
            return []
        
        # Check if we have a precomputed index
        index_key = f"{self.index_name}_{metric}_{len(documents)}"
        
        try:
            results = []
            distance_func = self.distance_metrics.get(metric, self._cosine_similarity)
            
            for i, doc in enumerate(documents):
                if "embedding" not in doc:
                    # Generate embedding if missing
                    text = doc.get("text", doc.get("content", str(doc)))
                    doc["embedding"] = await self.get_embedding(text)
                
                # Calculate similarity/distance
                if metric == "euclidean" or metric == "manhattan":
                    distance = distance_func(query_vector, doc["embedding"])
                    similarity = 1 / (1 + distance)  # Convert distance to similarity
                else:
                    similarity = distance_func(query_vector, doc["embedding"])
                
                if similarity >= threshold:
                    results.append({
                        "document": doc,
                        "similarity": similarity,
                        "score": similarity,
                        "index": i,
                        "distance": 1 - similarity if metric != "euclidean" else distance_func(query_vector, doc["embedding"])
                    })
            
            # Sort by similarity (descending)
            results.sort(key=lambda x: x["similarity"], reverse=True)
            
            # Return top_k results
            final_results = results[:top_k]
            
            response_time = time.time() - start_time
            self._update_stats("similarity_search", response_time, len(documents))
            
            return final_results
            
        except Exception as e:
            self._update_stats("similarity_search", time.time() - start_time, 0, True)
            return []
    
    def _cosine_similarity(self, vec1: List[float], vec2: List[float]) -> float:
        """Enhanced cosine similarity calculation"""
        if not vec1 or not vec2 or len(vec1) != len(vec2):
            return 0.0
        
        try:
            if NUMPY_AVAILABLE:
                v1 = np.array(vec1)
                v2 = np.array(vec2)
                
                dot_product = np.dot(v1, v2)
                norm_v1 = np.linalg.norm(v1)
                norm_v2 = np.linalg.norm(v2)
                
                if norm_v1 == 0 or norm_v2 == 0:
                    return 0.0
                
                return float(dot_product / (norm_v1 * norm_v2))
                
            else:
                # Manual calculation
                dot_product = sum(a * b for a, b in zip(vec1, vec2))
                norm_v1 = sum(a * a for a in vec1) ** 0.5
                norm_v2 = sum(b * b for b in vec2) ** 0.5
                
                if norm_v1 == 0 or norm_v2 == 0:
                    return 0.0
                
                return dot_product / (norm_v1 * norm_v2)
            
        except Exception:
            return 0.0
    
    def _euclidean_distance(self, vec1: List[float], vec2: List[float]) -> float:
        """Calculate Euclidean distance"""
        if not vec1 or not vec2 or len(vec1) != len(vec2):
            return float('inf')
        
        try:
            if NUMPY_AVAILABLE:
                v1 = np.array(vec1)
                v2 = np.array(vec2)
                return float(np.linalg.norm(v1 - v2))
            else:
                return sum((a - b) ** 2 for a, b in zip(vec1, vec2)) ** 0.5
        except Exception:
            return float('inf')
    
    def _dot_product(self, vec1: List[float], vec2: List[float]) -> float:
        """Calculate dot product"""
        if not vec1 or not vec2 or len(vec1) != len(vec2):
            return 0.0
        
        try:
            if NUMPY_AVAILABLE:
                return float(np.dot(vec1, vec2))
            else:
                return sum(a * b for a, b in zip(vec1, vec2))
        except Exception:
            return 0.0
    
    def _manhattan_distance(self, vec1: List[float], vec2: List[float]) -> float:
        """Calculate Manhattan distance"""
        if not vec1 or not vec2 or len(vec1) != len(vec2):
            return float('inf')
        
        try:
            return sum(abs(a - b) for a, b in zip(vec1, vec2))
        except Exception:
            return float('inf')
    
    async def add_documents(
        self,
        documents: List[Dict[str, Any]],
        index_name: str = None,
        auto_embed: bool = True,
        **kwargs
    ):
        """Enhanced document addition with automatic embedding"""
        index_name = index_name or self.index_name
        
        if index_name not in self.vector_store:
            self.vector_store[index_name] = []
        
        processed_docs = []
        
        for doc in documents:
            if auto_embed and "embedding" not in doc:
                # Generate embedding for document
                text = doc.get("text", doc.get("content", str(doc)))
                doc["embedding"] = await self.get_embedding(text)
                doc["embedding_model"] = "enhanced_internal"
                doc["embedding_timestamp"] = datetime.utcnow().isoformat()
            
            # Add metadata
            doc["added_at"] = datetime.utcnow().isoformat()
            doc["index"] = index_name
            doc["id"] = doc.get("id", f"doc_{len(self.vector_store[index_name])}")
            
            processed_docs.append(doc)
        
        self.vector_store[index_name].extend(processed_docs)
        
        return {
            "added": len(processed_docs),
            "total": len(self.vector_store[index_name]),
            "index": index_name
        }
    
    async def create_index(
        self,
        index_name: str,
        dimensions: int,
        metric: str = "cosine",
        **kwargs
    ):
        """Create a new vector index with optimization"""
        self.vector_store[index_name] = []
        self.indices[index_name] = {
            "dimensions": dimensions,
            "metric": metric,
            "created_at": datetime.utcnow().isoformat(),
            "document_count": 0
        }
        
        return {
            "status": "created",
            "index": index_name,
            "dimensions": dimensions,
            "metric": metric
        }
    
    async def get_index_stats(self, index_name: str = None) -> Dict[str, Any]:
        """Get statistics for vector index"""
        index_name = index_name or self.index_name
        
        documents = self.vector_store.get(index_name, [])
        
        return {
            "index_name": index_name,
            "document_count": len(documents),
            "has_embeddings": sum(1 for doc in documents if "embedding" in doc),
            "average_vector_length": (
                sum(len(doc.get("embedding", [])) for doc in documents) / max(1, len(documents))
                if documents else 0
            ),
            "index_size_mb": len(str(documents)) / (1024 * 1024),  # Rough estimate
            "last_updated": max(
                (doc.get("added_at", "") for doc in documents),
                default="never"
            )
        }
    
    def clear_index(self, index_name: str = None):
        """Clear vector index"""
        index_name = index_name or self.index_name
        
        if index_name in self.vector_store:
            del self.vector_store[index_name]
        
        if index_name in self.indices:
            del self.indices[index_name]
        
        return {"status": "cleared", "index": index_name}


# ========================================
# ANTHROPIC SERVICE (ENHANCED)
# ========================================

class AnthropicService(BaseAIService):
    """Enhanced Anthropic Claude API service"""
    
    def __init__(self, config: Dict[str, Any]):
        """Initialize Anthropic service"""
        super().__init__(config)
        self.api_key = config.get("anthropic_api_key") or os.getenv("ANTHROPIC_API_KEY")
        self.base_url = config.get("base_url", "https://api.anthropic.com/v1")
        
        # Enhanced fallback mode
        if not self.api_key:
            self.api_key = "test-key-for-development"
            self._test_mode = True
        else:
            self._test_mode = False
        
        # Anthropic-specific headers
        self.headers = {
            "x-api-key": self.api_key,
            "Content-Type": "application/json",
            "anthropic-version": "2023-06-01"
        }
        
        # Model configurations for Anthropic Claude
        self.model_configs = {
            "claude-3-5-sonnet-20241022": {"max_tokens": 8192, "cost_per_token": 0.000003},
            "claude-3-opus-20240229": {"max_tokens": 4096, "cost_per_token": 0.000015},
            "claude-3-sonnet-20240229": {"max_tokens": 4096, "cost_per_token": 0.000003},
            "claude-3-haiku-20240307": {"max_tokens": 4096, "cost_per_token": 0.00000025},
        }
    
    async def generate_text(
        self,
        prompt: str,
        model: str = None,
        max_tokens: int = None,
        temperature: float = None,
        system_prompt: str = None,
        **kwargs
    ) -> str:
        """Generate text using Anthropic Claude Messages API"""
        
        start_time = time.time()
        model = model or self.config.get("default_llm_model", "claude-3-5-sonnet-20241022")
        max_tokens = max_tokens or self.config.get("max_tokens", 1024)
        temperature = temperature or self.config.get("temperature", 0.7)
        
        # Enhanced test mode
        if self._test_mode or not INTERNAL_HTTP_AVAILABLE:
            response_time = time.time() - start_time
            result = f"Claude Response (CREATESONLINE): {_enhanced_ai_engine.generate_text(prompt, max_tokens)}"
            self._update_stats("generate_text", response_time, len(result.split()))
            return result
        
        # Check cache
        cache_key = self._get_cache_key(
            "generate_text",
            prompt=prompt,
            model=model,
            max_tokens=max_tokens,
            temperature=temperature,
            system_prompt=system_prompt
        )
        
        cached_result = self._get_cached_result(cache_key)
        if cached_result:
            return cached_result
        
        # Prepare Anthropic Messages API request
        request_data = {
            "model": model,
            "messages": [{"role": "user", "content": prompt}],
            "max_tokens": max_tokens,
            "temperature": temperature
        }
        
        if system_prompt:
            request_data["system"] = system_prompt
        
        try:
            client = AsyncHTTPClient()
            response = await client.post(
                url=f"{self.base_url}/messages",
                headers=self.headers,
                json=request_data,
                timeout=self.timeout
            )
            
            if response.status_code >= 400:
                raise ValueError(f"Anthropic API error: {response.status_code}")
                
            result = response.json()
            generated_text = result["content"][0]["text"]
            
            # Track usage
            usage = result.get("usage", {})
            total_tokens = usage.get("output_tokens", 0) + usage.get("input_tokens", 0)
            
            response_time = time.time() - start_time
            self._update_stats("generate_text", response_time, total_tokens)
            
            # Cache result
            self._set_cached_result(cache_key, generated_text)
            
            return generated_text
                
        except Exception as e:
            self._update_stats("generate_text", time.time() - start_time, 0, True)
            # Fallback to enhanced internal engine
            return f"Claude (Enhanced): {_enhanced_ai_engine.generate_text(prompt, max_tokens)}"
    
    async def get_embedding(self, text: str, **kwargs) -> List[float]:
        """Anthropic doesn't provide embeddings API, use enhanced internal engine"""
        return _enhanced_ai_engine.generate_embedding(text, kwargs.get("dimensions", 768))
    
    async def analyze_sentiment(self, text: str) -> Dict[str, Any]:
        """Analyze sentiment using Claude"""
        if self._test_mode:
            return _enhanced_ai_engine.classify_text(text, ["positive", "negative", "neutral"])
        
        prompt = f"Analyze the sentiment of this text and return only a JSON object with 'sentiment' (positive/negative/neutral), 'confidence' (0-1): {text}"
        
        try:
            response = await self.generate_text(
                prompt=prompt,
                max_tokens=100,
                temperature=0.1
            )
            return json.loads(response)
        except:
            sentiment_scores = _enhanced_ai_engine.classify_text(text, ["positive", "negative", "neutral"])
            best_sentiment = max(sentiment_scores.items(), key=lambda x: x[1])
            return {
                "sentiment": best_sentiment[0],
                "confidence": best_sentiment[1]
            }


# ========================================
# AI SERVICE FACTORY
# ========================================

def get_ai_service(config: Dict[str, Any] = None) -> BaseAIService:
    """Factory function to get appropriate AI service based on configuration.
    
    Args:
        config: Configuration dictionary with keys:
            - provider: 'openai', 'anthropic', 'local', 'vector' (default: 'openai')
            - openai_api_key or OPENAI_API_KEY env var
            - anthropic_api_key or ANTHROPIC_API_KEY env var
            - base_url: Custom API endpoint
            
    Returns:
        Appropriate AI service instance
    """
    config = config or {}
    provider = config.get("provider", "openai").lower()
    
    if provider == "anthropic" or provider == "claude":
        return AnthropicService(config)
    elif provider == "local" or provider == "ml":
        return LocalMLService(config)
    elif provider == "vector":
        return VectorService(config)
    else:  # Default to OpenAI
        return OpenAIService(config)