"""
Advanced Feedback Learning System for Toxo - Phase 2 Implementation

This module implements sophisticated feedback learning mechanisms including:
- Preference Learning (Bradley-Terry model, Ranking SVM)
- Online Learning with concept drift detection
- Real-time adaptation with forgetting mechanisms
- Multi-armed bandit for exploration/exploitation
- Active learning for strategic query selection
"""

import asyncio
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from typing import Dict, List, Any, Optional, Tuple, Union
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from enum import Enum
import json
import logging
from pathlib import Path
from collections import deque, defaultdict
import pickle

# ML libraries
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error
import scipy.stats as stats
from scipy.optimize import minimize

# Toxo imports
from ..utils.logger import get_logger
from ..utils.exceptions import LearningError, ValidationError
from ..core.config import ToxoConfig


class FeedbackType(Enum):
    """Types of feedback that can be processed."""
    BINARY_RATING = "binary_rating"
    DETAILED_CORRECTION = "detailed_correction"
    PREFERENCE_RANKING = "preference_ranking"
    CUSTOM_ANNOTATION = "custom_annotation"
    MULTI_DIMENSIONAL = "multi_dimensional"
    IMPLICIT_ACCEPTANCE = "implicit_acceptance"
    IMPLICIT_REJECTION = "implicit_rejection"
    TIME_SPENT = "time_spent"
    QUERY_REFINEMENT = "query_refinement"
    SESSION_COMPLETION = "session_completion"


class LearningStrategy(Enum):
    """Learning strategies for different scenarios."""
    BRADLEY_TERRY = "bradley_terry"
    RANKING_SVM = "ranking_svm"
    DEEP_PREFERENCE = "deep_preference"
    MULTI_ARMED_BANDIT = "multi_armed_bandit"
    CONTEXTUAL_BANDIT = "contextual_bandit"
    ACTIVE_LEARNING = "active_learning"
    META_LEARNING = "meta_learning"
    CONTINUAL_LEARNING = "continual_learning"


@dataclass
class FeedbackInstance:
    """Individual feedback instance with metadata."""
    feedback_id: str
    feedback_type: FeedbackType
    query: str
    response: str
    feedback_value: Union[float, int, str, Dict[str, Any]]
    confidence: float
    timestamp: datetime
    user_id: Optional[str] = None
    session_id: Optional[str] = None
    context: Dict[str, Any] = field(default_factory=dict)
    metadata: Dict[str, Any] = field(default_factory=dict)


@dataclass
class PreferenceComparison:
    """Pairwise preference comparison for ranking."""
    query: str
    response_a: str
    response_b: str
    preference: float  # -1 to 1, where 1 means A is strongly preferred
    confidence: float
    timestamp: datetime
    context: Dict[str, Any] = field(default_factory=dict)


@dataclass
class AdaptationTrigger:
    """Trigger for model adaptation."""
    trigger_id: str
    trigger_type: str
    threshold_value: float
    current_value: float
    timestamp: datetime
    metadata: Dict[str, Any] = field(default_factory=dict)


class BradleyTerryModel:
    """
    Bradley-Terry model for pairwise preference learning.
    """
    
    def __init__(self, num_items: int, regularization: float = 0.01):
        self.num_items = num_items
        self.regularization = regularization
        self.strengths = np.ones(num_items)  # Item strength parameters
        self.logger = get_logger(f"{__name__}.BradleyTerry")
    
    def fit(self, comparisons: List[Tuple[int, int, float]]) -> None:
        """
        Fit the Bradley-Terry model to pairwise comparisons.
        
        Args:
            comparisons: List of (item_a, item_b, preference) tuples
        """
        def negative_log_likelihood(strengths):
            nll = 0.0
            for i, j, preference in comparisons:
                if preference > 0:  # i preferred over j
                    prob_i_wins = strengths[i] / (strengths[i] + strengths[j])
                    nll -= np.log(prob_i_wins + 1e-10)
                elif preference < 0:  # j preferred over i
                    prob_j_wins = strengths[j] / (strengths[i] + strengths[j])
                    nll -= np.log(prob_j_wins + 1e-10)
            
            # Add regularization
            nll += self.regularization * np.sum(strengths**2)
            return nll
        
        # Optimize using scipy
        result = minimize(
            negative_log_likelihood,
            self.strengths,
            method='L-BFGS-B',
            bounds=[(0.01, 100.0)] * self.num_items
        )
        
        if result.success:
            self.strengths = result.x
            self.logger.info(f"Bradley-Terry model fitted successfully")
        else:
            self.logger.warning(f"Bradley-Terry optimization failed: {result.message}")
    
    def predict_preference(self, item_a: int, item_b: int) -> float:
        """Predict preference probability for item_a over item_b."""
        return self.strengths[item_a] / (self.strengths[item_a] + self.strengths[item_b])
    
    def get_rankings(self) -> List[int]:
        """Get items ranked by strength."""
        return np.argsort(self.strengths)[::-1].tolist()


class DeepPreferenceNetwork(nn.Module):
    """
    Deep neural network for learning complex preference patterns.
    """
    
    def __init__(self, input_dim: int, hidden_dims: List[int] = [256, 128, 64]):
        super().__init__()
        
        layers = []
        prev_dim = input_dim
        
        for hidden_dim in hidden_dims:
            layers.extend([
                nn.Linear(prev_dim, hidden_dim),
                nn.ReLU(),
                nn.Dropout(0.2),
                nn.BatchNorm1d(hidden_dim)
            ])
            prev_dim = hidden_dim
        
        layers.append(nn.Linear(prev_dim, 1))
        self.network = nn.Sequential(*layers)
    
    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return torch.sigmoid(self.network(x))


class MultiArmedBandit:
    """
    Multi-armed bandit for exploration/exploitation in response selection.
    """
    
    def __init__(self, num_arms: int, exploration_rate: float = 0.1):
        self.num_arms = num_arms
        self.exploration_rate = exploration_rate
        self.arm_counts = np.zeros(num_arms)
        self.arm_rewards = np.zeros(num_arms)
        self.total_count = 0
    
    def select_arm(self) -> int:
        """Select arm using epsilon-greedy strategy."""
        if np.random.random() < self.exploration_rate:
            return np.random.randint(self.num_arms)
        else:
            return np.argmax(self.get_arm_values())
    
    def update(self, arm: int, reward: float) -> None:
        """Update arm statistics with new reward."""
        self.arm_counts[arm] += 1
        self.arm_rewards[arm] += reward
        self.total_count += 1
    
    def get_arm_values(self) -> np.ndarray:
        """Get current estimated values for all arms."""
        values = np.zeros(self.num_arms)
        for i in range(self.num_arms):
            if self.arm_counts[i] > 0:
                values[i] = self.arm_rewards[i] / self.arm_counts[i]
        return values


class ConceptDriftDetector:
    """
    Detector for concept drift in feedback patterns.
    """
    
    def __init__(self, window_size: int = 100, sensitivity: float = 0.05):
        self.window_size = window_size
        self.sensitivity = sensitivity
        self.performance_history = deque(maxlen=window_size)
        self.baseline_performance = None
        self.drift_detected = False
    
    def add_performance(self, performance: float) -> bool:
        """
        Add new performance measurement and check for drift.
        
        Returns:
            True if drift is detected
        """
        self.performance_history.append(performance)
        
        if len(self.performance_history) < self.window_size:
            return False
        
        if self.baseline_performance is None:
            self.baseline_performance = np.mean(self.performance_history)
            return False
        
        current_performance = np.mean(list(self.performance_history)[-self.window_size//2:])
        
        # Statistical test for significant change
        _, p_value = stats.ttest_ind(
            list(self.performance_history)[:self.window_size//2],
            list(self.performance_history)[-self.window_size//2:]
        )
        
        self.drift_detected = p_value < self.sensitivity
        
        if self.drift_detected:
            self.baseline_performance = current_performance
        
        return self.drift_detected


class AdvancedFeedbackLearningSystem:
    """
    Comprehensive feedback learning system implementing multiple learning strategies.
    """
    
    def __init__(self, config: Optional[ToxoConfig] = None):
        self.config = config or ToxoConfig()
        self.logger = get_logger(__name__)
        
        # Feedback storage
        self.feedback_history: List[FeedbackInstance] = []
        self.preference_comparisons: List[PreferenceComparison] = []
        
        # Learning models
        self.bradley_terry_model: Optional[BradleyTerryModel] = None
        self.ranking_svm: Optional[SVR] = None
        self.deep_preference_net: Optional[DeepPreferenceNetwork] = None
        self.multi_armed_bandit: Optional[MultiArmedBandit] = None
        
        # Adaptation mechanisms
        self.concept_drift_detector = ConceptDriftDetector()
        self.adaptation_triggers: List[AdaptationTrigger] = []
        
        # Performance tracking
        self.performance_metrics: Dict[str, deque] = defaultdict(lambda: deque(maxlen=1000))
        self.learning_trajectory: List[Dict[str, Any]] = []
        
        # Feature extraction
        self.feature_scaler = StandardScaler()
        self.feature_cache: Dict[str, np.ndarray] = {}
        
        # Active learning
        self.uncertainty_threshold = 0.3
        self.query_budget = 100
        self.queries_used = 0
        
        self.logger.info("Advanced Feedback Learning System initialized")
    
    async def process_feedback(
        self,
        feedback: FeedbackInstance,
        immediate_adaptation: bool = True
    ) -> Dict[str, Any]:
        """
        Process new feedback and trigger learning updates.
        
        Args:
            feedback: Feedback instance to process
            immediate_adaptation: Whether to adapt immediately
            
        Returns:
            Processing results and adaptation info
        """
        try:
            self.logger.info(f"Processing feedback: {feedback.feedback_type.value}")
            
            # Store feedback
            self.feedback_history.append(feedback)
            
            # Extract features and update models
            features = await self._extract_feedback_features(feedback)
            
            # Process based on feedback type
            processing_result = await self._process_by_type(feedback, features)
            
            # Update performance metrics
            await self._update_performance_metrics(feedback, processing_result)
            
            # Check for concept drift
            drift_detected = self.concept_drift_detector.add_performance(
                processing_result.get('quality_score', 0.5)
            )
            
            if drift_detected:
                self.logger.warning("Concept drift detected - triggering adaptation")
                await self._trigger_adaptation("concept_drift")
            
            # Immediate adaptation if requested
            if immediate_adaptation:
                await self._apply_incremental_update(feedback, features)
            
            # Active learning query selection
            if await self._should_request_active_feedback(feedback):
                active_query = await self._generate_active_learning_query(feedback)
                processing_result['active_query'] = active_query
            
            return {
                'status': 'success',
                'feedback_id': feedback.feedback_id,
                'processing_result': processing_result,
                'drift_detected': drift_detected,
                'adaptation_triggered': immediate_adaptation,
                'performance_impact': await self._assess_performance_impact(feedback)
            }
            
        except Exception as e:
            self.logger.error(f"Error processing feedback: {str(e)}")
            raise LearningError(f"Failed to process feedback: {str(e)}")
    
    async def learn_preferences(
        self,
        comparisons: List[PreferenceComparison],
        strategy: LearningStrategy = LearningStrategy.BRADLEY_TERRY
    ) -> Dict[str, Any]:
        """
        Learn from preference comparisons using specified strategy.
        
        Args:
            comparisons: List of preference comparisons
            strategy: Learning strategy to use
            
        Returns:
            Learning results and model updates
        """
        try:
            self.logger.info(f"Learning preferences using {strategy.value}")
            
            self.preference_comparisons.extend(comparisons)
            
            if strategy == LearningStrategy.BRADLEY_TERRY:
                return await self._learn_bradley_terry(comparisons)
            elif strategy == LearningStrategy.RANKING_SVM:
                return await self._learn_ranking_svm(comparisons)
            elif strategy == LearningStrategy.DEEP_PREFERENCE:
                return await self._learn_deep_preference(comparisons)
            else:
                raise ValidationError(f"Unsupported learning strategy: {strategy}")
                
        except Exception as e:
            self.logger.error(f"Error learning preferences: {str(e)}")
            raise LearningError(f"Failed to learn preferences: {str(e)}")
    
    async def adapt_online(
        self,
        performance_feedback: Dict[str, float],
        adaptation_rate: float = 0.1
    ) -> Dict[str, Any]:
        """
        Perform online adaptation based on performance feedback.
        
        Args:
            performance_feedback: Current performance metrics
            adaptation_rate: Rate of adaptation (0-1)
            
        Returns:
            Adaptation results
        """
        try:
            self.logger.info("Performing online adaptation")
            
            # Update performance history
            for metric, value in performance_feedback.items():
                self.performance_metrics[metric].append(value)
            
            # Calculate adaptation signals
            adaptation_signals = await self._calculate_adaptation_signals(performance_feedback)
            
            # Apply gradual model updates
            update_results = await self._apply_gradual_updates(adaptation_signals, adaptation_rate)
            
            # Update learning trajectory
            self.learning_trajectory.append({
                'timestamp': datetime.now().isoformat(),
                'performance_feedback': performance_feedback,
                'adaptation_signals': adaptation_signals,
                'update_results': update_results
            })
            
            return {
                'status': 'success',
                'adaptation_signals': adaptation_signals,
                'update_results': update_results,
                'performance_trend': await self._analyze_performance_trend()
            }
            
        except Exception as e:
            self.logger.error(f"Error in online adaptation: {str(e)}")
            raise LearningError(f"Failed to adapt online: {str(e)}")
    
    async def _extract_feedback_features(self, feedback: FeedbackInstance) -> np.ndarray:
        """Extract features from feedback for learning."""
        # Create feature key for caching
        feature_key = f"{feedback.query}_{feedback.response}_{feedback.feedback_type.value}"
        
        if feature_key in self.feature_cache:
            return self.feature_cache[feature_key]
        
        # Extract basic features
        features = []
        
        # Text-based features
        features.extend([
            len(feedback.query.split()),  # Query length
            len(feedback.response.split()),  # Response length
            len(feedback.query),  # Query character count
            len(feedback.response),  # Response character count
        ])
        
        # Feedback-specific features
        if isinstance(feedback.feedback_value, (int, float)):
            features.append(float(feedback.feedback_value))
        else:
            features.append(0.0)
        
        features.append(feedback.confidence)
        
        # Context features
        features.extend([
            len(feedback.context),
            1.0 if feedback.user_id else 0.0,
            1.0 if feedback.session_id else 0.0
        ])
        
        # Convert to numpy array
        feature_array = np.array(features, dtype=np.float32)
        
        # Cache features
        self.feature_cache[feature_key] = feature_array
        
        return feature_array
    
    async def _process_by_type(
        self,
        feedback: FeedbackInstance,
        features: np.ndarray
    ) -> Dict[str, Any]:
        """Process feedback based on its type."""
        if feedback.feedback_type == FeedbackType.BINARY_RATING:
            return await self._process_binary_rating(feedback, features)
        elif feedback.feedback_type == FeedbackType.PREFERENCE_RANKING:
            return await self._process_preference_ranking(feedback, features)
        elif feedback.feedback_type == FeedbackType.DETAILED_CORRECTION:
            return await self._process_detailed_correction(feedback, features)
        elif feedback.feedback_type in [FeedbackType.IMPLICIT_ACCEPTANCE, FeedbackType.IMPLICIT_REJECTION]:
            return await self._process_implicit_feedback(feedback, features)
        else:
            return await self._process_generic_feedback(feedback, features)
    
    async def _process_binary_rating(
        self,
        feedback: FeedbackInstance,
        features: np.ndarray
    ) -> Dict[str, Any]:
        """Process binary rating feedback."""
        rating = float(feedback.feedback_value)
        
        # Update multi-armed bandit if available
        if self.multi_armed_bandit:
            # Assume response comes from a specific "arm" (strategy)
            arm_id = hash(feedback.response) % self.multi_armed_bandit.num_arms
            self.multi_armed_bandit.update(arm_id, rating)
        
        return {
            'type': 'binary_rating',
            'rating': rating,
            'quality_score': rating,
            'confidence': feedback.confidence
        }
    
    async def _process_preference_ranking(
        self,
        feedback: FeedbackInstance,
        features: np.ndarray
    ) -> Dict[str, Any]:
        """Process preference ranking feedback."""
        # Extract preference information
        preference_data = feedback.feedback_value
        if isinstance(preference_data, dict):
            preferred_response = preference_data.get('preferred_response')
            alternatives = preference_data.get('alternatives', [])
            
            # Create preference comparisons
            for alt_response in alternatives:
                comparison = PreferenceComparison(
                    query=feedback.query,
                    response_a=preferred_response,
                    response_b=alt_response,
                    preference=1.0,  # Preferred response is better
                    confidence=feedback.confidence,
                    timestamp=feedback.timestamp,
                    context=feedback.context
                )
                self.preference_comparisons.append(comparison)
        
        return {
            'type': 'preference_ranking',
            'preferences_extracted': len(self.preference_comparisons),
            'quality_score': 0.8,  # High quality for preference data
            'confidence': feedback.confidence
        }
    
    async def _process_detailed_correction(
        self,
        feedback: FeedbackInstance,
        features: np.ndarray
    ) -> Dict[str, Any]:
        """Process detailed correction feedback."""
        correction_data = feedback.feedback_value
        
        if isinstance(correction_data, dict):
            corrected_response = correction_data.get('corrected_response')
            error_type = correction_data.get('error_type')
            severity = correction_data.get('severity', 0.5)
            
            # Calculate quality score based on correction severity
            quality_score = max(0.0, 1.0 - severity)
            
            return {
                'type': 'detailed_correction',
                'error_type': error_type,
                'severity': severity,
                'quality_score': quality_score,
                'confidence': feedback.confidence,
                'has_correction': corrected_response is not None
            }
        
        return {
            'type': 'detailed_correction',
            'quality_score': 0.3,  # Low quality if correction is unclear
            'confidence': feedback.confidence
        }
    
    async def _process_implicit_feedback(
        self,
        feedback: FeedbackInstance,
        features: np.ndarray
    ) -> Dict[str, Any]:
        """Process implicit feedback signals."""
        if feedback.feedback_type == FeedbackType.IMPLICIT_ACCEPTANCE:
            quality_score = 0.7
        else:  # IMPLICIT_REJECTION
            quality_score = 0.3
        
        # Weight implicit feedback less than explicit
        confidence = feedback.confidence * 0.5
        
        return {
            'type': 'implicit_feedback',
            'signal': feedback.feedback_type.value,
            'quality_score': quality_score,
            'confidence': confidence
        }
    
    async def _process_generic_feedback(
        self,
        feedback: FeedbackInstance,
        features: np.ndarray
    ) -> Dict[str, Any]:
        """Process generic feedback types."""
        return {
            'type': 'generic',
            'quality_score': 0.5,
            'confidence': feedback.confidence
        }
    
    async def _learn_bradley_terry(self, comparisons: List[PreferenceComparison]) -> Dict[str, Any]:
        """Learn using Bradley-Terry model."""
        # Create item mapping
        responses = set()
        for comp in comparisons:
            responses.add(comp.response_a)
            responses.add(comp.response_b)
        
        response_to_id = {resp: i for i, resp in enumerate(responses)}
        
        # Convert comparisons to Bradley-Terry format
        bt_comparisons = []
        for comp in comparisons:
            id_a = response_to_id[comp.response_a]
            id_b = response_to_id[comp.response_b]
            bt_comparisons.append((id_a, id_b, comp.preference))
        
        # Initialize and fit model
        self.bradley_terry_model = BradleyTerryModel(len(responses))
        self.bradley_terry_model.fit(bt_comparisons)
        
        # Get rankings
        rankings = self.bradley_terry_model.get_rankings()
        ranked_responses = [list(responses)[i] for i in rankings]
        
        return {
            'model_type': 'bradley_terry',
            'num_items': len(responses),
            'num_comparisons': len(comparisons),
            'top_responses': ranked_responses[:5],
            'model_strengths': self.bradley_terry_model.strengths.tolist()
        }
    
    async def _learn_ranking_svm(self, comparisons: List[PreferenceComparison]) -> Dict[str, Any]:
        """Learn using Ranking SVM."""
        # Prepare training data
        X, y = [], []
        
        for comp in comparisons:
            # Extract features for both responses
            features_a = await self._extract_response_features(comp.query, comp.response_a)
            features_b = await self._extract_response_features(comp.query, comp.response_b)
            
            # Create pairwise difference features
            diff_features = features_a - features_b
            X.append(diff_features)
            y.append(comp.preference)
        
        if len(X) > 0:
            X = np.array(X)
            y = np.array(y)
            
            # Fit SVM
            self.ranking_svm = SVR(kernel='rbf', C=1.0)
            self.ranking_svm.fit(X, y)
            
            # Calculate training accuracy
            predictions = self.ranking_svm.predict(X)
            mse = mean_squared_error(y, predictions)
            
            return {
                'model_type': 'ranking_svm',
                'num_comparisons': len(comparisons),
                'training_mse': mse,
                'model_fitted': True
            }
        
        return {
            'model_type': 'ranking_svm',
            'num_comparisons': 0,
            'model_fitted': False
        }
    
    async def _learn_deep_preference(self, comparisons: List[PreferenceComparison]) -> Dict[str, Any]:
        """Learn using deep preference network."""
        if not comparisons:
            return {'model_type': 'deep_preference', 'model_fitted': False}
        
        # Prepare training data
        X, y = [], []
        
        for comp in comparisons:
            features_a = await self._extract_response_features(comp.query, comp.response_a)
            features_b = await self._extract_response_features(comp.query, comp.response_b)
            
            # Concatenate features
            combined_features = np.concatenate([features_a, features_b])
            X.append(combined_features)
            y.append((comp.preference + 1) / 2)  # Normalize to [0, 1]
        
        if len(X) > 10:  # Need sufficient data for deep learning
            X = torch.FloatTensor(X)
            y = torch.FloatTensor(y).unsqueeze(1)
            
            # Initialize network
            input_dim = X.shape[1]
            self.deep_preference_net = DeepPreferenceNetwork(input_dim)
            
            # Training
            optimizer = optim.Adam(self.deep_preference_net.parameters(), lr=0.001)
            criterion = nn.BCELoss()
            
            num_epochs = 100
            for epoch in range(num_epochs):
                optimizer.zero_grad()
                predictions = self.deep_preference_net(X)
                loss = criterion(predictions, y)
                loss.backward()
                optimizer.step()
            
            return {
                'model_type': 'deep_preference',
                'num_comparisons': len(comparisons),
                'final_loss': loss.item(),
                'model_fitted': True
            }
        
        return {
            'model_type': 'deep_preference',
            'insufficient_data': True,
            'model_fitted': False
        }
    
    async def _extract_response_features(self, query: str, response: str) -> np.ndarray:
        """Extract features for a query-response pair."""
        features = [
            len(query.split()),
            len(response.split()),
            len(query),
            len(response),
            response.count('?'),
            response.count('!'),
            response.count('.'),
            1.0 if any(word in response.lower() for word in ['yes', 'no']) else 0.0,
            1.0 if any(word in response.lower() for word in ['maybe', 'possibly']) else 0.0,
            len(set(query.lower().split()) & set(response.lower().split())) / max(len(query.split()), 1)
        ]
        
        return np.array(features, dtype=np.float32)
    
    async def _update_performance_metrics(
        self,
        feedback: FeedbackInstance,
        processing_result: Dict[str, Any]
    ) -> None:
        """Update performance metrics based on feedback."""
        quality_score = processing_result.get('quality_score', 0.5)
        confidence = processing_result.get('confidence', 0.5)
        
        self.performance_metrics['quality'].append(quality_score)
        self.performance_metrics['confidence'].append(confidence)
        self.performance_metrics['feedback_count'].append(len(self.feedback_history))
        
        # Calculate rolling averages
        if len(self.performance_metrics['quality']) >= 10:
            recent_quality = np.mean(list(self.performance_metrics['quality'])[-10:])
            self.performance_metrics['rolling_quality'].append(recent_quality)
    
    async def _trigger_adaptation(self, trigger_type: str) -> None:
        """Trigger model adaptation."""
        trigger = AdaptationTrigger(
            trigger_id=f"{trigger_type}_{datetime.now().timestamp()}",
            trigger_type=trigger_type,
            threshold_value=0.05,
            current_value=0.1,  # Example value
            timestamp=datetime.now()
        )
        
        self.adaptation_triggers.append(trigger)
        self.logger.info(f"Adaptation triggered: {trigger_type}")
    
    async def _apply_incremental_update(
        self,
        feedback: FeedbackInstance,
        features: np.ndarray
    ) -> None:
        """Apply incremental model updates."""
        # Update feature scaler
        if len(self.feedback_history) > 1:
            all_features = np.array([
                await self._extract_feedback_features(fb) for fb in self.feedback_history[-10:]
            ])
            self.feature_scaler.partial_fit(all_features)
    
    async def _should_request_active_feedback(self, feedback: FeedbackInstance) -> bool:
        """Determine if active learning query should be generated."""
        if self.queries_used >= self.query_budget:
            return False
        
        # Check uncertainty
        uncertainty = 1.0 - feedback.confidence
        return uncertainty > self.uncertainty_threshold
    
    async def _generate_active_learning_query(
        self,
        feedback: FeedbackInstance
    ) -> Dict[str, Any]:
        """Generate active learning query for strategic feedback collection."""
        self.queries_used += 1
        
        return {
            'query_type': 'clarification',
            'original_query': feedback.query,
            'original_response': feedback.response,
            'question': f"Could you provide more specific feedback about what could be improved in this response?",
            'expected_feedback_type': FeedbackType.DETAILED_CORRECTION.value,
            'priority': 'high' if feedback.confidence < 0.3 else 'medium'
        }
    
    async def _assess_performance_impact(self, feedback: FeedbackInstance) -> Dict[str, float]:
        """Assess the impact of feedback on performance."""
        if len(self.performance_metrics['quality']) < 2:
            return {'impact_score': 0.0}
        
        recent_quality = list(self.performance_metrics['quality'])[-5:]
        previous_quality = list(self.performance_metrics['quality'])[-10:-5] if len(self.performance_metrics['quality']) >= 10 else []
        
        if previous_quality:
            impact_score = np.mean(recent_quality) - np.mean(previous_quality)
        else:
            impact_score = 0.0
        
        return {
            'impact_score': impact_score,
            'recent_quality': np.mean(recent_quality),
            'trend': 'improving' if impact_score > 0.05 else 'declining' if impact_score < -0.05 else 'stable'
        }
    
    async def _calculate_adaptation_signals(
        self,
        performance_feedback: Dict[str, float]
    ) -> Dict[str, float]:
        """Calculate signals for model adaptation."""
        signals = {}
        
        for metric, value in performance_feedback.items():
            if metric in self.performance_metrics:
                history = list(self.performance_metrics[metric])
                if len(history) > 5:
                    recent_avg = np.mean(history[-5:])
                    overall_avg = np.mean(history)
                    signals[f"{metric}_deviation"] = recent_avg - overall_avg
                    signals[f"{metric}_trend"] = np.polyfit(range(len(history[-10:])), history[-10:], 1)[0] if len(history) >= 10 else 0.0
        
        return signals
    
    async def _apply_gradual_updates(
        self,
        adaptation_signals: Dict[str, float],
        adaptation_rate: float
    ) -> Dict[str, Any]:
        """Apply gradual model updates based on adaptation signals."""
        updates_applied = []
        
        for signal_name, signal_value in adaptation_signals.items():
            if abs(signal_value) > 0.1:  # Significant signal
                update_magnitude = signal_value * adaptation_rate
                updates_applied.append({
                    'signal': signal_name,
                    'value': signal_value,
                    'update_magnitude': update_magnitude
                })
        
        return {
            'updates_applied': updates_applied,
            'total_updates': len(updates_applied),
            'adaptation_rate': adaptation_rate
        }
    
    async def _analyze_performance_trend(self) -> Dict[str, Any]:
        """Analyze overall performance trends."""
        if len(self.performance_metrics['quality']) < 10:
            return {'trend': 'insufficient_data'}
        
        quality_history = list(self.performance_metrics['quality'])
        
        # Calculate trend
        x = np.arange(len(quality_history))
        slope, intercept = np.polyfit(x, quality_history, 1)
        
        # Calculate recent performance
        recent_performance = np.mean(quality_history[-10:])
        overall_performance = np.mean(quality_history)
        
        return {
            'trend_slope': slope,
            'trend_direction': 'improving' if slope > 0.01 else 'declining' if slope < -0.01 else 'stable',
            'recent_performance': recent_performance,
            'overall_performance': overall_performance,
            'performance_variance': np.var(quality_history),
            'total_feedback_count': len(self.feedback_history)
        }
    
    def save_state(self, path: Path) -> None:
        """Save the learning system state."""
        state = {
            'feedback_history': [asdict(fb) for fb in self.feedback_history],
            'preference_comparisons': [asdict(pc) for pc in self.preference_comparisons],
            'performance_metrics': {k: list(v) for k, v in self.performance_metrics.items()},
            'learning_trajectory': self.learning_trajectory,
            'adaptation_triggers': [asdict(at) for at in self.adaptation_triggers],
            'queries_used': self.queries_used,
            'feature_cache': {k: v.tolist() for k, v in self.feature_cache.items()}
        }
        
        with open(path, 'w') as f:
            json.dump(state, f, indent=2, default=str)
        
        self.logger.info(f"Learning system state saved to {path}")
    
    def load_state(self, path: Path) -> None:
        """Load the learning system state."""
        with open(path, 'r') as f:
            state = json.load(f)
        
        # Restore feedback history
        self.feedback_history = [
            FeedbackInstance(
                feedback_id=fb['feedback_id'],
                feedback_type=FeedbackType(fb['feedback_type']),
                query=fb['query'],
                response=fb['response'],
                feedback_value=fb['feedback_value'],
                confidence=fb['confidence'],
                timestamp=datetime.fromisoformat(fb['timestamp']),
                user_id=fb.get('user_id'),
                session_id=fb.get('session_id'),
                context=fb.get('context', {}),
                metadata=fb.get('metadata', {})
            ) for fb in state['feedback_history']
        ]
        
        # Restore other state
        self.performance_metrics = {k: deque(v, maxlen=1000) for k, v in state['performance_metrics'].items()}
        self.learning_trajectory = state['learning_trajectory']
        self.queries_used = state.get('queries_used', 0)
        self.feature_cache = {k: np.array(v) for k, v in state.get('feature_cache', {}).items()}
        
        self.logger.info(f"Learning system state loaded from {path}")
    
    def get_state(self) -> Dict[str, Any]:
        """Get the current state for serialization."""
        return {
            "config_available": self.config is not None,
            "feedback_history_count": len(getattr(self, 'feedback_history', [])),
            "preference_comparisons_count": len(getattr(self, 'preference_comparisons', [])),
            "bradley_terry_available": getattr(self, 'bradley_terry_model', None) is not None,
            "ranking_svm_available": getattr(self, 'ranking_svm', None) is not None,
            "deep_preference_net_available": getattr(self, 'deep_preference_net', None) is not None,
            "multi_armed_bandit_available": getattr(self, 'multi_armed_bandit', None) is not None,
            "concept_drift_detector_available": hasattr(self, 'concept_drift_detector') and self.concept_drift_detector is not None,
            "queries_used": getattr(self, 'queries_used', 0),
            "system_type": "AdvancedFeedbackLearningSystem"
        }


# Factory function for easy instantiation
def create_advanced_feedback_learning_system(config: Optional[ToxoConfig] = None) -> AdvancedFeedbackLearningSystem:
    """Create and return an Advanced Feedback Learning System instance."""
    return AdvancedFeedbackLearningSystem(config) 