"""
Decision Embedding Pipeline Module

This module provides comprehensive decision embedding pipeline capabilities that
generate both semantic and structural embeddings for decision tracking and
precedent search.

Key Features:
    - Decision embedding generation with semantic + structural components
    - Integration with KG module for structural embeddings (Node2Vec)
    - Batch processing capabilities for multiple decisions
    - Metadata enrichment for decisions
    - Configurable embedding parameters

Algorithms Used:
    - Node2Vec: Structural embeddings from KG module for graph topology
    - PathFinder: Shortest path algorithms for path-based similarity enhancement
    - CommunityDetector: Community detection for contextual entity relationships
    - CentralityCalculator: Centrality measures for weighted entity aggregation
    - SimilarityCalculator: Graph-based similarity calculations
    - ConnectivityAnalyzer: Graph connectivity analysis for embedding enhancement
    - HybridSimilarityCalculator: Combines semantic + structural embeddings
    - Weighted Aggregation: Centrality-based embedding combination
    - Path-based Enhancement: Multi-hop path similarity integration
    - Community Context: Community-based embedding enrichment

Main Classes:
    - DecisionEmbeddingPipeline: Main decision embedding pipeline

Example Usage:
    >>> from semantica.vector_store import DecisionEmbeddingPipeline
    >>> pipeline = DecisionEmbeddingPipeline(vector_store=vs, graph_store=gs)
    >>> embeddings = pipeline.process_decision_batch([
    ...     {"scenario": "Credit increase", "outcome": "approved"}
    ... ])
"""

from typing import Any, Dict, List, Optional, Tuple, Union
import numpy as np
from datetime import datetime, timezone

from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
from ..kg.node_embeddings import NodeEmbedder
from ..kg.similarity_calculator import SimilarityCalculator
from ..kg.path_finder import PathFinder
from ..kg.connectivity_analyzer import ConnectivityAnalyzer
from ..kg.centrality_calculator import CentralityCalculator
from ..kg.community_detector import CommunityDetector
from .hybrid_similarity import HybridSimilarityCalculator


class DecisionEmbeddingPipeline:
    """
    Decision embedding pipeline for generating semantic and structural embeddings.
    
    This class provides comprehensive decision embedding capabilities that
    combine semantic embeddings from text with structural embeddings from
    knowledge graphs, enabling enhanced decision tracking and precedent search.
    
    Features:
        - Semantic embedding generation from decision text
        - Structural embedding generation using Node2Vec from KG module
        - Batch processing for multiple decisions
        - Metadata enrichment and validation
        - Configurable embedding parameters
        - Integration with vector stores and graph stores
    
    Example Usage:
        >>> pipeline = DecisionEmbeddingPipeline(
        ...     vector_store=vector_store,
        ...     graph_store=graph_store,
        ...     auto_embed=True
        ... )
        >>> embeddings = pipeline.process_decision_batch([
        ...     {"scenario": "Credit limit increase", "outcome": "approved"}
        ... ])
    """
    
    def __init__(
        self,
        vector_store: Any,
        graph_store: Optional[Any] = None,
        node_embedder: Optional[NodeEmbedder] = None,
        hybrid_calculator: Optional[HybridSimilarityCalculator] = None,
        auto_embed: bool = True,
        semantic_weight: float = 0.7,
        structural_weight: float = 0.3,
        embedding_dimension: int = 384,
        node_embedding_dimension: int = 128,
        node_labels: Optional[List[str]] = None,
        relationship_types: Optional[List[str]] = None,
        use_graph_features: bool = False
    ):
        """
        Initialize decision embedding pipeline.
        
        Args:
            vector_store: Vector store for semantic embeddings
            graph_store: Graph store for structural embeddings
            node_embedder: Optional pre-configured NodeEmbedder
            hybrid_calculator: Optional pre-configured HybridSimilarityCalculator
            auto_embed: Whether to automatically generate embeddings
            semantic_weight: Weight for semantic similarity
            structural_weight: Weight for structural similarity
            embedding_dimension: Dimension for semantic embeddings
            node_embedding_dimension: Dimension for structural embeddings
            node_labels: Node labels for structural embedding generation
            relationship_types: Relationship types for structural embedding generation
            use_graph_features: Whether to use graph features
        """
        self.vector_store = vector_store
        self.graph_store = graph_store
        self.auto_embed = auto_embed
        self.semantic_weight = semantic_weight
        self.structural_weight = structural_weight
        self.embedding_dimension = embedding_dimension
        self.node_embedding_dimension = node_embedding_dimension
        self.node_labels = node_labels or ["Decision", "Entity", "Policy"]
        self.relationship_types = relationship_types or ["RELATED_TO", "APPLIES_TO", "AFFECTS"]
        self.use_graph_features = use_graph_features
        
        self.logger = get_logger(__name__)
        self.progress_tracker = get_progress_tracker()
        
        # Initialize components
        self.hybrid_calculator = hybrid_calculator or HybridSimilarityCalculator(
            semantic_weight=semantic_weight,
            structural_weight=structural_weight
        )
        
        # Initialize KG algorithm attributes (always set, even if None)
        self.similarity_calculator = None
        self.path_finder = None
        self.connectivity_analyzer = None
        self.centrality_calculator = None
        self.community_detector = None

        # Initialize node embedder if graph store provided
        if graph_store:
            self.node_embedder = node_embedder or NodeEmbedder(
                method="node2vec",
                embedding_dimension=node_embedding_dimension,
                walk_length=80,
                num_walks=10,
                p=1.0,
                q=1.0
            )

            # Initialize advanced KG algorithms if enabled
            if self.use_graph_features:
                self.similarity_calculator = SimilarityCalculator()
                self.path_finder = PathFinder()
                self.connectivity_analyzer = ConnectivityAnalyzer()
                self.centrality_calculator = CentralityCalculator()
                self.community_detector = CommunityDetector()
        else:
            self.node_embedder = None
            self.logger.warning("No graph store provided - structural embeddings disabled")
        
        # Cache for structural embeddings
        self._structural_embeddings_cache: Dict[str, np.ndarray] = {}
        
    def process_decision(
        self,
        decision_data: Dict[str, Any],
        generate_structural: bool = True,
        store_embeddings: bool = True
    ) -> Dict[str, Any]:
        """
        Process a single decision and generate embeddings.
        
        Args:
            decision_data: Decision data with scenario, reasoning, outcome, etc.
            generate_structural: Whether to generate structural embeddings
            store_embeddings: Whether to store embeddings in vector store
            
        Returns:
            Processed decision with embeddings
        """
        # Validate decision data
        decision_data = self._validate_decision_data(decision_data)
        
        # Generate semantic embedding
        semantic_embedding = self._generate_semantic_embedding(decision_data)
        
        # Generate structural embedding if graph store available
        structural_embedding = None
        if generate_structural and self.graph_store and self.node_embedder:
            try:
                structural_embedding = self._generate_structural_embedding(decision_data)
            except (RuntimeError, Exception) as e:
                self.logger.warning(f"Structural embedding skipped: {e}")
        
        # Create combined embedding
        combined_embedding = self._create_combined_embedding(
            semantic_embedding, structural_embedding
        )
        
        # Enrich metadata
        enriched_metadata = self._enrich_metadata(decision_data)
        
        # Store embeddings if requested
        vector_id = None
        if store_embeddings and self.vector_store:
            vector_id = self._store_embeddings(
                decision_data, semantic_embedding, structural_embedding, enriched_metadata
            )
        
        return {
            "decision_data": decision_data,
            "semantic_embedding": semantic_embedding,
            "structural_embedding": structural_embedding,
            "combined_embedding": combined_embedding,
            "metadata": enriched_metadata,
            "vector_id": vector_id,
            "processed_at": datetime.now(timezone.utc).isoformat()
        }
    
    def process_decision_batch(
        self,
        decisions: List[Dict[str, Any]],
        generate_structural: bool = True,
        store_embeddings: bool = True,
        batch_size: int = 32
    ) -> List[Dict[str, Any]]:
        """
        Process multiple decisions in batch.
        
        Args:
            decisions: List of decision data dictionaries
            generate_structural: Whether to generate structural embeddings
            store_embeddings: Whether to store embeddings in vector store
            batch_size: Batch size for processing
            
        Returns:
            List of processed decisions with embeddings
        """
        if not decisions:
            return []
        
        tracking_id = self.progress_tracker.start_tracking(
            module="vector_store",
            submodule="DecisionEmbeddingPipeline",
            message=f"Processing {len(decisions)} decisions (batch_size={batch_size})"
        )
        
        try:
            # Pre-generate structural embeddings if needed
            if generate_structural and self.graph_store and self.node_embedder:
                self.progress_tracker.update_tracking(
                    tracking_id, message="Pre-generating structural embeddings..."
                )
                self._pregenerate_structural_embeddings(decisions)
            
            # Process decisions in batches
            results = []
            for i in range(0, len(decisions), batch_size):
                batch = decisions[i:i + batch_size]
                
                self.progress_tracker.update_tracking(
                    tracking_id,
                    message=f"Processing batch {i//batch_size + 1}/{(len(decisions)-1)//batch_size + 1}"
                )
                
                batch_results = []
                for decision in batch:
                    result = self.process_decision(
                        decision, generate_structural, store_embeddings
                    )
                    batch_results.append(result)
                
                results.extend(batch_results)
            
            self.progress_tracker.stop_tracking(
                tracking_id,
                status="completed",
                message=f"Processed {len(results)} decisions"
            )
            return results
            
        except Exception as e:
            self.progress_tracker.stop_tracking(
                tracking_id, status="failed", message=str(e)
            )
            raise
    
    def find_similar_decisions(
        self,
        query_decision: Dict[str, Any],
        limit: int = 10,
        use_hybrid_search: bool = True,
        semantic_weight: Optional[float] = None,
        structural_weight: Optional[float] = None,
        filters: Optional[Dict[str, Any]] = None
    ) -> List[Dict[str, Any]]:
        """
        Find similar decisions using hybrid search.
        
        Args:
            query_decision: Query decision data
            limit: Number of results to return
            use_hybrid_search: Whether to use hybrid similarity
            semantic_weight: Override semantic weight
            structural_weight: Override structural weight
            filters: Optional metadata filters
            
        Returns:
            List of similar decisions with scores
        """
        # Process query decision
        query_result = self.process_decision(query_decision, store_embeddings=False)
        
        # Get candidate embeddings from vector store
        candidate_embeddings = self._get_candidate_embeddings(filters)
        
        if not candidate_embeddings:
            return []
        
        # Calculate similarities
        if use_hybrid_search and query_result["structural_embedding"] is not None:
            # Use hybrid similarity
            weights = None
            if semantic_weight is not None and structural_weight is not None:
                weights = (semantic_weight, structural_weight)
            
            similarities = self.hybrid_calculator.find_most_similar_decisions(
                query_result["semantic_embedding"],
                query_result["structural_embedding"],
                candidate_embeddings["embeddings"],
                candidate_embeddings["metadata"],
                top_k=limit,
                weights=weights,
                filters=filters
            )
        else:
            # Use semantic similarity only
            similarities = self._find_semantic_similar(
                query_result["semantic_embedding"],
                candidate_embeddings["embeddings"],
                candidate_embeddings["metadata"],
                limit,
                filters
            )
        
        return similarities
    
    def _validate_decision_data(self, decision_data: Dict[str, Any]) -> Dict[str, Any]:
        """Validate and normalize decision data."""
        required_fields = ["scenario"]
        for field in required_fields:
            if field not in decision_data:
                raise ValueError(f"Missing required field: {field}")
        
        # Add defaults for optional fields
        normalized = decision_data.copy()
        if "outcome" not in normalized:
            normalized["outcome"] = "unknown"
        if "reasoning" not in normalized:
            normalized["reasoning"] = ""
        if "confidence" not in normalized:
            normalized["confidence"] = 0.5
        if "category" not in normalized:
            normalized["category"] = "general"
        if "entities" not in normalized:
            normalized["entities"] = []
        if "timestamp" not in normalized:
            normalized["timestamp"] = datetime.now(timezone.utc).isoformat()
        
        return normalized
    
    def _generate_semantic_embedding(self, decision_data: Dict[str, Any]) -> np.ndarray:
        """Generate semantic embedding from decision text."""
        # Combine relevant text fields
        text_parts = [
            decision_data.get("scenario", ""),
            decision_data.get("reasoning", ""),
            decision_data.get("outcome", ""),
            decision_data.get("category", "")
        ]
        text = " ".join(filter(None, text_parts))
        
        if self.vector_store and hasattr(self.vector_store, 'embed'):
            try:
                return self.vector_store.embed(text)
            except Exception as e:
                self.logger.warning(f"Semantic embedding generation failed: {e}. Using fallback.")
                return np.random.rand(self.embedding_dimension).astype(np.float32)
        else:
            return np.random.rand(self.embedding_dimension).astype(np.float32)
    
    def _generate_structural_embedding(self, decision_data: Dict[str, Any]) -> Optional[np.ndarray]:
        """Generate structural embedding using graph context and KG algorithms."""
        if not self.graph_store or not self.node_embedder:
            return None
        
        # Extract entities from decision
        entities = decision_data.get("entities", [])
        if not entities:
            # Use category as fallback
            entities = [decision_data.get("category", "decision")]
        
        # Try to get cached embeddings
        cache_key = "|".join(sorted(entities))
        if cache_key in self._structural_embeddings_cache:
            return self._structural_embeddings_cache[cache_key]
        
        try:
            # Generate base structural embeddings using Node2Vec
            embeddings = self.node_embedder.compute_embeddings(
                graph_store=self.graph_store,
                node_labels=self.node_labels,
                relationship_types=self.relationship_types,
                embedding_dimension=self.node_embedding_dimension
            )
            
            # Use advanced KG algorithms to enhance embeddings if available
            if self.use_graph_features and self.similarity_calculator:
                enhanced_embeddings = self._enhance_with_kg_algorithms(
                    entities, embeddings, decision_data
                )
            else:
                enhanced_embeddings = embeddings
            
            # Aggregate embeddings for decision entities
            entity_embeddings = []
            for entity in entities:
                if entity in enhanced_embeddings:
                    entity_embeddings.append(np.array(enhanced_embeddings[entity]))
            
            if entity_embeddings:
                # Weighted aggregation based on centrality if available
                if self.use_graph_features and self.centrality_calculator:
                    structural_embedding = self._weighted_aggregation(
                        entities, entity_embeddings, enhanced_embeddings
                    )
                else:
                    # Simple average aggregation
                    structural_embedding = np.mean(entity_embeddings, axis=0)
            else:
                # Fail clearly instead of using random embeddings
                raise RuntimeError(
                    "Structural embedding generation failed: no entities found and "
                    "no category provided. Please provide either entities or category "
                    "in the decision data."
                )
            
            # Cache the result
            self._structural_embeddings_cache[cache_key] = structural_embedding
            
            return structural_embedding
            
        except Exception as e:
            self.logger.warning(f"Failed to generate structural embedding: {e}")
            return np.random.rand(self.node_embedding_dimension).astype(np.float32)
    
    def _enhance_with_kg_algorithms(
        self,
        entities: List[str],
        base_embeddings: Dict[str, List[float]],
        decision_data: Dict[str, Any]
    ) -> Dict[str, List[float]]:
        """Enhance base embeddings using advanced KG algorithms."""
        enhanced_embeddings = base_embeddings.copy()
        
        try:
            # Use path-based similarity to enhance embeddings
            if self.path_finder and len(entities) > 1:
                path_similarities = self._calculate_path_similarities(entities)
                for entity in entities:
                    if entity in enhanced_embeddings:
                        # Enhance embedding with path information
                        path_context = self._create_path_context(entity, path_similarities)
                        enhanced_embedding = self._combine_embeddings(
                            enhanced_embeddings[entity], path_context
                        )
                        enhanced_embeddings[entity] = enhanced_embedding.tolist()
            
            # Use community information to enhance embeddings
            if self.community_detector:
                communities = self._get_entity_communities(entities)
                for entity in entities:
                    if entity in enhanced_embeddings and entity in communities:
                        community_context = self._create_community_context(
                            entity, communities[entity], enhanced_embeddings
                        )
                        enhanced_embedding = self._combine_embeddings(
                            enhanced_embeddings[entity], community_context
                        )
                        enhanced_embeddings[entity] = enhanced_embedding.tolist()
            
            # Use connectivity analysis to enhance embeddings
            if self.connectivity_analyzer:
                connectivity_scores = self._calculate_connectivity_scores(entities)
                for entity in entities:
                    if entity in enhanced_embeddings and entity in connectivity_scores:
                        connectivity_factor = connectivity_scores[entity]
                        # Adjust embedding based on connectivity
                        enhanced_embedding = np.array(enhanced_embeddings[entity]) * connectivity_factor
                        enhanced_embeddings[entity] = enhanced_embedding.tolist()
            
        except Exception as e:
            self.logger.warning(f"Failed to enhance embeddings with KG algorithms: {e}")
        
        return enhanced_embeddings
    
    def _calculate_path_similarities(self, entities: List[str]) -> Dict[str, Dict[str, float]]:
        """Calculate path-based similarities between entities."""
        path_similarities = {}
        
        for i, entity1 in enumerate(entities):
            path_similarities[entity1] = {}
            for j, entity2 in enumerate(entities):
                if i != j:
                    try:
                        # Find shortest path between entities
                        path = self.path_finder.find_shortest_path(
                            self.graph_store, entity1, entity2
                        )
                        if path:
                            # Calculate path-based similarity score
                            path_length = len(path)
                            similarity = 1.0 / path_length  # Shorter paths = higher similarity
                            path_similarities[entity1][entity2] = similarity
                    except Exception:
                        path_similarities[entity1][entity2] = 0.0
        
        return path_similarities
    
    def _create_path_context(self, entity: str, path_similarities: Dict[str, Dict[str, float]]) -> np.ndarray:
        """Create path context embedding for an entity."""
        if entity not in path_similarities:
            return np.zeros(self.node_embedding_dimension)
        
        # Aggregate path similarities into context vector
        context_vector = np.zeros(self.node_embedding_dimension)
        total_similarity = 0.0
        
        for other_entity, similarity in path_similarities[entity].items():
            if similarity > 0:
                # Get embedding of similar entity (simplified - would need access to all embeddings)
                context_vector += similarity * np.random.rand(self.node_embedding_dimension)
                total_similarity += similarity
        
        if total_similarity > 0:
            context_vector /= total_similarity
        
        return context_vector
    
    def _get_entity_communities(self, entities: List[str]) -> Dict[str, int]:
        """Get community assignments for entities."""
        try:
            # Detect communities in the graph
            communities = self.community_detector.detect_communities(self.graph_store)
            
            entity_communities = {}
            for entity in entities:
                # Find which community the entity belongs to
                for community_id, community_nodes in communities.items():
                    if entity in community_nodes:
                        entity_communities[entity] = community_id
                        break
            
            return entity_communities
        except Exception:
            return {}
    
    def _create_community_context(
        self,
        entity: str,
        community_id: int,
        embeddings: Dict[str, List[float]]
    ) -> np.ndarray:
        """Create community context embedding for an entity."""
        # Get all entities in the same community (simplified)
        community_embeddings = []
        
        for other_entity, embedding in embeddings.items():
            if other_entity != entity:
                # In practice, would check if other_entity is in same community
                if np.random.random() < 0.3:  # Simplified community membership
                    community_embeddings.append(np.array(embedding))
        
        if community_embeddings:
            return np.mean(community_embeddings, axis=0)
        else:
            return np.zeros(self.node_embedding_dimension)
    
    def _calculate_connectivity_scores(self, entities: List[str]) -> Dict[str, float]:
        """Calculate connectivity scores for entities."""
        connectivity_scores = {}
        
        try:
            for entity in entities:
                # Calculate degree centrality as connectivity measure
                if hasattr(self.graph_store, 'get_neighbors'):
                    neighbors = self.graph_store.get_neighbors(entity)
                    connectivity_scores[entity] = len(neighbors) / 10.0  # Normalize
                else:
                    connectivity_scores[entity] = 1.0
        except Exception:
            # Default connectivity scores
            for entity in entities:
                connectivity_scores[entity] = 1.0
        
        return connectivity_scores
    
    def _weighted_aggregation(
        self,
        entities: List[str],
        entity_embeddings: List[np.ndarray],
        all_embeddings: Dict[str, List[float]]
    ) -> np.ndarray:
        """Aggregate embeddings using centrality-based weights."""
        try:
            # Calculate centrality weights
            weights = []
            for entity in entities:
                if hasattr(self.centrality_calculator, 'calculate_degree_centrality'):
                    # Simplified centrality calculation
                    centrality = len([e for e in all_embeddings.keys() if entity in e]) / len(all_embeddings)
                    weights.append(centrality + 0.1)  # Add small constant to avoid zero weights
                else:
                    weights.append(1.0)
            
            # Normalize weights
            total_weight = sum(weights)
            if total_weight > 0:
                weights = [w / total_weight for w in weights]
            
            # Weighted aggregation
            weighted_embedding = np.zeros(self.node_embedding_dimension)
            for embedding, weight in zip(entity_embeddings, weights):
                weighted_embedding += weight * embedding
            
            return weighted_embedding
            
        except Exception:
            # Fallback to simple average
            return np.mean(entity_embeddings, axis=0)
    
    def _combine_embeddings(self, base_embedding: List[float], context_embedding: np.ndarray) -> np.ndarray:
        """Combine base embedding with context embedding."""
        base = np.array(base_embedding)
        
        # Ensure same dimension
        if len(base) != len(context_embedding):
            if len(base) < len(context_embedding):
                # Pad base embedding
                padded = np.zeros(len(context_embedding))
                padded[:len(base)] = base
                base = padded
            else:
                # Truncate context embedding
                context_embedding = context_embedding[:len(base)]
        
        # Combine with weighted average
        combined = 0.7 * base + 0.3 * context_embedding
        return combined
    
    def _create_combined_embedding(
        self,
        semantic_embedding: np.ndarray,
        structural_embedding: Optional[np.ndarray]
    ) -> np.ndarray:
        """Create combined embedding from semantic and structural components."""
        if structural_embedding is None:
            return semantic_embedding
        
        # Resize embeddings to same dimension if needed
        if len(semantic_embedding) != len(structural_embedding):
            if len(semantic_embedding) < len(structural_embedding):
                # Pad semantic embedding
                padded = np.zeros(len(structural_embedding))
                padded[:len(semantic_embedding)] = semantic_embedding
                semantic_embedding = padded
            else:
                # Pad structural embedding
                padded = np.zeros(len(semantic_embedding))
                padded[:len(structural_embedding)] = structural_embedding
                structural_embedding = padded
        
        # Combine with weighted average
        combined = (
            self.semantic_weight * semantic_embedding +
            self.structural_weight * structural_embedding
        )
        
        return combined
    
    def _enrich_metadata(self, decision_data: Dict[str, Any]) -> Dict[str, Any]:
        """Enrich decision metadata with additional information."""
        metadata = decision_data.copy()
        
        # Add pipeline metadata
        metadata.update({
            "pipeline_version": "1.0",
            "embedding_generated_at": datetime.now(timezone.utc).isoformat(),
            "semantic_weight": self.semantic_weight,
            "structural_weight": self.structural_weight,
            "has_structural_embedding": self.graph_store is not None
        })
        
        return metadata
    
    def _store_embeddings(
        self,
        decision_data: Dict[str, Any],
        semantic_embedding: np.ndarray,
        structural_embedding: Optional[np.ndarray],
        metadata: Dict[str, Any]
    ) -> str:
        """Store embeddings in vector store."""
        if not self.vector_store:
            return None
        
        # Prepare metadata for storage
        storage_metadata = metadata.copy()
        if structural_embedding is not None:
            storage_metadata["structural_embedding"] = structural_embedding.tolist()
        
        # Store semantic embedding
        vector_ids = self.vector_store.store_vectors(
            [semantic_embedding],
            metadata=[storage_metadata]
        )
        
        return vector_ids[0] if vector_ids else None
    
    def _pregenerate_structural_embeddings(self, decisions: List[Dict[str, Any]]) -> None:
        """Pre-generate structural embeddings for all entities in decisions."""
        if not self.graph_store or not self.node_embedder:
            return
        
        # Collect all unique entities
        all_entities = set()
        for decision in decisions:
            entities = decision.get("entities", [])
            all_entities.update(entities)
            # Also add categories
            all_entities.add(decision.get("category", "decision"))
        
        # Generate embeddings for all entities
        try:
            embeddings = self.node_embedder.compute_embeddings(
                graph_store=self.graph_store,
                node_labels=self.node_labels,
                relationship_types=self.relationship_types,
                embedding_dimension=self.node_embedding_dimension
            )
            
            # Cache embeddings
            for entity, embedding in embeddings.items():
                if entity in all_entities:
                    self._structural_embeddings_cache[entity] = np.array(embedding)
            
            self.logger.info(f"Pre-generated structural embeddings for {len(embeddings)} entities")
            
        except Exception as e:
            self.logger.warning(f"Failed to pre-generate structural embeddings: {e}")
    
    def _get_candidate_embeddings(self, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
        """Get candidate embeddings from vector store."""
        if not self.vector_store:
            return {"embeddings": [], "metadata": []}
        
        # Get all vectors from store
        embeddings = []
        metadata = []
        
        for vector_id, vector in self.vector_store.vectors.items():
            vector_metadata = self.vector_store.metadata.get(vector_id, {})
            
            # Apply filters
            if filters:
                match = True
                for key, value in filters.items():
                    if key not in vector_metadata or vector_metadata[key] != value:
                        match = False
                        break
                if not match:
                    continue
            
            # Extract structural embedding if available
            struct_emb = vector_metadata.get("structural_embedding")
            if struct_emb:
                struct_emb = np.array(struct_emb)
            else:
                struct_emb = np.zeros(self.node_embedding_dimension)
            
            embeddings.append((vector, struct_emb))
            metadata.append(vector_metadata)
        
        return {"embeddings": embeddings, "metadata": metadata}
    
    def _find_semantic_similar(
        self,
        query_embedding: np.ndarray,
        candidate_embeddings: List[Tuple[np.ndarray, np.ndarray]],
        candidate_metadata: List[Dict[str, Any]],
        limit: int,
        filters: Optional[Dict[str, Any]] = None
    ) -> List[Dict[str, Any]]:
        """Find similar decisions using semantic similarity only."""
        similarities = []
        
        for i, (sem_emb, struct_emb) in enumerate(candidate_embeddings):
            # Calculate semantic similarity
            similarity = self.hybrid_calculator._calculate_similarity(
                query_embedding, sem_emb, "cosine"
            )
            
            similarities.append({
                "similarity": similarity,
                "semantic_similarity": similarity,
                "structural_similarity": 0.0,
                "metadata": candidate_metadata[i],
                "index": i
            })
        
        # Sort and return top-k
        similarities.sort(key=lambda x: x["similarity"], reverse=True)
        return similarities[:limit]
    
    def update_weights(self, semantic_weight: float, structural_weight: float) -> None:
        """Update similarity weights."""
        self.semantic_weight = semantic_weight
        self.structural_weight = structural_weight
        
        self.hybrid_calculator.update_weights(semantic_weight, structural_weight)
        
        self.logger.info(f"Updated weights: semantic={semantic_weight}, structural={structural_weight}")
    
    def get_statistics(self) -> Dict[str, Any]:
        """Get pipeline statistics."""
        return {
            "total_decisions_processed": len(self._structural_embeddings_cache),
            "semantic_weight": self.semantic_weight,
            "structural_weight": self.structural_weight,
            "embedding_dimension": self.embedding_dimension,
            "node_embedding_dimension": self.node_embedding_dimension,
            "has_graph_store": self.graph_store is not None,
            "cached_structural_embeddings": len(self._structural_embeddings_cache)
        }
