"""
Enhanced evidence-based search strategy for complex query resolution.

This strategy addresses common issues with candidate discovery and evidence gathering:
1. Multi-stage candidate discovery with fallback mechanisms
2. Adaptive query generation based on past performance
3. Cross-constraint search capabilities
4. Source diversity management
"""

import math
from dataclasses import dataclass, field
from datetime import datetime, UTC
from typing import Any, Dict, List, Optional, Set, Tuple

from langchain_core.language_models import BaseChatModel

from ...utilities.search_utilities import remove_think_tags
from ..candidates.base_candidate import Candidate
from ..constraints.base_constraint import Constraint, ConstraintType
from ..evidence.base_evidence import Evidence
from .evidence_based_strategy import EvidenceBasedStrategy


@dataclass
class QueryPattern:
    """Pattern for generating search queries."""

    pattern: str
    success_rate: float = 0.0
    usage_count: int = 0
    constraint_types: List[ConstraintType] = field(default_factory=list)


@dataclass
class SourceProfile:
    """Profile for tracking source effectiveness."""

    source_name: str
    success_rate: float = 0.0
    usage_count: int = 0
    specialties: List[str] = field(default_factory=list)
    last_used: Optional[datetime] = None


class EnhancedEvidenceBasedStrategy(EvidenceBasedStrategy):
    """
    Enhanced evidence-based strategy with improved candidate discovery.

    Key improvements:
    1. Multi-stage candidate discovery
    2. Adaptive query patterns
    3. Cross-constraint capabilities
    4. Source diversity tracking
    """

    def __init__(
        self,
        model: BaseChatModel,
        search: Any,
        all_links_of_system: List[str],
        max_iterations: int = 20,
        confidence_threshold: float = 0.85,
        candidate_limit: int = 20,  # Increased for better candidate variety
        evidence_threshold: float = 0.6,
        max_search_iterations: int = 2,
        questions_per_iteration: int = 3,
        min_candidates_threshold: int = 10,  # Increased to ensure we have enough candidates
        enable_pattern_learning: bool = True,
    ):
        """Initialize the enhanced evidence-based strategy."""
        # Call parent initializer with required arguments
        super().__init__(
            model=model,
            search=search,
            all_links_of_system=all_links_of_system,
            max_iterations=max_iterations,
            confidence_threshold=confidence_threshold,
            candidate_limit=candidate_limit,
            evidence_threshold=evidence_threshold,
            max_search_iterations=max_search_iterations,
            questions_per_iteration=questions_per_iteration,
        )

        # Enhanced-specific attributes
        self.min_candidates_threshold = min_candidates_threshold
        self.enable_pattern_learning = enable_pattern_learning

        # Pattern learning
        self.query_patterns: Dict[str, QueryPattern] = (
            self._initialize_patterns()
        )
        self.source_profiles: Dict[str, SourceProfile] = (
            self._initialize_sources()
        )

        # Failure tracking
        self.failed_queries: Set[str] = set()
        self.constraint_relationships: Dict[str, List[str]] = {}

    def _initialize_patterns(self) -> Dict[str, QueryPattern]:
        """Initialize query patterns for different constraint types."""
        patterns = {
            "property_basic": QueryPattern(
                pattern="{value} {constraint_type}",
                constraint_types=[ConstraintType.PROPERTY],
            ),
            "property_character": QueryPattern(
                pattern="character with {value}",
                constraint_types=[ConstraintType.PROPERTY],
            ),
            "event_show": QueryPattern(
                pattern="TV show {value}",
                constraint_types=[ConstraintType.EVENT],
            ),
            "statistic_episodes": QueryPattern(
                pattern="{value} episodes series",
                constraint_types=[ConstraintType.STATISTIC],
            ),
            "cross_constraint": QueryPattern(
                pattern="{value1} AND {value2}", constraint_types=[]
            ),
            "semantic_expansion": QueryPattern(
                pattern='"{value}" OR "{synonym1}" OR "{synonym2}"',
                constraint_types=[],
            ),
        }
        return patterns

    def _initialize_sources(self) -> Dict[str, SourceProfile]:
        """Initialize source profiles for diversity tracking."""
        sources = {
            "wikipedia": SourceProfile(
                source_name="wikipedia",
                specialties=["characters", "properties", "general"],
            ),
            "imdb": SourceProfile(
                source_name="imdb",
                specialties=["tv_shows", "episodes", "statistics"],
            ),
            "fandom": SourceProfile(
                source_name="fandom",
                specialties=["characters", "properties", "backstory"],
            ),
            "tv_databases": SourceProfile(
                source_name="tv_databases",
                specialties=["shows", "episodes", "statistics"],
            ),
            "web": SourceProfile(source_name="web", specialties=["general"]),
        }
        return sources

    def analyze_topic(self, query: str) -> Dict:
        """Analyze a topic using enhanced evidence-based approach."""
        # Initialize
        self.all_links_of_system.clear()
        self.questions_by_iteration = []
        self.findings = []
        self.iteration = 0

        if self.progress_callback:
            self.progress_callback(
                "Enhanced Evidence-Based Strategy initialized - beginning analysis",
                1,
                {
                    "phase": "initialization",
                    "strategy": "enhanced_evidence_based",
                },
            )

        # Extract constraints
        if self.progress_callback:
            self.progress_callback(
                "Extracting verifiable constraints from query...",
                3,
                {"phase": "constraint_extraction", "query_length": len(query)},
            )

        self.constraints = self.constraint_analyzer.extract_constraints(query)

        if self.progress_callback:
            self.progress_callback(
                f"Found {len(self.constraints)} constraints - analyzing relationships",
                5,
                {
                    "phase": "constraint_analysis",
                    "constraint_count": len(self.constraints),
                    "constraint_types": {
                        c.type.name: c.weight for c in self.constraints[:5]
                    },
                },
            )

        # Analyze constraint relationships
        self._analyze_constraint_relationships()

        # Find initial candidates with enhanced discovery
        if self.progress_callback:
            self.progress_callback(
                "Starting enhanced candidate discovery (multi-stage)",
                8,
                {"phase": "candidate_discovery_start", "stages": 5},
            )

        self._enhanced_candidate_discovery()

        # Main evidence-gathering loop
        while (
            self.iteration < self.max_iterations
            and not self._has_sufficient_answer()
        ):
            self.iteration += 1

            # Progress for iteration
            if self.progress_callback:
                base_progress = 40
                iteration_progress = base_progress + int(
                    (self.iteration / self.max_iterations) * 50
                )

                self.progress_callback(
                    f"Iteration {self.iteration}/{self.max_iterations} - gathering evidence",
                    iteration_progress,
                    {
                        "phase": "iteration_start",
                        "iteration": self.iteration,
                        "max_iterations": self.max_iterations,
                        "candidates_count": len(self.candidates),
                        "top_candidate": (
                            self.candidates[0].name
                            if self.candidates
                            else "None"
                        ),
                    },
                )

            # Adaptive evidence gathering
            self._adaptive_evidence_gathering()

            # Score and prune with pattern learning
            if self.progress_callback:
                self.progress_callback(
                    f"Scoring {len(self.candidates)} candidates based on evidence",
                    None,
                    {
                        "phase": "scoring_start",
                        "candidate_count": len(self.candidates),
                    },
                )

            self._score_and_prune_adaptive()

            # Add iteration finding
            iteration_finding = {
                "phase": f"Iteration {self.iteration}",
                "content": self._format_iteration_summary(),
                "timestamp": self._get_timestamp(),
                "metadata": {
                    "candidates": len(self.candidates),
                    "patterns_used": len(self.query_patterns),
                    "source_diversity": self._calculate_source_diversity(),
                },
            }
            self.findings.append(iteration_finding)

            # Adaptive candidate discovery if needed
            if len(self.candidates) < self.min_candidates_threshold:
                if self.progress_callback:
                    self.progress_callback(
                        f"Too few candidates ({len(self.candidates)}) - searching for more",
                        None,
                        {
                            "phase": "adaptive_discovery",
                            "current_candidates": len(self.candidates),
                            "threshold": self.min_candidates_threshold,
                        },
                    )
                self._adaptive_candidate_discovery()

        # Final verification with source diversity
        if self.progress_callback:
            self.progress_callback(
                "Starting final verification of top candidates",
                90,
                {
                    "phase": "final_verification_start",
                    "top_candidates": [c.name for c in self.candidates[:3]],
                },
            )

        self._enhanced_final_verification()

        # Generate final answer
        if self.progress_callback:
            self.progress_callback(
                "Synthesizing final answer based on evidence",
                95,
                {
                    "phase": "synthesis_start",
                    "candidates_evaluated": len(self.candidates),
                    "evidence_pieces": sum(
                        len(c.evidence) for c in self.candidates
                    ),
                },
            )

        result = self._synthesize_final_answer(query)

        if self.progress_callback:
            top_candidate = self.candidates[0] if self.candidates else None
            self.progress_callback(
                f"Analysis complete - {top_candidate.name if top_candidate else 'No answer found'}",
                100,
                {
                    "phase": "complete",
                    "final_answer": (
                        top_candidate.name
                        if top_candidate
                        else "Unable to determine"
                    ),
                    "confidence": top_candidate.score if top_candidate else 0,
                    "candidates_evaluated": len(self.candidates),
                    "evidence_pieces": sum(
                        len(c.evidence) for c in self.candidates
                    ),
                    "iterations_used": self.iteration,
                },
            )

        return result

    def _enhanced_candidate_discovery(self):
        """Enhanced multi-stage candidate discovery."""
        all_candidates = []
        discovery_stages = [
            ("broad", "Broad discovery", self._broad_discovery_search),
            ("focused", "Focused constraints", self._focused_constraint_search),
            ("cross", "Cross-constraint", self._cross_constraint_search),
            ("semantic", "Semantic expansion", self._semantic_expansion_search),
            ("temporal", "Temporal search", self._temporal_search),
            (
                "character_db",
                "Character databases",
                self._character_database_search,
            ),
            ("fallback", "Fallback search", self._fallback_search),
        ]

        stage_progress_base = 10
        stage_progress_increment = 5

        for i, (stage_name, stage_desc, stage_func) in enumerate(
            discovery_stages
        ):
            if self.progress_callback:
                progress = stage_progress_base + (i * stage_progress_increment)
                self.progress_callback(
                    f"{stage_desc} search - looking for candidates [{i + 1}/5]",
                    progress,
                    {
                        "phase": f"discovery_{stage_name}",
                        "stage": i + 1,
                        "total_stages": 5,
                        "candidates_found": len(all_candidates),
                        "stage_description": stage_desc,
                    },
                )

            stage_candidates = stage_func()
            new_candidates = len(stage_candidates)
            all_candidates.extend(stage_candidates)

            if self.progress_callback and new_candidates > 0:
                self.progress_callback(
                    f"Found {new_candidates} candidates in {stage_desc} stage",
                    None,
                    {
                        "phase": f"discovery_{stage_name}_results",
                        "new_candidates": new_candidates,
                        "total_candidates": len(all_candidates),
                    },
                )

            # Early exit if we have enough candidates
            if len(all_candidates) >= self.min_candidates_threshold * 2:
                if self.progress_callback:
                    self.progress_callback(
                        f"Found sufficient candidates ({len(all_candidates)}) - ending discovery",
                        30,
                        {
                            "phase": "discovery_complete",
                            "total_candidates": len(all_candidates),
                        },
                    )
                break

        # Deduplicate and add candidates
        self._add_unique_candidates(all_candidates)

        # If still not enough candidates, try more aggressive discovery
        if (
            len(self.candidates) < self.min_candidates_threshold
            and all_candidates
        ):
            if self.progress_callback:
                self.progress_callback(
                    f"Only {len(self.candidates)} candidates - trying more aggressive search",
                    None,
                    {"phase": "aggressive_discovery"},
                )
            self._aggressive_supplemental_search()

        if self.progress_callback:
            self.progress_callback(
                f"Candidate discovery complete - {len(self.candidates)} unique candidates found",
                35,
                {
                    "phase": "discovery_complete",
                    "unique_candidates": len(self.candidates),
                    "candidate_names": [c.name for c in self.candidates[:10]],
                },
            )

    def _broad_discovery_search(self) -> List[Candidate]:
        """Initial broad search to discover potential candidates."""
        candidates = []

        # Extended patterns for broader discovery
        extended_patterns = [
            "{value}",
            "character {value}",
            "TV character {value}",
            "fictional character {value}",
            "{value} television",
            "{value} TV show",
            "{value} series",
        ]

        # Apply patterns to constraints
        for constraint in self.constraints[:4]:  # Increased from 3
            for pattern_template in extended_patterns:
                query = pattern_template.replace("{value}", constraint.value)

                if query not in self.failed_queries and len(query) > 3:
                    if self.progress_callback:
                        self.progress_callback(
                            f"Broad search: {query[:50]}...",
                            None,
                            {
                                "phase": "broad_search_query",
                                "query": query,
                                "constraint_type": constraint.type.value,
                            },
                        )

                    results = self._execute_adaptive_search(query)
                    extracted = (
                        self._extract_candidates_with_enhanced_validation(
                            results, query
                        )
                    )
                    candidates.extend(extracted)

                    if not extracted:
                        self.failed_queries.add(query)

        return candidates

    def _focused_constraint_search(self) -> List[Candidate]:
        """Focused search using specific constraint values."""
        candidates = []

        # Get high-weight constraints
        high_weight_constraints = sorted(
            self.constraints, key=lambda c: c.weight, reverse=True
        )[:5]

        for i, constraint in enumerate(high_weight_constraints):
            # Generate adaptive queries based on past performance
            queries = self._generate_adaptive_queries(constraint)

            if self.progress_callback:
                self.progress_callback(
                    f"Focused search: {constraint.description[:40]}... [{i + 1}/{len(high_weight_constraints)}]",
                    None,
                    {
                        "phase": "focused_constraint",
                        "constraint": constraint.description,
                        "constraint_weight": constraint.weight,
                        "constraint_rank": i + 1,
                    },
                )

            for query in queries[:3]:  # Limit queries per constraint
                if query not in self.failed_queries:
                    results = self._execute_adaptive_search(query)
                    extracted = self._extract_candidates_from_results(
                        results, query
                    )

                    if extracted:
                        candidates.extend(extracted)
                        # Learn from successful query
                        self._learn_query_pattern(
                            query, constraint, success=True
                        )
                    else:
                        self.failed_queries.add(query)

        return candidates

    def _cross_constraint_search(self) -> List[Candidate]:
        """Search using combinations of constraints."""
        candidates = []

        # Find related constraint pairs
        constraint_pairs = self._find_related_constraint_pairs()

        if self.progress_callback and constraint_pairs:
            self.progress_callback(
                f"Cross-constraint search - {len(constraint_pairs)} pairs identified",
                None,
                {
                    "phase": "cross_constraint_start",
                    "pair_count": len(constraint_pairs),
                },
            )

        for i, (constraint1, constraint2) in enumerate(
            constraint_pairs[:5]
        ):  # Limit combinations
            query = self._build_cross_constraint_query(constraint1, constraint2)

            if self.progress_callback:
                self.progress_callback(
                    f"Combining: {constraint1.type.value} + {constraint2.type.value}",
                    None,
                    {
                        "phase": "cross_constraint_pair",
                        "pair": i + 1,
                        "constraint1": constraint1.description[:30],
                        "constraint2": constraint2.description[:30],
                    },
                )

            if query not in self.failed_queries:
                results = self._execute_adaptive_search(query)
                extracted = self._extract_candidates_from_results(
                    results, query
                )

                if extracted:
                    candidates.extend(extracted)
                    # Update constraint relationships
                    self._update_constraint_relationships(
                        constraint1, constraint2, True
                    )
                else:
                    self.failed_queries.add(query)

        return candidates

    def _semantic_expansion_search(self) -> List[Candidate]:
        """Search using semantic variations of constraints."""
        candidates = []

        # Use general semantic variations without specific terms
        for constraint in self.constraints[:4]:
            variations = self._generate_semantic_variations(constraint)

            for variation in variations[:3]:  # Limit variations
                if variation not in self.failed_queries:
                    results = self._execute_adaptive_search(variation)
                    extracted = (
                        self._extract_candidates_with_enhanced_validation(
                            results, variation
                        )
                    )

                    if extracted:
                        candidates.extend(extracted)
                    else:
                        self.failed_queries.add(variation)

        return candidates

    def _fallback_search(self) -> List[Candidate]:
        """Fallback search with relaxed constraints."""
        candidates = []

        # Relax constraints by combining fewer requirements
        relaxed_queries = self._generate_relaxed_queries()

        for query in relaxed_queries[:3]:
            if query not in self.failed_queries:
                results = self._execute_adaptive_search(query)
                extracted = self._extract_candidates_from_results(
                    results, query
                )
                candidates.extend(extracted)

        return candidates

    def _adaptive_evidence_gathering(self):
        """Enhanced evidence gathering with source diversity."""
        evidence_gathered = 0
        source_usage = {}
        total_to_gather = (
            min(5, len(self.candidates)) * 2
        )  # Max 2 constraints per candidate

        if self.progress_callback:
            self.progress_callback(
                f"Gathering evidence for top {min(5, len(self.candidates))} candidates",
                None,
                {
                    "phase": "evidence_gathering_start",
                    "candidates_to_process": min(5, len(self.candidates)),
                    "total_evidence_needed": total_to_gather,
                },
            )

        for i, candidate in enumerate(self.candidates[:5]):
            unverified = candidate.get_unverified_constraints(self.constraints)

            if self.progress_callback:
                self.progress_callback(
                    f"Processing {candidate.name} [{i + 1}/{min(5, len(self.candidates))}]",
                    None,
                    {
                        "phase": "candidate_evidence",
                        "candidate": candidate.name,
                        "candidate_rank": i + 1,
                        "unverified_constraints": len(unverified),
                    },
                )

            for j, constraint in enumerate(
                unverified[:2]
            ):  # Limit per candidate
                # Select diverse source
                source = self._select_diverse_source(
                    source_usage, constraint.type
                )

                # Generate evidence query
                query = self._generate_evidence_query(candidate, constraint)

                if self.progress_callback:
                    self.progress_callback(
                        f"Searching {source} for: {query[:40]}...",
                        None,
                        {
                            "phase": "evidence_search",
                            "candidate": candidate.name,
                            "constraint": constraint.description[:50],
                            "source": source,
                            "query": query,
                        },
                    )

                # Execute search with selected source
                results = self._execute_search_with_source(query, source)

                # Extract and evaluate evidence
                evidence = self.evidence_evaluator.extract_evidence(
                    results.get("current_knowledge", ""),
                    candidate.name,
                    constraint,
                )

                candidate.add_evidence(constraint.id, evidence)
                evidence_gathered += 1

                if self.progress_callback:
                    self.progress_callback(
                        f"Evidence found: {evidence.confidence:.0%} confidence",
                        None,
                        {
                            "phase": "evidence_result",
                            "candidate": candidate.name,
                            "constraint": constraint.description[:50],
                            "confidence": evidence.confidence,
                            "evidence_type": evidence.type.value,
                            "progress": f"{evidence_gathered}/{total_to_gather}",
                        },
                    )

                # Update source profile
                self._update_source_profile(source, evidence.confidence)

        if self.progress_callback:
            self.progress_callback(
                f"Evidence gathering complete - {evidence_gathered} pieces collected",
                None,
                {
                    "phase": "evidence_complete",
                    "evidence_count": evidence_gathered,
                    "source_diversity": self._calculate_source_diversity(),
                    "sources_used": len(
                        [s for s in source_usage.values() if s > 0]
                    ),
                },
            )

    def _generate_adaptive_queries(self, constraint: Constraint) -> List[str]:
        """Generate queries based on past performance."""
        queries = []

        # Get successful patterns for this constraint type
        successful_patterns = sorted(
            [
                p
                for p in self.query_patterns.values()
                if constraint.type in p.constraint_types
                and p.success_rate > 0.3
            ],
            key=lambda p: p.success_rate,
            reverse=True,
        )

        # Apply patterns
        for pattern in successful_patterns[:3]:
            query = self._apply_pattern_to_constraint(pattern, constraint)
            queries.append(query)

        # Add semantic variations
        semantic_queries = self._generate_semantic_variations(constraint)
        queries.extend(semantic_queries[:2])

        # Add fallback basic query
        queries.append(f"{constraint.value} {constraint.type.value}")

        return queries

    def _apply_pattern_to_constraint(
        self, pattern: QueryPattern, constraint: Constraint
    ) -> str:
        """Apply a pattern to a constraint to create a query."""
        query = pattern.pattern

        # Replace placeholders
        query = query.replace("{value}", constraint.value)
        query = query.replace("{constraint_type}", constraint.type.value)

        # Add synonyms if pattern requires them
        if "{synonym" in query:
            synonyms = self._get_synonyms(constraint.value)
            for i, synonym in enumerate(synonyms[:2], 1):
                query = query.replace(f"{{synonym{i}}}", synonym)

        return query

    def _build_cross_constraint_query(
        self, constraint1: Constraint, constraint2: Constraint
    ) -> str:
        """Build query combining multiple constraints."""
        # Identify common terms
        common_terms = self._find_common_terms(constraint1, constraint2)

        if common_terms:
            base = " ".join(common_terms)
            query = f"{base} {constraint1.value} {constraint2.value}"
        else:
            query = f"{constraint1.value} AND {constraint2.value}"

        # Add type-specific context
        if (
            constraint1.type == ConstraintType.PROPERTY
            and constraint2.type == ConstraintType.EVENT
        ):
            query += " TV show character"

        return query

    def _select_diverse_source(
        self, source_usage: Dict[str, int], constraint_type: ConstraintType
    ) -> str:
        """Select source to ensure diversity."""
        # Get specialized sources for constraint type
        specialized = self._get_specialized_sources(constraint_type)

        # Sort by usage (least used first) and success rate
        available_sources = sorted(
            self.source_profiles.values(),
            key=lambda s: (source_usage.get(s.source_name, 0), -s.success_rate),
        )

        # Prefer specialized sources
        for source in available_sources:
            if source.source_name in specialized:
                source_usage[source.source_name] = (
                    source_usage.get(source.source_name, 0) + 1
                )
                return source.source_name

        # Fallback to least used source
        selected = available_sources[0].source_name
        source_usage[selected] = source_usage.get(selected, 0) + 1
        return selected

    def _get_specialized_sources(
        self, constraint_type: ConstraintType
    ) -> List[str]:
        """Get specialized sources for constraint type."""
        specialization_map = {
            ConstraintType.PROPERTY: ["fandom", "wikipedia"],
            ConstraintType.EVENT: ["imdb", "tv_databases"],
            ConstraintType.STATISTIC: ["imdb", "tv_databases"],
            ConstraintType.LOCATION: ["wikipedia"],
        }
        return specialization_map.get(constraint_type, ["web"])

    def _update_pattern_success(self, pattern: QueryPattern, success: bool):
        """Update pattern success rate."""
        pattern.usage_count += 1
        if success:
            # Exponential moving average
            alpha = 0.3
            pattern.success_rate = (
                alpha * 1.0 + (1 - alpha) * pattern.success_rate
            )
        else:
            pattern.success_rate = (1 - 0.3) * pattern.success_rate

    def _update_source_profile(self, source_name: str, confidence: float):
        """Update source profile based on evidence quality."""
        if source_name in self.source_profiles:
            profile = self.source_profiles[source_name]
            profile.usage_count += 1
            profile.last_used = datetime.now(UTC)

            # Update success rate based on confidence
            alpha = 0.3
            success = 1.0 if confidence >= self.evidence_threshold else 0.0
            profile.success_rate = (
                alpha * success + (1 - alpha) * profile.success_rate
            )

    def _analyze_constraint_relationships(self):
        """Analyze relationships between constraints."""
        for i, constraint1 in enumerate(self.constraints):
            for constraint2 in self.constraints[i + 1 :]:
                # Check for common terms or semantic similarity
                common_terms = self._find_common_terms(constraint1, constraint2)
                if common_terms:
                    self.constraint_relationships[constraint1.id] = (
                        self.constraint_relationships.get(constraint1.id, [])
                        + [constraint2.id]
                    )
                    self.constraint_relationships[constraint2.id] = (
                        self.constraint_relationships.get(constraint2.id, [])
                        + [constraint1.id]
                    )

    def _find_related_constraint_pairs(
        self,
    ) -> List[Tuple[Constraint, Constraint]]:
        """Find constraint pairs that might work well together."""
        pairs = []

        # Use analyzed relationships
        for constraint_id, related_ids in self.constraint_relationships.items():
            constraint = next(
                (c for c in self.constraints if c.id == constraint_id), None
            )
            if constraint:
                for related_id in related_ids:
                    related = next(
                        (c for c in self.constraints if c.id == related_id),
                        None,
                    )
                    if related and (constraint, related) not in pairs:
                        pairs.append((constraint, related))

        # Add type-based pairs
        property_constraints = [
            c for c in self.constraints if c.type == ConstraintType.PROPERTY
        ]
        event_constraints = [
            c for c in self.constraints if c.type == ConstraintType.EVENT
        ]

        for prop in property_constraints[:2]:
            for event in event_constraints[:2]:
                if (prop, event) not in pairs:
                    pairs.append((prop, event))

        return pairs

    def _calculate_source_diversity(self) -> float:
        """Calculate source diversity score."""
        if not self.source_profiles:
            return 0.0

        used_sources = [
            s for s in self.source_profiles.values() if s.usage_count > 0
        ]
        if not used_sources:
            return 0.0

        # Calculate entropy of source usage
        total_usage = sum(s.usage_count for s in used_sources)
        entropy = 0.0

        for source in used_sources:
            if source.usage_count > 0:
                p = source.usage_count / total_usage
                entropy -= p * (math.log2(p) if p > 0 else 0)

        # Normalize by maximum possible entropy
        max_entropy = -math.log2(1 / len(used_sources))
        return entropy / max_entropy if max_entropy > 0 else 0.0

    def _execute_adaptive_search(self, query: str) -> Dict:
        """Execute search with adaptive source selection."""
        # Simple wrapper for consistency - could be enhanced with source selection
        return self._execute_search(query)

    def _execute_search_with_source(self, query: str, source: str) -> Dict:
        """Execute search with specific source."""
        # For now, use the standard search - could be enhanced to use specific sources
        return self._execute_search(query)

    def _get_synonyms(self, term: str) -> List[str]:
        """Get synonyms for a term - generic implementation."""
        # Use LLM to generate synonyms dynamically
        try:
            prompt = f"List 3 synonyms or similar terms for '{term}'. Return only the synonyms, one per line."
            response = self.model.invoke(prompt)
            content = remove_think_tags(response.content)
            synonyms = [
                line.strip() for line in content.split("\n") if line.strip()
            ]
            return synonyms[:3]
        except:
            return []  # Return empty list on error

    def _find_common_terms(
        self, constraint1: Constraint, constraint2: Constraint
    ) -> List[str]:
        """Find common terms between constraints."""
        terms1 = set(constraint1.value.lower().split())
        terms2 = set(constraint2.value.lower().split())
        return list(terms1.intersection(terms2))

    def _generate_semantic_variations(
        self, constraint: Constraint
    ) -> List[str]:
        """Generate semantically similar queries"""
        variations = []

        # Basic variations based on constraint type
        base_value = (
            constraint.value
            if hasattr(constraint, "value")
            else constraint.description
        )

        # Add contextual variations based on type
        if constraint.type == ConstraintType.PROPERTY:
            variations.extend(
                [
                    f"entity with {base_value}",
                    f"subject {base_value}",
                    f"thing {base_value}",
                    f"{base_value} entity",
                ]
            )
        elif constraint.type == ConstraintType.EVENT:
            variations.extend(
                [
                    f"event {base_value}",
                    f"occurrence {base_value}",
                    f"{base_value} happened",
                    f"when {base_value}",
                ]
            )
        elif constraint.type == ConstraintType.STATISTIC:
            variations.extend(
                [
                    f"number {base_value}",
                    f"count {base_value}",
                    f"{base_value} total",
                    f"quantity {base_value}",
                ]
            )
        else:
            # Generic variations
            variations.extend(
                [
                    f"{base_value}",
                    f"specific {base_value}",
                    f"about {base_value}",
                    f"regarding {base_value}",
                ]
            )

        return variations[:5]  # Limit variations

    def _generate_evidence_query(
        self, candidate: Candidate, constraint: Constraint
    ) -> str:
        """Generate evidence query for a candidate-constraint pair."""
        prompt = f"""Create a search query to verify if "{candidate.name}" satisfies this constraint:
Constraint: {constraint.description}
Type: {constraint.type.value}

Create a specific search query that would find evidence about whether this candidate meets this constraint.
Return only the search query, no explanation."""

        response = self.model.invoke(prompt)
        query = remove_think_tags(response.content).strip()

        # Fallback to simple query if needed
        if not query or len(query) < 5:
            query = f"{candidate.name} {constraint.value}"

        return query

    def _add_unique_candidates(self, candidates: List[Candidate]):
        """Add unique candidates to the main list."""
        existing_names = {c.name.lower() for c in self.candidates}

        for candidate in candidates:
            if candidate.name.lower() not in existing_names:
                self.candidates.append(candidate)
                existing_names.add(candidate.name.lower())

        # Limit total candidates
        self.candidates = self.candidates[: self.candidate_limit]

    def _adaptive_candidate_discovery(self):
        """Adaptive candidate discovery when we have too few candidates."""
        # Use unused constraints or different strategies
        unused_constraints = [
            c
            for c in self.constraints
            if not any(c.id in cand.evidence for cand in self.candidates)
        ]

        if unused_constraints:
            # Try cross-constraint search with unused constraints
            candidates = self._cross_constraint_search()
            self._add_unique_candidates(candidates)

        # If still not enough, try semantic expansion
        if len(self.candidates) < self.min_candidates_threshold:
            candidates = self._semantic_expansion_search()
            self._add_unique_candidates(candidates)

    def _generate_relaxed_queries(self) -> List[str]:
        """Generate relaxed queries for fallback search."""
        queries = []

        # Take most important constraints and relax them
        important_constraints = sorted(
            self.constraints, key=lambda c: c.weight, reverse=True
        )[:3]

        for constraint in important_constraints:
            # Simple value-based query
            queries.append(constraint.value)

            # Type-based query
            queries.append(f"{constraint.type.value} {constraint.value}")

        return queries

    def _learn_query_pattern(
        self, query: str, constraint: Constraint, success: bool
    ):
        """Learn from query success/failure."""
        if not self.enable_pattern_learning:
            return

        # Extract pattern from query
        # This is a simplified version - could be enhanced with ML
        pattern_key = f"{constraint.type.value}_custom"

        if pattern_key not in self.query_patterns:
            self.query_patterns[pattern_key] = QueryPattern(
                pattern=query, constraint_types=[constraint.type]
            )

        self._update_pattern_success(self.query_patterns[pattern_key], success)

    def _update_constraint_relationships(
        self, constraint1: Constraint, constraint2: Constraint, success: bool
    ):
        """Update constraint relationship tracking."""
        if success:
            # Strengthen the relationship
            if constraint1.id not in self.constraint_relationships:
                self.constraint_relationships[constraint1.id] = []
            if (
                constraint2.id
                not in self.constraint_relationships[constraint1.id]
            ):
                self.constraint_relationships[constraint1.id].append(
                    constraint2.id
                )

            if constraint2.id not in self.constraint_relationships:
                self.constraint_relationships[constraint2.id] = []
            if (
                constraint1.id
                not in self.constraint_relationships[constraint2.id]
            ):
                self.constraint_relationships[constraint2.id].append(
                    constraint1.id
                )

    def _wikipedia_search(self) -> List[Candidate]:
        """Search Wikipedia for general information."""
        candidates = []

        # Generic Wikipedia searches based on constraints
        for constraint in self.constraints[:3]:
            queries = [
                f"Wikipedia {constraint.value}",
                f"Wikipedia list {constraint.value}",
                f"Wikipedia {constraint.type.value} {constraint.value}",
            ]

            for query in queries:
                if query not in self.failed_queries:
                    results = self._execute_adaptive_search(query)
                    extracted = (
                        self._extract_candidates_with_enhanced_validation(
                            results, query
                        )
                    )
                    candidates.extend(extracted)

                    if not extracted:
                        self.failed_queries.add(query)

        return candidates

    def _domain_specific_search(self) -> List[Candidate]:
        """Search domain-specific sources based on constraint types."""
        candidates = []

        # Map constraint types to relevant domains
        domain_map = {
            ConstraintType.PROPERTY: ["database", "encyclopedia", "wiki"],
            ConstraintType.EVENT: ["timeline", "history", "archive"],
            ConstraintType.STATISTIC: ["data", "statistics", "numbers"],
            ConstraintType.LOCATION: ["geography", "places", "maps"],
        }

        for constraint in self.constraints[:3]:
            domains = domain_map.get(constraint.type, ["general"])

            for domain in domains:
                query = f"{domain} {constraint.value}"

                if query not in self.failed_queries:
                    results = self._execute_adaptive_search(query)
                    extracted = (
                        self._extract_candidates_with_enhanced_validation(
                            results, query
                        )
                    )
                    candidates.extend(extracted)

                    if not extracted:
                        self.failed_queries.add(query)

        return candidates

    def _aggressive_supplemental_search(self):
        """Aggressive supplemental search when we need more candidates."""
        additional_candidates = []

        # Try different query formulations
        query_templates = [
            "list of {value}",
            "examples of {value}",
            "types of {value}",
            "{value} instances",
            "specific {value}",
            "named {value}",
        ]

        for constraint in self.constraints:
            for template in query_templates:
                query = template.replace("{value}", constraint.value)

                if query not in self.failed_queries:
                    results = self._execute_adaptive_search(query)
                    extracted = (
                        self._extract_candidates_with_enhanced_validation(
                            results, query
                        )
                    )
                    additional_candidates.extend(extracted)

                    if len(additional_candidates) > 50:  # Enough candidates
                        break

            if len(additional_candidates) > 50:
                break

        self._add_unique_candidates(additional_candidates)

    def _extract_candidates_with_enhanced_validation(
        self, results: Dict, query: str
    ) -> List[Candidate]:
        """Extract candidates with better validation."""
        candidates = self._extract_candidates_from_results(results, query)

        # Additional validation to filter out non-candidates
        validated = []
        for candidate in candidates:
            # Basic validation - not too short, not a common word
            if len(candidate.name) > 2 and not self._is_common_word(
                candidate.name
            ):
                validated.append(candidate)

        return validated[:15]  # Limit per search

    def _is_common_word(self, name: str) -> bool:
        """Check if a name is likely a common word rather than a proper name."""
        common_words = {
            "the",
            "and",
            "or",
            "but",
            "in",
            "on",
            "at",
            "to",
            "for",
            "of",
            "with",
            "by",
            "from",
            "up",
            "about",
            "into",
            "it",
            "is",
            "was",
            "are",
        }
        return name.lower() in common_words

    def _score_and_prune_adaptive(self):
        """Score and prune candidates with adaptive thresholds."""
        old_count = len(self.candidates)

        # Use the parent's scoring method
        super()._score_and_prune_candidates()

        # Additional adaptive pruning based on iteration
        if self.iteration > self.max_iterations / 2:
            # Be more aggressive in later iterations
            min_score = (
                max(0.3, self.candidates[0].score * 0.4)
                if self.candidates
                else 0.3
            )
            self.candidates = [
                c for c in self.candidates if c.score >= min_score
            ]

        if self.progress_callback:
            pruned = old_count - len(self.candidates)
            self.progress_callback(
                f"Scored candidates - kept {len(self.candidates)}, pruned {pruned}",
                None,
                {
                    "phase": "scoring_complete",
                    "candidates_kept": len(self.candidates),
                    "candidates_pruned": pruned,
                    "top_score": self.candidates[0].score
                    if self.candidates
                    else 0,
                    "min_score_threshold": (
                        min_score
                        if self.iteration > self.max_iterations / 2
                        else "adaptive"
                    ),
                },
            )

    def _enhanced_final_verification(self):
        """Enhanced final verification with source diversity."""
        # Use parent's verification as base
        super()._final_verification()

        # Additional verification with unused sources
        if self.candidates:
            top_candidate = self.candidates[0]

            # Find least-used sources
            unused_sources = [
                s for s in self.source_profiles.values() if s.usage_count < 2
            ]

            for source in unused_sources[:2]:
                weak_constraints = [
                    c
                    for c in self.constraints
                    if c.id not in top_candidate.evidence
                    or top_candidate.evidence[c.id].confidence
                    < self.evidence_threshold
                ]

                if weak_constraints:
                    constraint = weak_constraints[0]
                    query = self._generate_evidence_query(
                        top_candidate, constraint
                    )

                    # Use specific source
                    results = self._execute_search_with_source(
                        query, source.source_name
                    )
                    evidence = self.evidence_evaluator.extract_evidence(
                        results.get("current_knowledge", ""),
                        top_candidate.name,
                        constraint,
                    )

                    if evidence.confidence > (
                        top_candidate.evidence.get(
                            constraint.id, Evidence(claim="", confidence=0)
                        ).confidence
                    ):
                        top_candidate.add_evidence(constraint.id, evidence)

            # Final re-scoring
            self._score_and_prune_adaptive()
