"""
Constrained search strategy that progressively narrows down candidates.

This strategy mimics human problem-solving by:
1. Starting with the most restrictive constraints
2. Finding candidates that match those constraints
3. Progressively checking additional constraints
4. Narrowing down the candidate pool step by step
"""

from datetime import datetime, UTC
from typing import Any, Dict, List

from langchain_core.language_models import BaseChatModel
from loguru import logger

from ...utilities.search_utilities import remove_think_tags
from ..candidates.base_candidate import Candidate
from ..constraints.base_constraint import Constraint, ConstraintType
from ..evidence.base_evidence import Evidence, EvidenceType
from .evidence_based_strategy import EvidenceBasedStrategy


class ConstrainedSearchStrategy(EvidenceBasedStrategy):
    """
    Strategy that progressively narrows down candidates using constraints.

    Key approach:
    1. Rank constraints by restrictiveness
    2. Start with most restrictive constraint
    3. Find candidates matching that constraint
    4. Progressively filter using additional constraints
    5. Gather evidence only for promising candidates
    """

    def __init__(
        self,
        model: BaseChatModel,
        search: Any,
        all_links_of_system: List[str],
        max_iterations: int = 20,
        confidence_threshold: float = 0.85,
        candidate_limit: int = 100,  # Increased to get more candidates
        evidence_threshold: float = 0.6,
        max_search_iterations: int = 2,
        questions_per_iteration: int = 3,
        min_candidates_per_stage: int = 20,  # Need more candidates before filtering
        settings_snapshot=None,
    ):
        """Initialize the constrained search strategy."""
        super().__init__(
            model=model,
            search=search,
            all_links_of_system=all_links_of_system,
            max_iterations=max_iterations,
            confidence_threshold=confidence_threshold,
            candidate_limit=candidate_limit,
            evidence_threshold=evidence_threshold,
            max_search_iterations=max_search_iterations,
            questions_per_iteration=questions_per_iteration,
            settings_snapshot=settings_snapshot,
        )

        self.min_candidates_per_stage = min_candidates_per_stage
        self.constraint_ranking: List[Constraint] = []
        self.stage_candidates: Dict[int, List[Candidate]] = {}

        # Enable direct search optimization for entity identification
        # Note: parent class already sets this, but we ensure it's True
        self.use_direct_search = True
        logger.info(
            f"ConstrainedSearchStrategy init: use_direct_search={self.use_direct_search}"
        )

    def analyze_topic(self, query: str) -> Dict:
        """Analyze topic using progressive constraint narrowing."""
        # Initialize
        self.all_links_of_system.clear()
        self.questions_by_iteration = []
        self.findings = []
        self.iteration = 0

        if self.progress_callback:
            self.progress_callback(
                "Analyzing query to identify constraints and rank by restrictiveness",
                2,
                {"phase": "initialization", "strategy": "constrained_search"},
            )

        # Extract and rank constraints
        self.constraints = self.constraint_analyzer.extract_constraints(query)
        self.constraint_ranking = self._rank_constraints_by_restrictiveness()

        if self.progress_callback:
            ranking_summary = ", ".join(
                [
                    f"{i + 1}. {c.description[:30]}..."
                    for i, c in enumerate(self.constraint_ranking[:3])
                ]
            )
            self.progress_callback(
                f"Found {len(self.constraints)} constraints. Order: {ranking_summary}",
                5,
                {
                    "phase": "constraint_ranking",
                    "constraint_count": len(self.constraints),
                    "ranking": [
                        (c.description, c.type.value)
                        for c in self.constraint_ranking
                    ],
                },
            )

        # Add initial analysis finding
        initial_finding = {
            "phase": "Constraint Analysis",
            "content": self._format_constraint_analysis(),
            "timestamp": self._get_timestamp(),
        }
        self.findings.append(initial_finding)

        # Progressive constraint search
        self._progressive_constraint_search()

        # Add search summary finding
        search_finding = {
            "phase": "Progressive Search Summary",
            "content": self._format_search_summary(),
            "timestamp": self._get_timestamp(),
        }
        self.findings.append(search_finding)

        # Evidence gathering for final candidates
        self._focused_evidence_gathering()

        # Add evidence summary finding
        evidence_finding = {
            "phase": "Evidence Summary",
            "content": self._format_evidence_summary(),
            "timestamp": self._get_timestamp(),
        }
        self.findings.append(evidence_finding)

        # Add comprehensive debug summary
        debug_finding = {
            "phase": "Debug Summary",
            "content": self._format_debug_summary(),
            "timestamp": self._get_timestamp(),
            "metadata": {
                "total_searches": (
                    len(self.search_history)
                    if hasattr(self, "search_history")
                    else 0
                ),
                "final_candidates": len(self.candidates),
                "constraints_used": len(self.constraints),
                "stages_completed": len(self.stage_candidates),
            },
        }
        self.findings.append(debug_finding)

        # Final synthesis
        return self._synthesize_final_answer(query)

    def _rank_constraints_by_restrictiveness(self) -> List[Constraint]:
        """Rank constraints from most to least restrictive."""
        # Scoring system for restrictiveness
        restrictiveness_scores = []

        for constraint in self.constraints:
            score = 0

            # Type-based scoring
            if constraint.type == ConstraintType.STATISTIC:
                score += 10  # Numbers are usually very restrictive
            elif constraint.type == ConstraintType.EVENT:
                score += 8  # Events/time periods are restrictive
            elif constraint.type == ConstraintType.LOCATION:
                score += 6  # Locations are moderately restrictive
            elif constraint.type == ConstraintType.PROPERTY:
                score += 4  # Properties are less restrictive

            # Specificity scoring
            if constraint.value:
                # Check for specific markers
                if any(char.isdigit() for char in constraint.value):
                    score += 5  # Contains numbers
                if len(constraint.value.split()) > 3:
                    score += 3  # Longer, more specific
                if any(
                    term in constraint.value.lower()
                    for term in ["specific", "exact", "only", "must"]
                ):
                    score += 2  # Explicit specificity

            restrictiveness_scores.append((constraint, score))

        # Sort by score (highest first)
        ranked = sorted(
            restrictiveness_scores, key=lambda x: x[1], reverse=True
        )
        return [constraint for constraint, _ in ranked]

    def _progressive_constraint_search(self):
        """Progressively search using constraints from most to least restrictive."""
        current_candidates = []

        for stage, constraint in enumerate(self.constraint_ranking):
            self.current_stage = stage  # Track current stage for logging
            if self.progress_callback:
                stage_desc = f"[{constraint.type.value}] {constraint.value} ({len(current_candidates)} candidates)"
                self.progress_callback(
                    f"Stage {stage + 1}/{len(self.constraint_ranking)}: {stage_desc}",
                    10 + (stage * 15),
                    {
                        "phase": "progressive_search",
                        "stage": stage + 1,
                        "total_stages": len(self.constraint_ranking),
                        "constraint": constraint.description,
                        "constraint_type": constraint.type.value,
                        "constraint_value": constraint.value,
                        "current_candidates": len(current_candidates),
                        "search_intent": f"Finding entities matching: {constraint.value}",
                    },
                )

            if stage == 0:
                # First stage - find initial candidates
                current_candidates = self._search_with_single_constraint(
                    constraint
                )
            else:
                # Subsequent stages - filter existing candidates
                current_candidates = self._filter_candidates_with_constraint(
                    current_candidates, constraint
                )

            # Store stage results
            self.stage_candidates[stage] = current_candidates.copy()

            if self.progress_callback:
                candidate_names = ", ".join(
                    [c.name for c in current_candidates[:3]]
                )
                more = (
                    f" (+{len(current_candidates) - 3})"
                    if len(current_candidates) > 3
                    else ""
                )
                change = len(current_candidates) - len(
                    self.stage_candidates.get(stage - 1, [])
                )
                change_str = f" (Δ{change:+d})" if stage > 0 else ""

                self.progress_callback(
                    f"Stage {stage + 1} complete: {len(current_candidates)} candidates{change_str}. {candidate_names}{more}",
                    None,
                    {
                        "phase": "stage_complete",
                        "stage": stage + 1,
                        "candidates_found": len(current_candidates),
                        "candidates_delta": change if stage > 0 else 0,
                        "sample": [c.name for c in current_candidates[:10]],
                    },
                )

            # Add stage finding
            stage_finding = {
                "phase": f"Stage {stage + 1} - {constraint.type.value}",
                "content": self._format_stage_results(
                    stage, constraint, current_candidates
                ),
                "timestamp": self._get_timestamp(),
            }
            self.findings.append(stage_finding)

            # Continue applying constraints unless we have very few candidates
            if len(current_candidates) <= 3:
                if self.progress_callback:
                    self.progress_callback(
                        f"Too few candidates ({len(current_candidates)}) - stopping constraint application",
                        None,
                        {
                            "phase": "early_stop",
                            "candidates_remaining": len(current_candidates),
                        },
                    )
                break

            # Stop if no candidates remain
            if not current_candidates:
                # Backtrack to previous stage if possible
                if stage > 0:
                    current_candidates = self.stage_candidates[stage - 1]
                break

        self.candidates = current_candidates[: self.candidate_limit]

    def _search_with_single_constraint(
        self, constraint: Constraint
    ) -> List[Candidate]:
        """Search for candidates using a single constraint."""
        candidates = []

        # Generate targeted queries for this constraint
        queries = self._generate_constraint_specific_queries(constraint)

        # Add more diverse query patterns
        additional_queries = self._generate_additional_queries(constraint)
        queries.extend(additional_queries)

        # Diversify query execution order
        import random

        # Security: random.shuffle for query order diversity, not security-sensitive
        random.shuffle(queries)

        for i, query in enumerate(queries[:20]):  # Increased query limit
            if self.progress_callback:
                # Show query and what we're looking for
                self.progress_callback(
                    f"Q{i + 1}/{min(20, len(queries))}: '{query}' | Found: {len(candidates)} candidates",
                    None,
                    {
                        "phase": "constraint_search",
                        "query": query,
                        "query_index": i + 1,
                        "total_queries": min(20, len(queries)),
                        "constraint_type": constraint.type.value,
                        "constraint_value": constraint.value,
                        "candidates_so_far": len(candidates),
                        "search_context": f"Stage {getattr(self, 'current_stage', 0) + 1}: {constraint.value}",
                    },
                )

            results = self._execute_search(query)

            # Validate search results before extraction
            if self._validate_search_results(results, constraint):
                extracted = self._extract_relevant_candidates(
                    results, constraint
                )
                candidates.extend(extracted)

                # Track stage information in search history
                if self.search_history:
                    self.search_history[-1]["stage"] = getattr(
                        self, "current_stage", 0
                    )
                    self.search_history[-1]["results_count"] = len(extracted)
                    self.search_history[-1]["results_preview"] = results.get(
                        "current_knowledge", ""
                    )[:200]
            else:
                logger.info(f"Skipping invalid results for query: {query}")

            # Continue searching to build a comprehensive list
            # Don't stop too early - we want diversity
            if len(candidates) >= self.candidate_limit * 2:
                break

        return self._deduplicate_candidates(candidates)

    def _generate_additional_queries(self, constraint: Constraint) -> List[str]:
        """Generate additional diverse queries for better coverage."""
        queries = []
        base_value = constraint.value

        # Add reference source queries
        queries.extend(
            [
                f"reference {base_value}",
                f"authoritative {base_value}",
                f"official {base_value}",
            ]
        )

        # Add structured data queries
        if constraint.type == ConstraintType.STATISTIC:
            queries.extend(
                [
                    f"spreadsheet {base_value}",
                    f"dataset {base_value}",
                    f"statistical analysis {base_value}",
                    f"quantitative {base_value}",
                ]
            )
        elif constraint.type == ConstraintType.PROPERTY:
            queries.extend(
                [
                    f"characterized by {base_value}",
                    f"known for {base_value}",
                    f"featuring {base_value}",
                ]
            )
        else:
            # Generic comprehensive queries
            queries.extend(
                [
                    f"exhaustive {base_value}",
                    f"thorough {base_value}",
                    f"detailed {base_value}",
                ]
            )

        return queries

    def _generate_constraint_specific_queries(
        self, constraint: Constraint
    ) -> List[str]:
        """Generate queries specific to a constraint type."""
        queries = []
        base_value = constraint.value

        # Add context from other constraints for more targeted searches
        context_parts = []
        if hasattr(self, "constraints") and self.constraints:
            for other_constraint in self.constraints[
                :2
            ]:  # Use top 2 constraints for context
                if other_constraint.id != constraint.id:
                    context_parts.append(other_constraint.value)

        # Base queries using the constraint description
        if hasattr(constraint, "description") and constraint.description:
            queries.append(constraint.description)
            if context_parts:
                queries.append(f"{constraint.description} {context_parts[0]}")

        # Type-specific patterns
        if constraint.type == ConstraintType.STATISTIC:
            # Numeric constraints - look for quantitative information
            queries.extend(
                [
                    f"list {base_value}",
                    f"complete {base_value}",
                    f"all {base_value}",
                    f"comprehensive {base_value}",
                    f"database {base_value}",
                    f"statistics {base_value}",
                    f"data {base_value}",
                    f"comparison {base_value}",
                ]
            )

        elif constraint.type == ConstraintType.EVENT or (
            hasattr(constraint.type, "value")
            and constraint.type.value == "temporal"
        ):
            # Time-based constraints
            queries.extend(
                [
                    f"during {base_value}",
                    f"in {base_value}",
                    f"list {base_value}",
                    f"comprehensive {base_value}",
                    f"all from {base_value}",
                    f"complete list {base_value}",
                    f"history {base_value}",
                    f"timeline {base_value}",
                ]
            )

        elif constraint.type == ConstraintType.PROPERTY:
            # Property constraints - characteristics and attributes
            queries.extend(
                [
                    f"with {base_value}",
                    f"having {base_value}",
                    f"characterized by {base_value}",
                    f"examples {base_value}",
                    f"instances {base_value}",
                    f"who {base_value}",
                    f"which {base_value}",
                    f"known for {base_value}",
                ]
            )
        else:
            # Generic queries
            queries.extend(
                [
                    f"{base_value}",
                    f"list {base_value}",
                    f"examples {base_value}",
                    f"all {base_value}",
                    f"complete {base_value}",
                ]
            )

        # Add combined queries with other constraints
        if context_parts:
            queries.extend(
                [
                    f"{base_value} {context_parts[0]}",
                    f"list {base_value} with {context_parts[0]}",
                    f"{base_value} and {context_parts[0]}",
                ]
            )

        return queries

    def _filter_candidates_with_constraint(
        self, candidates: List[Candidate], constraint: Constraint
    ) -> List[Candidate]:
        """Filter existing candidates using an additional constraint."""
        filtered = []

        for candidate in candidates:
            # Check if candidate matches the constraint
            query = f"{candidate.name} {constraint.value}"

            results = self._execute_search(query)

            # Quick evidence check
            evidence = self._quick_evidence_check(
                results, candidate, constraint
            )

            if evidence.confidence >= 0.5:  # Lower threshold for filtering
                candidate.add_evidence(constraint.id, evidence)
                filtered.append(candidate)

        return filtered

    def _extract_relevant_candidates(
        self, results: Dict, constraint: Constraint
    ) -> List[Candidate]:
        """Extract candidates relevant to a specific constraint."""
        content = results.get("current_knowledge", "")

        # If no content, return empty list
        if not content or "Error" in content or "No results found" in content:
            logger.warning(
                f"No valid content to extract candidates from for constraint: {constraint.description}"
            )
            return []

        # Determine what type of entity we're looking for
        entity_type = getattr(self, "entity_type", "entity")

        # Use LLM to extract entities matching the constraint
        prompt = f"""Analyze these search results and extract {entity_type} names that could satisfy this constraint:

Constraint: {constraint.description}
Type: {constraint.type.value}
Value: {constraint.value}

Search Results:
{content}

Your task:
1. Understand what the constraint is asking for
2. Identify mentions of specific {entity_type} names in the search results
3. Extract only those names that could potentially satisfy the constraint
4. Focus on proper nouns and specific names

Important:
- Extract actual {entity_type} names, not descriptions or categories
- If the search results mention a specific {entity_type} that matches the constraint criteria, extract it
- Be thorough - don't miss names that are mentioned in passing
- Consider the context to determine if a name is relevant to the constraint

Return one {entity_type} name per line. Only include names that could satisfy the constraint."""

        try:
            response = self.model.invoke(prompt)
            extracted_text = remove_think_tags(response.content)

            candidates = []
            seen_names = set()  # Track unique names

            for line in extracted_text.strip().split("\n"):
                name = line.strip()
                # Remove common list markers and clean up
                name = name.lstrip("- •·*0123456789.").strip()

                # Skip empty lines or very short names
                if not name or len(name) <= 2:
                    continue

                # Normalize for deduplication
                normalized_name = name.lower()
                if normalized_name in seen_names:
                    continue

                # Exclude meta-commentary patterns
                exclude_patterns = [
                    "search result",
                    "based on",
                    "provided",
                    "found",
                    "does not",
                    "doesn't",
                    "cannot",
                    "there are no",
                    "according to",
                    "mentions",
                    "discusses",
                    "shows that",
                    "indicates",
                    "suggests",
                    "appears",
                    "seems",
                    "search",
                    "constraint",
                    "extract",
                    "entity",
                ]

                # Check if it's meta-commentary
                is_meta = any(
                    pattern in name.lower() for pattern in exclude_patterns
                )
                is_too_long = (
                    len(name.split()) > 10
                )  # Very long strings are usually explanations
                is_sentence = name.endswith(".") and len(name.split()) > 5

                if not is_meta and not is_too_long and not is_sentence:
                    # Accept various name formats
                    if (
                        name[0].isupper()  # Capitalized
                        or any(c.isupper() for c in name)  # Has capitals
                        or any(c.isdigit() for c in name)  # Contains numbers
                        or any(
                            c in name
                            for c in ["-", "&", "/", ":", "(", ")", '"', "'"]
                        )  # Special chars
                        or len(name.split()) <= 6
                    ):  # Reasonable length phrases
                        candidate = Candidate(name=name)
                        candidates.append(candidate)
                        seen_names.add(normalized_name)

            # Log extraction results for debugging
            logger.info(
                f"Extracted {len(candidates)} candidates for constraint: {constraint.description}"
            )
            if candidates:
                logger.debug(
                    f"Sample candidates: {[c.name for c in candidates[:5]]}"
                )

            return candidates[:50]  # Limit per search

        except Exception:
            logger.exception("Error extracting candidates")
            return []

    def _quick_evidence_check(
        self, results: Dict, candidate: Candidate, constraint: Constraint
    ) -> Evidence:
        """Quick evidence check for filtering with enhanced scoring."""
        content = results.get("current_knowledge", "")
        search_results = results.get("search_results", [])

        # Initialize confidence components
        name_presence = 0.0
        constraint_presence = 0.0
        co_occurrence = 0.0
        context_quality = 0.0

        candidate_lower = candidate.name.lower()
        value_lower = constraint.value.lower()
        content_lower = content.lower()

        # Check candidate name presence
        if candidate_lower in content_lower:
            name_count = content_lower.count(candidate_lower)
            name_presence = min(
                0.3 + (name_count * 0.05), 0.4
            )  # More occurrences = higher confidence

        # Check constraint value presence
        if value_lower in content_lower:
            value_count = content_lower.count(value_lower)
            constraint_presence = min(0.3 + (value_count * 0.05), 0.4)

        # Check co-occurrence and proximity
        if name_presence > 0 and constraint_presence > 0:
            # Find all positions
            name_positions = []
            start = 0
            while start < len(content_lower):
                pos = content_lower.find(candidate_lower, start)
                if pos == -1:
                    break
                name_positions.append(pos)
                start = pos + 1

            value_positions = []
            start = 0
            while start < len(content_lower):
                pos = content_lower.find(value_lower, start)
                if pos == -1:
                    break
                value_positions.append(pos)
                start = pos + 1

            # Calculate minimum distance
            if name_positions and value_positions:
                min_distance = min(
                    abs(n - v) for n in name_positions for v in value_positions
                )

                if min_distance < 100:  # Very close proximity
                    co_occurrence = 0.2
                elif min_distance < 200:  # Close proximity
                    co_occurrence = 0.15
                elif min_distance < 500:  # Moderate proximity
                    co_occurrence = 0.1
                else:  # Same document
                    co_occurrence = 0.05

        # Check result quality
        if search_results:
            # Count how many results mention both candidate and constraint
            relevant_results = 0
            for result in search_results[:10]:
                title = result.get("title", "").lower()
                snippet = result.get("snippet", "").lower()

                if (
                    candidate_lower in title or candidate_lower in snippet
                ) and (value_lower in title or value_lower in snippet):
                    relevant_results += 1

            context_quality = min(relevant_results * 0.05, 0.2)

        # Calculate final confidence
        confidence = (
            name_presence
            + constraint_presence
            + co_occurrence
            + context_quality
        )

        # Apply constraint type weight
        if constraint.type == ConstraintType.STATISTIC:
            confidence *= 1.1  # Numeric constraints need precise matching
        elif constraint.type == ConstraintType.PROPERTY:
            confidence *= 0.95  # Properties can be more flexible

        return Evidence(
            claim=f"Evidence for {candidate.name} matching {constraint.description}",
            confidence=min(confidence, 1.0),
            type=EvidenceType.INFERENCE,
            source="quick_evidence_check",
            metadata={
                "name_presence": name_presence,
                "constraint_presence": constraint_presence,
                "co_occurrence": co_occurrence,
                "context_quality": context_quality,
            },
        )

    def _focused_evidence_gathering(self):
        """Gather detailed evidence for the narrowed candidates."""
        if self.progress_callback:
            constraint_count = len(self.constraints)
            evidence_needed = len(self.candidates) * constraint_count
            self.progress_callback(
                f"Verifying {len(self.candidates)} candidates against {constraint_count} constraints ({evidence_needed} checks)",
                80,
                {
                    "phase": "evidence_gathering",
                    "candidate_count": len(self.candidates),
                    "constraint_count": constraint_count,
                    "total_evidence_needed": evidence_needed,
                },
            )

        for i, candidate in enumerate(self.candidates):
            for j, constraint in enumerate(self.constraints):
                # Skip if we already have evidence from filtering
                if constraint.id in candidate.evidence:
                    continue

                # Detailed evidence search
                query = f'"{candidate.name}" {constraint.value} verification'
                results = self._execute_search(query)

                evidence = self.evidence_evaluator.extract_evidence(
                    results.get("current_knowledge", ""),
                    candidate.name,
                    constraint,
                )

                candidate.add_evidence(constraint.id, evidence)

                if (
                    self.progress_callback and i < 5
                ):  # Report progress for top candidates
                    conf_emoji = (
                        "✓"
                        if evidence.confidence >= self.evidence_threshold
                        else "○"
                    )
                    self.progress_callback(
                        f"{conf_emoji} {candidate.name} | {constraint.type.value}: {evidence.confidence:.0%}",
                        None,
                        {
                            "phase": "evidence_detail",
                            "candidate": candidate.name,
                            "constraint": constraint.description,
                            "constraint_type": constraint.type.value,
                            "confidence": evidence.confidence,
                            "evidence_type": evidence.type.value,
                            "meets_threshold": evidence.confidence
                            >= self.evidence_threshold,
                        },
                    )

        # Final scoring
        for candidate in self.candidates:
            candidate.calculate_score(self.constraints)

        # Sort by score
        self.candidates.sort(key=lambda c: c.score, reverse=True)

    def _deduplicate_candidates(
        self, candidates: List[Candidate]
    ) -> List[Candidate]:
        """Remove duplicate candidates."""
        seen = {}
        unique = []

        for candidate in candidates:
            key = candidate.name.lower().strip()
            if key not in seen:
                seen[key] = candidate
                unique.append(candidate)

        return unique

    def _format_constraint_analysis(self) -> str:
        """Format initial constraint analysis."""
        analysis = "**Query Constraint Analysis**\n\n"
        analysis += f"Total constraints identified: {len(self.constraints)}\n\n"
        analysis += "**Constraint Ranking (by restrictiveness):**\n"

        for i, constraint in enumerate(self.constraint_ranking):
            score = self._calculate_restrictiveness_score(constraint)
            analysis += (
                f"{i + 1}. [{constraint.type.value}] {constraint.description}\n"
            )
            analysis += f"   Restrictiveness score: {score}\n"
            analysis += f"   Value: {constraint.value}\n\n"

        return analysis

    def _format_debug_summary(self) -> str:
        """Format comprehensive debug summary."""
        summary = "**Debug Summary**\n\n"

        # Constraint analysis
        summary += "**Constraint Processing:**\n"
        for i, constraint in enumerate(self.constraint_ranking):
            score = self._calculate_restrictiveness_score(constraint)
            summary += f"{i + 1}. [{constraint.type.value}] {constraint.value} (score: {score})\n"

        # Search progression
        summary += "\n**Search Progression:**\n"
        if hasattr(self, "stage_candidates"):
            for stage, candidates in self.stage_candidates.items():
                summary += f"Stage {stage + 1}: {len(candidates)} candidates\n"

        # Evidence coverage
        summary += "\n**Evidence Coverage:**\n"

        for i, candidate in enumerate(self.candidates[:5]):
            evidence_count = len(candidate.evidence)
            satisfied = sum(
                1
                for c in self.constraints
                if c.id in candidate.evidence
                and candidate.evidence[c.id].confidence
                >= self.evidence_threshold
            )

            summary += f"{i + 1}. {candidate.name}: {evidence_count} evidence, "
            summary += f"{satisfied}/{len(self.constraints)} constraints\n"

        # Search statistics
        summary += "\n**Search Statistics:**\n"
        total_discovered = (
            sum(len(c) for c in self.stage_candidates.values())
            if hasattr(self, "stage_candidates")
            else 0
        )
        summary += f"Total candidates discovered: {total_discovered}\n"
        summary += f"Final candidates: {len(self.candidates)}\n"
        summary += f"Constraints: {len(self.constraints)}\n"

        return summary

    def _calculate_restrictiveness_score(self, constraint: Constraint) -> int:
        """Calculate restrictiveness score for a constraint."""
        score = 0

        # Type-based scoring
        if constraint.type == ConstraintType.STATISTIC:
            score += 10
        elif constraint.type == ConstraintType.EVENT:
            score += 8
        elif constraint.type == ConstraintType.LOCATION:
            score += 6
        elif constraint.type == ConstraintType.PROPERTY:
            score += 4

        # Specificity scoring
        if constraint.value:
            if any(char.isdigit() for char in constraint.value):
                score += 5
            if len(constraint.value.split()) > 3:
                score += 3
            if any(
                term in constraint.value.lower()
                for term in ["specific", "exact", "only", "must"]
            ):
                score += 2

        return score

    def _format_stage_results(
        self, stage: int, constraint: Constraint, candidates: List[Candidate]
    ) -> str:
        """Format results for a search stage with detailed information."""
        result = f"**Search Stage {stage + 1}**\n\n"
        result += f"Constraint: {constraint.description}\n"
        result += f"Type: {constraint.type.value}\n"
        result += f"Search Value: {constraint.value}\n"
        result += f"Candidates found: {len(candidates)}\n\n"

        # Add search statistics
        result += "**Search Statistics:**\n"
        if hasattr(self, "search_history"):
            stage_searches = [
                s for s in self.search_history if s.get("stage", -1) == stage
            ]
            result += f"- Queries executed: {len(stage_searches)}\n"
            result += f"- Total results analyzed: {getattr(self, f'stage_{stage}_results_count', 0)}\n"

        result += f"- Candidates before filtering: {getattr(self, f'stage_{stage}_raw_candidates', len(candidates))}\n"
        result += f"- Candidates after deduplication: {len(candidates)}\n\n"

        if candidates:
            result += "**Top Candidates:**\n"
            # Group candidates to show variety
            grouped = self._group_similar_candidates(candidates[:20])
            for group_name, group_items in grouped.items():
                result += f"\n{group_name} ({len(group_items)} items):\n"
                for i, candidate in enumerate(group_items[:5]):
                    result += f"  {i + 1}. {candidate.name}\n"
                if len(group_items) > 5:
                    result += f"  ... and {len(group_items) - 5} more\n"
        else:
            result += "No candidates found for this constraint.\n"

        # Add sample search results for debugging
        if hasattr(self, "search_history") and candidates:
            result += "\n**Sample Search Results:**\n"
            recent_searches = [
                s
                for s in self.search_history[-3:]
                if s.get("stage", -1) == stage
            ]
            for search in recent_searches[:2]:
                result += f"- Query: '{search.get('query', '')}'\n"
                if "results_preview" in search:
                    result += (
                        f"  Preview: {search['results_preview'][:100]}...\n"
                    )

        return result

    def _format_search_summary(self) -> str:
        """Format progressive search summary."""
        summary = "**Progressive Search Summary**\n\n"

        # Show search progression
        summary += "**Stage-by-Stage Filtering:**\n"
        prev_count = 0

        for stage, candidates in self.stage_candidates.items():
            constraint = (
                self.constraint_ranking[stage]
                if stage < len(self.constraint_ranking)
                else None
            )
            if constraint:
                count = len(candidates)
                change = count - prev_count if stage > 0 else count
                change_str = f" ({change:+d})" if stage > 0 else ""

                summary += f"\nStage {stage + 1} [{constraint.type.value}]: {constraint.value[:40]}\n"
                summary += f"  Results: {count} candidates{change_str}\n"

                if candidates:
                    # Group candidates by type
                    grouped = self._group_similar_candidates(candidates[:20])
                    for group_name, group_items in grouped.items():
                        summary += f"  {group_name}: {len(group_items)} items\n"
                        for item in group_items[:3]:
                            summary += f"    • {item.name}\n"
                        if len(group_items) > 3:
                            summary += (
                                f"    ... and {len(group_items) - 3} more\n"
                            )

                prev_count = count

        summary += (
            f"\n**Final Result: {len(self.candidates)} candidates selected**\n"
        )

        return summary

    def _format_evidence_summary(self) -> str:
        """Format evidence gathering summary."""
        summary = "**Evidence Gathering Summary**\n\n"

        for i, candidate in enumerate(self.candidates[:5]):
            summary += f"**{i + 1}. {candidate.name}**\n"

            for constraint in self.constraints:
                evidence = candidate.evidence.get(constraint.id)
                if evidence:
                    conf_str = f"{evidence.confidence:.0%}"
                    summary += (
                        f"  • {constraint.description[:40]}...: {conf_str}\n"
                    )
                else:
                    summary += (
                        f"  • {constraint.description[:40]}...: No evidence\n"
                    )

            summary += f"  Overall Score: {candidate.score:.2f}\n\n"

        return summary

    # Commented out to use the parent's optimized _execute_search method
    '''def _execute_search(self, search_query: str) -> Dict:
        """Execute a comprehensive search using source-based strategy for complex queries."""
        if not hasattr(self, "search_history"):
            self.search_history = []

        self.search_history.append(
            {
                "query": search_query,
                "timestamp": self._get_timestamp(),
                "iteration": getattr(self, "iteration", 0),
            }
        )

        # Debug: Check if search engine is available
        if not hasattr(self, "search") or self.search is None:
            logger.error(f"No search engine configured for query: {search_query}")
            logger.error(f"Strategy attributes: {list(self.__dict__.keys())}")
            return {"current_knowledge": "", "search_results": []}

        try:
            # Log that we're attempting to use source-based strategy
            logger.info(f"Attempting source-based search for: {search_query}")

            # For complex queries, use source-based strategy with multiple iterations
            if hasattr(self, "source_strategy"):
                source_strategy = self.source_strategy
            else:
                logger.info("Creating new SourceBasedSearchStrategy instance")
                source_strategy = SourceBasedSearchStrategy(
                    model=self.model,
                    search=self.search,
                    all_links_of_system=self.all_links_of_system,
                    include_text_content=True,
                    use_cross_engine_filter=False,  # We'll handle filtering ourselves
                    use_atomic_facts=False,
                )
                source_strategy.max_iterations = (
                    1  # More efficient with single iteration
                )
                source_strategy.questions_per_iteration = (
                    9  # More questions for broader coverage
                )

            # Use source-based strategy for complex search
            try:
                # Set a simple progress callback if we have one
                if self.progress_callback:

                    def sub_callback(msg, prog, data):
                        # Don't propagate all sub-progress updates
                        if "phase" in data and data["phase"] in [
                            "search_complete",
                            "final_results",
                        ]:
                            self.progress_callback(f"Sub-search: {msg}", None, data)

                    source_strategy.set_progress_callback(sub_callback)

                logger.info("Executing source-based search...")
                # Run the search
                result = source_strategy.analyze_topic(search_query)

                if (
                    result
                    and "current_knowledge" in result
                    and "all_links_of_system" in result
                ):
                    search_results = result.get("all_links_of_system", [])

                    # Extract the most relevant information from the findings
                    knowledge_parts = []
                    if "findings" in result:
                        for finding in result["findings"]:
                            if "content" in finding and finding["content"]:
                                knowledge_parts.append(finding["content"])

                    # Also include search results summaries
                    for i, link in enumerate(search_results[:15]):  # More results
                        if isinstance(link, dict):
                            title = link.get("title", "")
                            snippet = link.get("snippet", "")
                            content = link.get("content", "")
                            url = link.get("link", link.get("url", ""))

                            if title or snippet:
                                result_text = f"\nResult {i+1}: {title}"
                                if url:
                                    result_text += f"\nURL: {url}"
                                if snippet:
                                    result_text += f"\nSnippet: {snippet}"
                                if content and content != snippet:
                                    result_text += f"\nContent: {content[:500]}..."
                                knowledge_parts.append(result_text)

                    current_knowledge = "\n\n".join(knowledge_parts)

                    return {
                        "current_knowledge": current_knowledge,
                        "search_results": search_results,
                        "detailed_findings": result.get("findings", []),
                    }
                else:
                    # Fallback to simple search
                    logger.warning(
                        "Source-based search returned empty results, falling back to simple search"
                    )
                    return self._simple_search(search_query)

            except Exception as e:
                logger.exception("Source-based search failed")
                logger.warning("Falling back to simple search")
                return self._simple_search(search_query)

        except Exception as e:
            logger.error(f"Error during search for '{search_query}': {str(e)}")
            return {
                "current_knowledge": f"Error during search: {str(e)}",
                "search_results": [],
            }'''

    def _simple_search(self, search_query: str) -> Dict:
        """Fallback simple search using search engine directly."""
        try:
            # Use the search engine directly for simple queries
            search_results = self.search.run(search_query)

            if search_results and isinstance(search_results, list):
                # Format search results into a knowledge string
                content_parts = []

                for i, result in enumerate(search_results[:15]):  # More results
                    title = result.get("title", "Untitled")
                    snippet = result.get("snippet", "")
                    content = result.get("content", "")
                    url = result.get("link", result.get("url", ""))

                    content_parts.append(f"Result {i + 1}: {title}")
                    if url:
                        content_parts.append(f"URL: {url}")
                    if snippet:
                        content_parts.append(f"Snippet: {snippet}")
                    if content and content != snippet:
                        content_parts.append(
                            f"Content preview: {content[:300]}..."
                        )
                    content_parts.append("")  # Empty line between results

                current_knowledge = "\n".join(content_parts)

                return {
                    "current_knowledge": current_knowledge,
                    "search_results": search_results,
                }
            else:
                # Return empty knowledge if no results
                return {
                    "current_knowledge": f"No results found for query: {search_query}",
                    "search_results": [],
                }
        except Exception as e:
            logger.exception("Simple search error")
            return {
                "current_knowledge": f"Search error: {e!s}",
                "search_results": [],
            }

    def _validate_search_results(
        self, results: Dict, constraint: Constraint
    ) -> bool:
        """Validate that search results contain relevant information."""
        if not results:
            return False

        content = results.get("current_knowledge", "")
        search_results = results.get("search_results", [])

        # Basic validation checks
        if not content or len(content) < 50:  # Too short to be meaningful
            logger.debug(f"Content too short: {len(content)} characters")
            return False

        if "Error" in content and len(content) < 100:
            logger.debug(f"Error in results: {content[:100]}")
            return False

        if "No results found" in content:
            logger.debug("No results found")
            return False

        # For stats/numeric constraints, check for related terms
        if constraint.type == ConstraintType.STATISTIC:
            # Look for related terms about TV shows, episodes, etc
            relevant_terms = [
                "tv",
                "show",
                "series",
                "episode",
                "season",
                "program",
                "character",
                "fiction",
            ]
            content_lower = content.lower()

            term_found = any(term in content_lower for term in relevant_terms)
            if not term_found:
                logger.debug(
                    "No relevant TV/show terms found for statistic constraint"
                )
                return False
        else:
            # Check for relevance to constraint using key terms
            constraint_terms = [
                term
                for term in constraint.value.lower().split()
                if len(term) > 2
                and term
                not in ["the", "and", "with", "for", "had", "his", "her"]
            ]
            content_lower = content.lower()

            # Count how many meaningful terms appear
            if constraint_terms:
                term_matches = sum(
                    1 for term in constraint_terms if term in content_lower
                )
                relevance_ratio = term_matches / len(constraint_terms)

                # Require at least one term match
                if relevance_ratio < 0.2:
                    logger.debug(
                        f"Low relevance: {relevance_ratio:.0%} term matches"
                    )
                    return False

        # Check search results quality
        if search_results and isinstance(search_results, list):
            valid_results = sum(
                1
                for r in search_results
                if isinstance(r, dict) and (r.get("title") or r.get("snippet"))
            )
            if valid_results < 1:
                logger.debug("No valid search results with title/snippet")
                return False

        return True

    def _get_timestamp(self) -> str:
        """Get current timestamp."""
        return datetime.now(UTC).isoformat()

    def _group_similar_candidates(
        self, candidates: List[Candidate]
    ) -> Dict[str, List[Candidate]]:
        """Group candidates by similar characteristics."""
        grouped = {}

        for candidate in candidates:
            # Try to determine group type based on name patterns
            name = candidate.name.lower()

            if any(
                keyword in name
                for keyword in ["model", "llm", "gpt", "claude", "gemini"]
            ):
                group = "AI Models"
            elif any(
                keyword in name
                for keyword in ["country", "nation", "republic", "kingdom"]
            ):
                group = "Countries"
            elif any(
                keyword in name for keyword in ["city", "town", "village"]
            ):
                group = "Cities"
            elif any(
                keyword in name for keyword in ["year", "century", "decade"]
            ):
                group = "Time Periods"
            elif any(
                keyword in name
                for keyword in ["person", "mr", "ms", "dr", "prof"]
            ):
                group = "People"
            elif any(c.isdigit() for c in name):
                group = "Numeric Items"
            else:
                # Default grouping based on first word
                first_word = (
                    candidate.name.split()[0]
                    if candidate.name.split()
                    else "Other"
                )
                group = f"{first_word} Items"

            if group not in grouped:
                grouped[group] = []
            grouped[group].append(candidate)

        return grouped
