import random
import uuid
from typing import Dict, List, Any, Optional

from loguru import logger

from .base_strategy import BaseSearchStrategy
from .source_based_strategy import SourceBasedSearchStrategy
from .focused_iteration_strategy import FocusedIterationStrategy
from ..findings.topic import Topic, TopicGraph
from ..findings.repository import FindingsRepository
from ...utilities.json_utils import extract_json, get_llm_response_text
from ...utilities.thread_context import preserve_research_context


class TopicOrganizationStrategy(BaseSearchStrategy):
    """
    Strategy that organizes search results into topics with lead sources.

    This strategy reuses SourceBasedSearchStrategy to gather sources,
    then organizes them into topic clusters without generating full text.
    Each topic has a lead source and supporting sources that can be validated.
    """

    def __init__(
        self,
        search,
        model,
        citation_handler=None,
        use_cross_engine_filter: bool = True,
        filter_reorder: bool = True,
        filter_reindex: bool = True,
        cross_engine_max_results: int = None,
        all_links_of_system=None,
        settings_snapshot=None,
        search_original_query: bool = True,
        min_sources_per_topic: int = 1,
        max_topics: int = 5,
        similarity_threshold: float = 0.7,
        enable_refinement: bool = False,
        max_refinement_iterations: int = 2,
        generate_text: bool = True,
        use_focused_iteration: bool = False,  # New parameter to use FocusedIteration
    ):
        """Initialize the topic organization strategy."""
        super().__init__(
            all_links_of_system=all_links_of_system,
            settings_snapshot=settings_snapshot,
            search_original_query=search_original_query,
        )

        self.model = model
        self.search = search

        # Store cross-engine filter parameters for refinement
        self.use_cross_engine_filter = use_cross_engine_filter
        self.filter_reorder = filter_reorder
        self.filter_reindex = filter_reindex
        self.cross_engine_max_results = cross_engine_max_results

        # Create citation handler if not provided (needed for text generation)
        if citation_handler is None and generate_text:
            from local_deep_research.citation_handler import CitationHandler

            self.citation_handler = CitationHandler(
                model, settings_snapshot=settings_snapshot
            )
        else:
            self.citation_handler = citation_handler

        # Topic organization parameters
        self.min_sources_per_topic = min_sources_per_topic
        self.max_topics = max_topics
        self.similarity_threshold = similarity_threshold
        self.enable_refinement = enable_refinement
        self.max_refinement_iterations = max_refinement_iterations
        self.refinement_questions = []  # Track all questions asked
        self.iteration_history = []  # Track what happened in each iteration
        self.generate_text = generate_text

        # Initialize findings repository for proper formatting
        self.findings_repository = FindingsRepository(model)

        # Create a no-op citation handler to skip text generation
        class NoTextCitationHandler:
            def analyze_followup(
                self, query, results, previous_knowledge, nr_of_links
            ):
                return {"content": "", "documents": []}

        # Store which strategy we're using
        self.use_focused_iteration = use_focused_iteration

        # Log TopicOrganizationStrategy configuration
        logger.info(
            f"TopicOrganizationStrategy configuration - use_focused_iteration: {use_focused_iteration}, enable_refinement: {enable_refinement}"
        )
        logger.debug(
            f"TopicOrganizationStrategy - min_sources_per_topic: {min_sources_per_topic}, max_topics: {max_topics}, generate_text: {generate_text}"
        )

        # NOTE: Relevance filtering is DISABLED for TopicOrganizationStrategy
        # Testing showed that pre-filtering sources reduced accuracy from 88.9% to 66.7%
        # because it filtered out sources containing correct answers.
        # The topic organization process itself does a good job of selecting relevant sources.
        logger.info(
            "TopicOrganizationStrategy: Relevance filtering is disabled (improves accuracy)"
        )

        # Initialize source gathering strategy based on configuration
        # Pass our own all_links_of_system so it accumulates properly
        # For topic organization, we typically want more sources, so disable cross-engine filter by default
        # unless explicitly enabled
        if use_focused_iteration:
            # Use FocusedIterationStrategy for better factual question performance
            logger.info(
                "TopicOrganizationStrategy using FocusedIterationStrategy for source gathering (optimized for factual questions)"
            )
            self.source_strategy = FocusedIterationStrategy(
                search=search,
                model=model,
                citation_handler=NoTextCitationHandler(),  # Use no-op handler to skip text generation
                all_links_of_system=self.all_links_of_system,  # Share our list with source strategy
                max_iterations=8,  # HARDCODED: Force optimal settings
                questions_per_iteration=5,  # HARDCODED: Force optimal settings
                use_browsecomp_optimization=True,  # Enable browsecomp optimization for better question generation
                settings_snapshot=settings_snapshot,
            )
        else:
            # Default to SourceBasedSearchStrategy
            logger.info(
                "TopicOrganizationStrategy using SourceBasedSearchStrategy for source gathering (default)"
            )
            self.source_strategy = SourceBasedSearchStrategy(
                search=search,
                model=model,
                citation_handler=NoTextCitationHandler(),  # Use no-op handler to skip text generation
                include_text_content=False,  # Also disable text content retrieval
                use_cross_engine_filter=False,  # Disable cross-engine filter to get more sources for topic organization
                filter_reorder=filter_reorder,
                filter_reindex=filter_reindex,
                cross_engine_max_results=cross_engine_max_results,
                all_links_of_system=self.all_links_of_system,  # Share our list with source strategy
                settings_snapshot=settings_snapshot,
                search_original_query=search_original_query,
            )

        # Topic graph to store organized topics
        self.topic_graph = TopicGraph()

    def _extract_topics_from_sources(
        self,
        sources: List[Dict[str, Any]],
        query: str,
        existing_topics: List[Topic] = None,
    ) -> List[Topic]:
        """
        Use LLM to identify topics from search results by iterating through each source.
        """
        if not sources:
            return []

        import uuid

        existing_topics = existing_topics or []
        topics = list(existing_topics)  # Start with existing topics

        # Process all sources - removed artificial limit
        sources_to_process = sources

        # Shuffle sources to avoid ordering bias in topic organization
        # Note: random.shuffle is used here for non-security purposes (preventing ML bias)
        sources_to_process = (
            sources_to_process.copy()
        )  # Create a copy to avoid modifying original
        random.shuffle(sources_to_process)  # DevSkim: ignore DS148264

        logger.info(
            f"Processing {len(sources_to_process)} sources iteratively for topic organization (shuffled)"
        )
        sources_organized = 0
        sources_deleted = 0
        sources_failed = 0

        for i, source in enumerate(sources_to_process):
            # Log progress periodically (every 5 sources for better feedback)
            if i > 0 and i % 5 == 0:
                progress_msg = f"Processing source {i}/{len(sources_to_process)}: {sources_organized} organized, {sources_deleted} deleted, {sources_failed} failed, {len(topics)} topics"
                logger.log("MILESTONE", progress_msg)
                # Also update the progress callback
                if self.progress_callback:
                    progress_percent = 40 + int(
                        (i / len(sources_to_process)) * 20
                    )  # Progress from 40% to 60%
                    self.progress_callback(
                        progress_msg,
                        progress_percent,
                        {"phase": "topic_extraction"},
                    )
                logger.info(
                    f"Topic organization progress: {i}/{len(sources_to_process)} sources processed, {len(topics)} topics created"
                )

            # Format existing topics for context with source counts
            topics_info = []
            for j, topic in enumerate(topics):
                lead = topic.lead_source
                source_count = len(topic.get_all_sources())
                topics_info.append(
                    f"Topic {j}: {topic.title} ({source_count} sources)\nLead snippet: {lead.get('snippet', '')}"
                )

            existing_topics_str = (
                "\n\n".join(topics_info) if topics_info else "No topics yet"
            )

            # Calculate progress
            sources_remaining = len(sources_to_process) - i - 1

            # Calculate ideal number of topics based on total sources
            # Using square root: if we have N sources, we want roughly sqrt(N) topics with sqrt(N) sources each
            import math

            total_sources = len(sources_to_process)
            ideal_topics = int(math.sqrt(total_sources))
            ideal_sources_per_topic = ideal_topics  # Same as number of topics
            current_topics = len(topics)

            # Create prompt for this specific source
            source_prompt = f"""
For the research query: "{query}"

PROGRESS: Processing source {i + 1} of {total_sources} ({sources_remaining} remaining)

TARGET DISTRIBUTION:
- Total sources to organize: {total_sources}
- Current topics: {current_topics}
- Ideal balance: ~{ideal_topics} topics with ~{ideal_sources_per_topic} sources each
- {"Too many topics - prefer adding to existing ones" if current_topics > ideal_topics else "Can create new topics if needed" if current_topics < ideal_topics else "Good balance - be selective"}

GUIDELINES:
- Aim for balanced distribution: ~{ideal_sources_per_topic} sources per topic
- Prefer adding to existing topics when there's a clear thematic match
- Only create a new topic if the source represents a distinctly different aspect
- Consider: Topics with <{ideal_sources_per_topic // 2} sources need more sources; topics with >{ideal_sources_per_topic * 1.5:.0f} sources are getting full
- Avoid overloading: If a topic has {ideal_sources_per_topic * 2}+ sources, consider if the new source might better start a related subtopic
- Use "d" ONLY for sources that are COMPLETELY UNRELATED (e.g., wrong topic entirely, spam, or error pages). When in doubt, keep the source!

CURRENT SOURCE TO CATEGORIZE:
Title: {source.get("title", "Untitled")}
URL: {source.get("link", "")}
Snippet: {source.get("snippet", "")}

EXISTING TOPICS (showing source count):
{existing_topics_str}

Respond with:
- Use "-" to create a new topic
- Use topic number (0-{len(topics) - 1}) to add to existing topic
- Add "+" after the number if this source could be a better lead for that topic
- Use "d" to delete ONLY if source is about a completely different subject (not just low relevance)

Examples: 0, 2+, -, 1, 3+, d

Response:"""

            # Debug log the full prompt for the first few sources and periodically
            if i < 3 or i % 50 == 0:
                logger.debug(
                    f"Topic organization prompt for source {i + 1}:\n{source_prompt}"
                )  # Log full prompt

            try:
                response = self.model.invoke(source_prompt)
                response_text = str(
                    response.content
                    if hasattr(response, "content")
                    else response
                ).strip()

                # Parse the compact response format
                is_potential_lead = False

                if response_text.lower() == "d":
                    # Delete/skip this source as irrelevant
                    sources_deleted += 1
                    logger.info(
                        f"Skipping source {i} '{source.get('title', '')}' - marked as irrelevant"
                    )
                    continue  # Skip to next source
                if response_text == "-":
                    # Create new topic
                    topic_index = -1
                else:
                    # Check if ends with + (potential lead)
                    is_potential_lead = response_text.endswith("+")

                    # Remove the + if present and parse the number
                    if is_potential_lead:
                        topic_index = int(response_text[:-1])
                    else:
                        topic_index = int(response_text)

                if topic_index >= 0 and topic_index < len(topics):
                    # Add to existing topic
                    topics[topic_index].add_supporting_source(source)
                    sources_organized += 1
                    logger.info(
                        f"Added source {i} '{source.get('title', '')}' to topic '{topics[topic_index].title}'"
                    )

                    # If marked as potential lead, re-evaluate this topic's lead
                    if (
                        is_potential_lead
                        and len(topics[topic_index].get_all_sources()) >= 2
                    ):
                        logger.debug(
                            f"Source {i} marked as potential lead for topic {topic_index}"
                        )
                        lead_changed = self._reselect_lead_for_single_topic(
                            topics[topic_index], topics
                        )
                        if lead_changed:
                            logger.info(
                                f"Updated lead source for topic '{topics[topic_index].title}'"
                            )
                elif topic_index == -1:
                    # Create new topic with this source as lead
                    new_topic = Topic(
                        id=str(uuid.uuid4()),
                        title=source.get("title", f"Topic {len(topics) + 1}"),
                        lead_source=source,
                    )
                    topics.append(new_topic)
                    sources_organized += 1
                    logger.info(
                        f"Created new topic '{new_topic.title}' with source {i} as lead"
                    )
                else:
                    logger.warning(
                        f"Invalid topic index {topic_index} for source {i}"
                    )
                    sources_failed += 1

            except Exception:
                logger.warning(
                    f"Error processing source {i} '{source.get('title', '')}'. Creating as new topic."
                )
                # On error, create as new topic to ensure all sources are organized
                try:
                    new_topic = Topic(
                        id=str(uuid.uuid4()),
                        title=source.get("title", f"Topic {len(topics) + 1}"),
                        lead_source=source,
                    )
                    topics.append(new_topic)
                    sources_organized += 1
                    logger.info(
                        f"Created recovery topic for source {i} after error"
                    )
                except Exception as recovery_error:
                    logger.exception(
                        f"Failed to create recovery topic for source {i}: {recovery_error}"
                    )
                    sources_failed += 1

        # Count sources in new topics before removing existing topics
        new_topics_count = (
            len(topics) - len(existing_topics)
            if existing_topics
            else len(topics)
        )

        # Remove original existing topics from the result (we only want the newly organized ones)
        if existing_topics:
            topics = topics[len(existing_topics) :]

        # Final summary
        total_processed = sources_organized + sources_deleted + sources_failed
        logger.info(
            f"Topic organization complete: {sources_organized}/{len(sources_to_process)} sources organized into {new_topics_count} new topics, {sources_deleted} deleted, {sources_failed} failed"
        )
        logger.info(
            f"Source accounting: organized={sources_organized}, deleted={sources_deleted}, failed={sources_failed}, total_processed={total_processed}, total_input={len(sources_to_process)}"
        )

        # Log topic distribution for new topics only
        topic_sizes = [len(topic.get_all_sources()) for topic in topics]
        if topic_sizes:
            avg_size = sum(topic_sizes) / len(topic_sizes)
            logger.info(
                f"New topics distribution: min={min(topic_sizes)}, max={max(topic_sizes)}, avg={avg_size:.1f} sources per topic"
            )

        # Log final milestone
        logger.log(
            "MILESTONE",
            f"Organized {sources_organized} sources into {len(topics)} topics ({sources_deleted} deleted, {sources_failed} failed)",
        )

        # Log any sources that weren't accounted for
        total_accounted = sources_organized + sources_deleted + sources_failed
        if total_accounted != len(sources_to_process):
            logger.error(
                f"SOURCE ACCOUNTING ERROR: {len(sources_to_process) - total_accounted} sources unaccounted for!"
            )
            logger.error(
                f"Input: {len(sources_to_process)}, Organized: {sources_organized}, Deleted: {sources_deleted}, Failed: {sources_failed}"
            )

        return topics

    def _reselect_lead_for_single_topic(
        self, topic: Topic, all_topics: List[Topic]
    ) -> bool:
        """
        Re-evaluate and potentially update the lead source for a single topic.
        Returns True if the lead was changed, False otherwise.
        """
        # Need at least 2 sources to consider changing lead
        if len(topic.get_all_sources()) < 2:
            return False

        # Get context from other topics (just titles for efficiency)
        other_topic_context = []
        for other_topic in all_topics:
            if other_topic.id != topic.id:
                other_topic_context.append(f"- {other_topic.title}")

        # Get all sources from this topic
        all_sources = topic.get_all_sources()
        sources_list = []
        current_lead_idx = 0

        for idx, source in enumerate(all_sources):
            title = source.get("title", "Untitled")
            snippet = source.get("snippet", "")
            url = source.get("url", "")

            # Extract domain from URL
            from urllib.parse import urlparse

            try:
                domain = urlparse(url).netloc if url else ""
            except (ValueError, AttributeError):
                domain = ""

            # Format source info without restrictions
            source_info = f"{idx}: [{domain}] {title}"
            if snippet:
                source_info += f"\n   {snippet}"

            if source == topic.lead_source:
                current_lead_idx = idx
                source_info += " [CURRENT LEAD]"

            sources_list.append(source_info)

        # Create minimal prompt for lead selection
        selection_prompt = f"""
Topic: {topic.title}

Sources in this topic:
{chr(10).join(sources_list)}

Other topics (context):
{chr(10).join(other_topic_context) if other_topic_context else "None"}

Which source number (0-{len(all_sources) - 1}) should be the lead? (most comprehensive/representative)

Response (just the number):"""

        try:
            response = self.model.invoke(selection_prompt)
            response_text = str(
                response.content if hasattr(response, "content") else response
            ).strip()

            # Parse the number
            new_lead_idx = int(response_text)

            if (
                0 <= new_lead_idx < len(all_sources)
                and new_lead_idx != current_lead_idx
            ):
                new_lead = all_sources[new_lead_idx]
                old_title = topic.lead_source.get("title", "Unknown")
                new_title = new_lead.get("title", "Unknown")

                logger.info(
                    f"Updating lead for topic: '{old_title}' -> '{new_title}'"
                )

                topic.update_lead_source(new_lead)
                # Update topic title to match new lead
                topic.title = new_lead.get("title", topic.title)
                return True

            return False

        except (ValueError, IndexError) as e:
            logger.debug(
                f"Could not parse lead selection for topic '{topic.title}': {e}"
            )
            return False
        except Exception:
            logger.warning(
                f"Error in lead re-selection for topic '{topic.title}'"
            )
            return False

    def _find_topic_relationships(self, topics: List[Topic]) -> None:
        """
        Identify relationships between topics and update the topic graph.
        """
        if len(topics) < 2:
            return

        # For now, don't auto-link topics since we don't have keywords
        # This could be enhanced to compare source titles/snippets for similarity
        pass

    def _filter_topics_by_relevance(
        self, topics: List[Topic], query: str
    ) -> List[Topic]:
        """
        Filter topics by their relevance to the original research question.
        For each topic, provides context from other topics and all sources within the topic.
        """
        if not topics:
            return []

        relevant_topics = []

        for topic in topics:
            # 1. Get lead sources from OTHER topics for context
            other_topic_leads = []
            for other_topic in topics:
                if other_topic.id != topic.id:
                    lead = other_topic.lead_source
                    other_topic_leads.append(
                        f"- {lead.get('title', 'Untitled')}: {lead.get('snippet', '')}"
                    )

            # 2. Get ALL sources from THIS topic
            topic_sources = []
            # Lead source
            lead = topic.lead_source
            topic_sources.append(
                f"LEAD: {lead.get('title', 'Untitled')}\nSnippet: {lead.get('snippet', '')}"
            )
            # Supporting sources
            for source in topic.supporting_sources:
                topic_sources.append(
                    f"- {source.get('title', 'Untitled')}: {source.get('snippet', '')}"
                )

            # 3. Create relevance evaluation prompt
            relevance_prompt = f"""
Evaluate if this topic is relevant for answering the research question.

RESEARCH QUESTION: {query}

CURRENT TOPIC: {topic.title}
Sources in this topic:
{chr(10).join(topic_sources)}

OTHER TOPICS IDENTIFIED (for context):
{chr(10).join(other_topic_leads) if other_topic_leads else "None"}

Does this topic contain information that helps answer the research question?
Consider:
- Does it address the main question or important sub-questions?
- Does it provide necessary background or context?
- Does it offer evidence, examples, or explanations relevant to the query?

Respond with only "yes" or "no".
"""

            try:
                # Call model directly with prompt
                response = self.model.invoke(relevance_prompt)
                response_text = str(
                    response.content
                    if hasattr(response, "content")
                    else response
                )

                # Check if response indicates relevance
                if "yes" in response_text.lower().strip():
                    relevant_topics.append(topic)
                    logger.info(
                        f"Topic '{topic.title}' deemed RELEVANT to query"
                    )
                else:
                    logger.info(
                        f"Topic '{topic.title}' deemed NOT relevant to query"
                    )
                    # Remove from topic graph if not relevant
                    if topic.id in self.topic_graph.topics:
                        del self.topic_graph.topics[topic.id]

            except Exception:
                logger.exception(
                    f"Error evaluating relevance for topic '{topic.title}', keeping it"
                )
                relevant_topics.append(topic)  # Keep on error

        return relevant_topics

    def _reselect_lead_sources(self, topics: List[Topic]) -> None:
        """
        Re-evaluate and potentially update the lead source for each topic after adding new sources.
        For each topic, the LLM sees all sources in that topic and lead sources from other topics.
        """
        if not topics or len(topics) < 1:
            return

        for i, topic in enumerate(topics):
            # Skip if topic has too few sources
            if len(topic.get_all_sources()) < 2:
                continue

            # Get lead sources from OTHER topics for context
            other_leads = []
            for j, other_topic in enumerate(topics):
                if i != j:
                    lead = other_topic.lead_source
                    other_leads.append(
                        f"Topic {j + 1}: {lead.get('title', 'Untitled')}\nSnippet: {lead.get('snippet', '')}"
                    )

            # Get ALL sources from THIS topic
            topic_sources = []
            all_sources = topic.get_all_sources()
            for idx, source in enumerate(all_sources):
                source_info = f"Source {idx}:\nTitle: {source.get('title', 'Untitled')}\nSnippet: {source.get('snippet', '')}"
                if source == topic.lead_source:
                    source_info += " [CURRENT LEAD]"
                topic_sources.append(source_info)

            # Prompt to select best lead
            selection_prompt = f"""
Select the best lead source for this topic cluster.

TOPIC: {topic.title}
Current sources in this topic:
{chr(10).join(topic_sources)}

OTHER TOPICS (for context):
{chr(10).join(other_leads) if other_leads else "None"}

Which source number (0-{len(all_sources) - 1}) should be the lead source for this topic?
The lead should be the most comprehensive and representative source.

Respond with only the number.
"""

            try:
                response = self.model.invoke(selection_prompt)
                response_text = str(
                    response.content
                    if hasattr(response, "content")
                    else response
                ).strip()

                # Parse the number
                new_lead_idx = int(response_text)

                if 0 <= new_lead_idx < len(all_sources):
                    new_lead = all_sources[new_lead_idx]
                    if new_lead != topic.lead_source:
                        logger.info(
                            f"Updating lead source for topic '{topic.title}' from '{topic.lead_source.get('title')}' to '{new_lead.get('title')}'"
                        )
                        topic.update_lead_source(new_lead)
                        # Update topic title to match new lead
                        topic.title = new_lead.get("title", topic.title)

            except (ValueError, IndexError):
                logger.warning(
                    f"Could not parse lead selection response for topic '{topic.title}': {response_text}"
                )
                # Keep existing lead
            except Exception:
                logger.exception(
                    f"Error reselecting lead for topic '{topic.title}'"
                )
                # Keep existing lead

    def _reorganize_topics(self, topics: List[Topic]) -> List[Topic]:
        """
        Reorganize sources across topics after lead source updates.
        For each topic, re-evaluate if sources still belong or should move to other topics.
        Can also create new topics if sources don't fit anywhere.
        """
        if not topics or len(topics) < 2:
            return topics

        # Track sources that need reassignment
        all_reassignments = []

        for i, topic in enumerate(topics):
            if len(topic.supporting_sources) < 1:
                continue

            # Get lead sources from ALL topics (including this one)
            all_leads = []
            for j, t in enumerate(topics):
                lead = t.lead_source
                all_leads.append(
                    f"Topic {j}: {lead.get('title', 'Untitled')}\nSnippet: {lead.get('snippet', '')}"
                )

            # Get all supporting sources from THIS topic
            sources_to_evaluate = []
            for idx, source in enumerate(topic.supporting_sources):
                sources_to_evaluate.append(
                    {
                        "index": idx,
                        "source": source,
                        "info": f"Source {idx}:\nTitle: {source.get('title', 'Untitled')}\nSnippet: {source.get('snippet', '')}",
                    }
                )

            if not sources_to_evaluate:
                continue

            # Build prompt for reorganization
            reorganize_prompt = f"""
Evaluate which topic each source belongs to based on the lead sources.

CURRENT TOPIC {i}: {topic.title}

ALL TOPIC LEADS:
{chr(10).join(all_leads)}

SOURCES TO EVALUATE FROM TOPIC {i}:
{chr(10).join([s["info"] for s in sources_to_evaluate])}

For each source, determine:
1. Which topic index (0-{len(topics) - 1}) it best fits with
2. Or -1 if it should form a new topic
3. Or -2 if it should be removed (not relevant)

Format as JSON list:
[
  {{"source_index": 0, "target_topic": 1}},
  {{"source_index": 1, "target_topic": -1}},
]

Consider similarity to lead sources when deciding.
"""

            try:
                response = self.model.invoke(reorganize_prompt)
                response_text = get_llm_response_text(response)

                reassignments = extract_json(response_text, expected_type=list)
                if reassignments is None:
                    reassignments = []

                # Process reassignments
                for assignment in reassignments:
                    src_idx = assignment.get("source_index")
                    target = assignment.get("target_topic")

                    if src_idx is None or target is None:
                        continue

                    if 0 <= src_idx < len(sources_to_evaluate):
                        source_data = sources_to_evaluate[src_idx]

                        if target == i:
                            # Keep in current topic
                            continue
                        if target == -2:
                            # Remove from topic
                            logger.info(
                                f"Removing source '{source_data['source'].get('title')}' from topic '{topic.title}'"
                            )
                            topic.reject_source(source_data["source"])
                        elif target == -1:
                            # Mark for new topic
                            all_reassignments.append(
                                {
                                    "source": source_data["source"],
                                    "from_topic": topic,
                                    "to_new_topic": True,
                                }
                            )
                        elif 0 <= target < len(topics):
                            # Move to another topic
                            all_reassignments.append(
                                {
                                    "source": source_data["source"],
                                    "from_topic": topic,
                                    "to_topic": topics[target],
                                }
                            )

            except Exception:
                logger.exception(
                    f"Error reorganizing sources for topic '{topic.title}'"
                )
                # Keep sources as is

        # Apply reassignments
        new_topic_sources = []
        for reassignment in all_reassignments:
            source = reassignment["source"]
            from_topic = reassignment["from_topic"]

            # Remove from original topic
            if source in from_topic.supporting_sources:
                from_topic.supporting_sources.remove(source)

            if reassignment.get("to_new_topic"):
                new_topic_sources.append(source)
            elif "to_topic" in reassignment:
                to_topic = reassignment["to_topic"]
                to_topic.add_supporting_source(source)
                logger.info(
                    f"Moved source '{source.get('title')}' from '{from_topic.title}' to '{to_topic.title}'"
                )

        # Create new topic if we have enough sources
        if len(new_topic_sources) >= self.min_sources_per_topic:
            # Select lead for new topic
            new_topic = Topic(
                id=str(uuid.uuid4()),
                title=new_topic_sources[0].get("title", "New Topic"),
                lead_source=new_topic_sources[0],
            )
            for source in new_topic_sources[1:]:
                new_topic.add_supporting_source(source)

            topics.append(new_topic)
            logger.info(
                f"Created new topic '{new_topic.title}' with {len(new_topic_sources)} sources"
            )

        # Remove topics that now have too few sources
        valid_topics = []
        for topic in topics:
            if len(topic.get_all_sources()) >= self.min_sources_per_topic:
                valid_topics.append(topic)
            else:
                logger.info(
                    f"Removing topic '{topic.title}' - too few sources after reorganization"
                )

        return valid_topics

    def _generate_refinement_question(
        self, topics: List[Topic], original_query: str
    ) -> Optional[str]:
        """
        Generate a follow-up question to improve confidence in answering the original query.
        """
        if not self.enable_refinement or not topics:
            return None

        # Format current topics and their source counts
        topic_summary = []
        for topic in topics:
            source_count = len(topic.get_all_sources())
            lead = topic.lead_source
            topic_summary.append(
                f"- {topic.title} ({source_count} sources)\n  Lead: {lead.get('snippet', '')}"
            )

        # Format previous questions to avoid repetition
        prev_questions_str = (
            "\n".join(f"- {q}" for q in self.refinement_questions)
            if self.refinement_questions
            else "None"
        )

        refinement_prompt = f"""
Based on the current research state, generate ONE follow-up question to improve confidence in answering the original query.

ORIGINAL QUERY: {original_query}

CURRENT TOPICS FOUND:
{chr(10).join(topic_summary)}

PREVIOUS REFINEMENT QUESTIONS ASKED:
{prev_questions_str}

CRITICAL: The follow-up question must stay focused on answering the ORIGINAL QUERY above.

Analyze what specific aspects of the original query are NOT well covered by current topics.
The refinement should fill gaps that directly help answer the original question better.

Generate a short, specific follow-up question that:
1. MUST directly relate to the ORIGINAL QUERY (not tangential topics)
2. Fills a specific gap in answering the original query
3. Is different from previous questions
4. Stays within the scope of the original query
5. Is concise and searchable (max 15 words)

If the current topics comprehensively answer ALL aspects of the original query, respond with "NONE".

Otherwise, respond with only the follow-up question, nothing else.
"""

        try:
            response = self.model.invoke(refinement_prompt)
            response_text = str(
                response.content if hasattr(response, "content") else response
            ).strip()

            if "none" in response_text.lower() or not response_text:
                logger.info("Model determined no refinement needed")
                return None

            # Clean up the question
            question = response_text.strip('"').strip("'").strip()
            if len(question.split()) > 20:  # Sanity check
                question = question  # Don't truncate the question

            logger.info(f"Generated refinement question: {question}")
            self.refinement_questions.append(question)
            return question

        except Exception:
            logger.exception("Error generating refinement question")
            return None

    def analyze_topic(self, query: str) -> Dict:
        """
        Analyze a topic by organizing search results into topic clusters.
        """
        logger.info(f"Starting topic organization for: {query}")

        strategy_name = (
            "focused iteration"
            if self.use_focused_iteration
            else "source-based"
        )
        self._update_progress(
            f"Gathering sources using {strategy_name} strategy",
            10,
            {"phase": "source_gathering"},
        )

        # Use source-based strategy to gather sources
        # Wrap in preserve_research_context to maintain context
        @preserve_research_context
        def _call_source_strategy(q):
            return self.source_strategy.analyze_topic(q)

        source_results = _call_source_strategy(query)

        strategy_name = (
            "FocusedIteration" if self.use_focused_iteration else "Source-based"
        )
        logger.info(
            f"{strategy_name} strategy returned {len(source_results.get('all_links_of_system', []))} total sources"
        )
        logger.info(
            f"{strategy_name} iterations: {source_results.get('iterations', 0)}"
        )

        # Log questions by iteration
        questions = source_results.get("questions_by_iteration", {})
        for iter_num, iter_questions in questions.items():
            logger.info(
                f"Iteration {iter_num}: {len(iter_questions)} questions"
            )

        # The source_based_strategy already added sources to its all_links_of_system with proper indices
        # We need to use those sources which already have sequential indices
        all_sources = source_results.get("all_links_of_system", [])

        logger.info(f"Using {len(all_sources)} sources for topic extraction")

        # Update our all_links_of_system to match what source_based_strategy collected
        # This ensures continuity of numbering
        self.all_links_of_system = source_results.get("all_links_of_system", [])

        # Add indices to all sources in all_links_of_system if they don't have them
        for i, source in enumerate(self.all_links_of_system, 1):
            if "index" not in source:
                source["index"] = str(i)

        if not all_sources:
            logger.warning("No sources found to organize into topics")
            return {
                # Standard fields
                "findings": [],
                "iterations": source_results.get("iterations", 0),
                "questions_by_iteration": source_results.get(
                    "questions_by_iteration", {}
                ),
                "formatted_findings": "No sources found to organize into topics.",
                "current_knowledge": "",
                "all_links_of_system": self.all_links_of_system,
                # Topic-specific fields
                "topics": [],
                "topic_graph": self.topic_graph.to_dict(),
                "source_count": 0,
            }

        self._update_progress(
            f"Starting to organize {len(all_sources)} sources into topics...",
            40,
            {"phase": "topic_extraction", "source_count": len(all_sources)},
        )

        # Extract topics from sources
        logger.info(
            f"Starting topic extraction with {len(all_sources)} sources"
        )
        topics = self._extract_topics_from_sources(all_sources, query)
        logger.info(
            f"Extracted {len(topics)} topics from {len(all_sources)} sources"
        )

        # Log topic details
        total_organized = 0
        for topic in topics:
            topic_sources = len(topic.get_all_sources())
            total_organized += topic_sources
            logger.info(f"Topic '{topic.title}' has {topic_sources} sources")

        # Calculate sources lost/deleted during organization
        sources_lost = len(all_sources) - total_organized
        if sources_lost > 0:
            logger.warning(
                f"SOURCE LOSS: {sources_lost} sources not organized ({total_organized}/{len(all_sources)}) - likely deleted as irrelevant"
            )
        else:
            logger.info(
                f"All sources organized successfully: {total_organized} sources in {len(topics)} topics"
            )

        # Track initial state (not included in LLM context)
        self.iteration_history.append(
            {
                "iteration": 0,
                "type": "initial",
                "question": query,
                "topics_count": len(topics),
                "sources_count": len(all_sources),
                "source_based_iterations": source_results.get("iterations", 0),
                "questions_by_iteration": source_results.get(
                    "questions_by_iteration", {}
                ),
                "topics": [
                    {"title": t.title, "sources": len(t.get_all_sources())}
                    for t in topics
                ]
                if topics
                else [],
            }
        )

        if not topics:
            logger.warning("Could not extract topics from sources")
            return {
                # Standard fields
                "findings": [],
                "iterations": source_results.get("iterations", 0),
                "questions_by_iteration": source_results.get(
                    "questions_by_iteration", {}
                ),
                "formatted_findings": "Could not identify distinct topics from sources.",
                "current_knowledge": "",
                "all_links_of_system": self.all_links_of_system,
                # Topic-specific fields
                "topics": [],
                "topic_graph": self.topic_graph.to_dict(),
                "source_count": len(all_sources),
            }

        self._update_progress(
            f"Validating {len(topics)} topics",
            60,
            {"phase": "validation", "topic_count": len(topics)},
        )

        # Validate topics based on source count
        validated_topics = []
        for topic in topics:
            source_count = len(topic.get_all_sources())

            # Accept topics with enough sources
            if source_count >= self.min_sources_per_topic:
                validated_topics.append(topic)
                self.topic_graph.add_topic(topic)
                logger.info(
                    f"Topic '{topic.title}' accepted with {source_count} sources"
                )
            else:
                logger.info(
                    f"Topic '{topic.title}' rejected with only {source_count} sources"
                )

        self._update_progress(
            "Finding topic relationships", 80, {"phase": "relationship_mapping"}
        )

        # Find relationships between topics
        self._find_topic_relationships(validated_topics)

        self._update_progress(
            "Evaluating topic relevance to query",
            85,
            {"phase": "relevance_filtering"},
        )

        # Filter topics by relevance to the original query
        relevant_topics = validated_topics  # Skip relevance filtering
        logger.info(
            f"Filtered from {len(validated_topics)} to {len(relevant_topics)} relevant topics"
        )

        # Refinement loop - can do multiple iterations
        while (
            self.enable_refinement
            and len(self.refinement_questions) < self.max_refinement_iterations
        ):
            refinement_question = self._generate_refinement_question(
                relevant_topics, query
            )

            if not refinement_question:
                # No more refinements needed
                logger.info(
                    "No further refinement needed - stopping refinement loop"
                )
                break

            logger.info(
                f"Starting refinement iteration {len(self.refinement_questions)} with question: {refinement_question}"
            )

            self._update_progress(
                f"Refining topics with follow-up search (iteration {len(self.refinement_questions) + 1}/{self.max_refinement_iterations})",
                85
                + (
                    10
                    * len(self.refinement_questions)
                    / self.max_refinement_iterations
                ),
                {"phase": "refinement", "question": refinement_question},
            )

            # Search for more sources using the refinement question
            combined_query = f"{query} {refinement_question}"

            logger.info(
                f"Before refinement: {len(self.all_links_of_system)} sources in all_links_of_system"
            )

            # Create a no-op citation handler for refinement strategy
            class NoTextCitationHandler:
                def analyze_followup(
                    self, query, results, previous_knowledge, nr_of_links
                ):
                    return {"content": "", "documents": []}

            # Create a NEW source-based strategy with an empty list for isolation
            # This ensures it starts fresh and doesn't interfere with our existing sources
            refinement_source_strategy = SourceBasedSearchStrategy(
                search=self.search,
                model=self.model,
                citation_handler=NoTextCitationHandler(),  # Use no-op handler to skip text generation
                include_text_content=False,  # Also disable text content retrieval
                use_cross_engine_filter=False,  # Keep same settings as main strategy
                filter_reorder=self.filter_reorder,
                filter_reindex=self.filter_reindex,
                cross_engine_max_results=self.cross_engine_max_results,
                all_links_of_system=[],  # Start with EMPTY list for isolation
                settings_snapshot=self.settings_snapshot,
                search_original_query=self.search_original_query,
            )

            # Wrap in preserve_research_context to maintain context
            @preserve_research_context
            def _call_refinement_search(q):
                return refinement_source_strategy.analyze_topic(q)

            refinement_results = _call_refinement_search(combined_query)

            # Get the new sources from the isolated refinement search
            new_sources = refinement_results.get("all_links_of_system", [])
            logger.info(f"Refinement search found {len(new_sources)} sources")

            # Update indices for new sources to continue from our existing count
            existing_count = len(
                self.all_links_of_system
            )  # Track count before adding new sources
            start_index = existing_count
            for i, source in enumerate(new_sources):
                source["index"] = str(start_index + i + 1)

            # Add ALL sources from refinement to our all_links_of_system
            # The citation handler will handle any duplicates properly
            self.all_links_of_system.extend(new_sources)
            logger.info(
                f"After refinement: {len(self.all_links_of_system)} total sources"
            )

            if new_sources:
                logger.info(
                    f"Found {len(new_sources)} new sources from refinement iteration {len(self.refinement_questions)}"
                )

                # Add new sources to existing topics or create new ones
                updated_topics = self._extract_topics_from_sources(
                    new_sources, query, existing_topics=relevant_topics
                )

                # Track this refinement iteration (not included in LLM context)
                self.iteration_history.append(
                    {
                        "iteration": len(self.refinement_questions),
                        "type": "refinement",
                        "question": refinement_question,
                        "new_sources": len(new_sources),
                        "total_sources": len(self.all_links_of_system),
                        "sources_before_refinement": existing_count,
                        "sources_after_refinement": len(
                            self.all_links_of_system
                        ),
                        "topics_before": len(relevant_topics),
                        "topics_after": len(updated_topics),
                        "new_topics_added": len(updated_topics)
                        - len(relevant_topics),
                        "topics": [
                            {
                                "title": t.title,
                                "sources": len(t.get_all_sources()),
                            }
                            for t in updated_topics
                        ],
                        "source_based_iterations": refinement_results.get(
                            "iterations", 0
                        ),
                        "refinement_found_sources": refinement_results.get(
                            "all_links_of_system", []
                        )
                        is not None,
                    }
                )

                # Re-select lead sources after adding new sources
                logger.info("Re-evaluating lead sources for all topics")
                self._reselect_lead_sources(updated_topics)

                # Reorganize sources across topics based on new lead sources
                logger.info("Reorganizing sources across topics")
                updated_topics = self._reorganize_topics(updated_topics)

                # Re-validate and filter
                validated_refined = []
                for topic in updated_topics:
                    if (
                        len(topic.get_all_sources())
                        >= self.min_sources_per_topic
                    ):
                        validated_refined.append(topic)

                relevant_topics = validated_refined  # Skip relevance filtering
                logger.info(
                    f"After refinement iteration {len(self.refinement_questions)}: {len(relevant_topics)} relevant topics"
                )
            else:
                logger.info(
                    f"No new sources found in refinement iteration {len(self.refinement_questions)}"
                )

        self._update_progress(
            "Finalizing topic organization",
            95,
            {"phase": "finalizing", "final_topic_count": len(relevant_topics)},
        )

        # Collect all sources from relevant topics only
        # These are the ONLY sources we want to use for citations
        topic_sources = []
        for topic in relevant_topics:
            topic_sources.extend(topic.get_all_sources())

        # Ensure all sources have 'link' field for compatibility
        for source in topic_sources:
            if "url" in source and "link" not in source:
                source["link"] = source["url"]

        # Generate text AFTER re-indexing sources
        if self.generate_text and self.citation_handler:
            self._update_progress(
                "Generating text synthesis from topics",
                96,
                {"phase": "text_generation"},
            )
            generated_text = self._generate_topic_based_text(
                relevant_topics, query
            )
        else:
            generated_text = ""

        # Format the topic graph as a structured knowledge representation
        # Include iteration history if we have refinements
        topic_graph_str = self._format_topic_graph_as_knowledge(
            relevant_topics, query
        )

        # Add iteration history to the topic graph output
        if self.iteration_history and len(self.iteration_history) > 1:
            history_lines = ["\n## Research Refinement History"]
            for hist in self.iteration_history:
                if hist["type"] == "initial":
                    history_lines.append(
                        f"\n**Initial Search:** {hist['sources_count']} sources → {hist['topics_count']} topics"
                    )
                else:
                    history_lines.append(
                        f'\n**Refinement {hist["iteration"]}:** "{hist["question"]}"'
                    )
                    history_lines.append(
                        f"  - Added {hist['new_sources']} new sources"
                    )
                    history_lines.append(
                        f"  - Topics: {hist['topics_before']} → {hist['topics_after']}"
                    )
            topic_graph_str += "\n".join(history_lines)

        self._update_progress(
            "Topic organization complete",
            100,
            {"phase": "complete", "final_topic_count": len(relevant_topics)},
        )

        # Create findings with sources included in content
        findings = []

        # If text generation is enabled, add the generated answer first
        if generated_text:
            answer_finding = {
                "phase": "Research Answer",
                "content": generated_text,
                "question": query,
                "search_results": topic_sources,  # Changed from documents to search_results for format_findings
            }
            findings.append(answer_finding)

        # Always add the topic organization overview
        topic_finding = {
            "phase": "Topic Organization",
            "content": topic_graph_str,
            "question": query,
            "search_results": topic_sources,  # Changed from documents to search_results
            "topics": [topic.to_dict() for topic in relevant_topics],
        }
        findings.append(topic_finding)

        # Add individual findings for each topic with their sources
        for i, topic in enumerate(relevant_topics, 1):
            topic_finding = {
                "phase": f"Topic {i}: {topic.title}",
                "content": self._format_single_topic_with_sources(topic),
                "question": query,
                "search_results": topic.get_all_sources(),  # Changed from documents to search_results
            }
            findings.append(topic_finding)

        # Combine source strategy questions with our refinement questions
        questions_by_iteration = dict(
            source_results.get("questions_by_iteration", {})
        )

        # Add our refinement questions if any
        if self.refinement_questions:
            # Find the highest iteration number from source strategy
            max_iter = (
                max(questions_by_iteration.keys())
                if questions_by_iteration
                else 0
            )

            # Add refinement questions as subsequent iterations
            for i, ref_question in enumerate(self.refinement_questions, 1):
                questions_by_iteration[max_iter + i] = [
                    f"Topic Refinement: {ref_question}"
                ]

        # Set questions by iteration for proper formatting
        self.findings_repository.set_questions_by_iteration(
            questions_by_iteration
        )

        # Format findings properly with sources using findings repository
        # Combine generated text with topic findings to include refinement history
        topic_findings_text = self._format_topic_findings(
            relevant_topics, query
        )
        if generated_text:
            synthesized_content = f"{generated_text}\n\n{topic_findings_text}"
        else:
            synthesized_content = topic_findings_text
        formatted_findings = self.findings_repository.format_findings_to_text(
            findings, synthesized_content
        )

        # Use generated text as current_knowledge if available (for benchmarks/API compatibility)
        # Otherwise fall back to topic graph structure
        current_knowledge = (
            generated_text if generated_text else topic_graph_str
        )

        return {
            # Standard fields expected by the system
            "findings": findings,
            "iterations": source_results.get("iterations", 0)
            + len(self.refinement_questions),
            "questions_by_iteration": questions_by_iteration,  # Now includes refinement questions
            "formatted_findings": formatted_findings,
            "current_knowledge": current_knowledge,  # Generated answer or topic graph
            "all_links_of_system": self.all_links_of_system,
            # Topic-specific fields
            "topics": [topic.to_dict() for topic in relevant_topics],
            "topic_graph": self.topic_graph.to_dict(),
            "topic_graph_str": topic_graph_str,  # Keep topic graph available separately
            "source_count": len(topic_sources),
            # Iteration history (not included in LLM context, just for display)
            "iteration_history": self.iteration_history,
            "refinement_questions": self.refinement_questions,
        }

    def _format_single_topic_with_sources(self, topic: Topic) -> str:
        """
        Format a single topic with all its sources listed.
        """
        lines = [
            f"**Total Sources:** {len(topic.get_all_sources())}",
            "",
            "**Lead Source:**",
            f"- [{topic.lead_source.get('title', 'Untitled')}]({topic.lead_source.get('link', '')})",
        ]

        # Add snippet from lead source if available
        if topic.lead_source.get("snippet"):
            lines.append(f"  > {topic.lead_source.get('snippet', '')}")

        if topic.supporting_sources:
            lines.append("")
            lines.append(
                f"**Supporting Sources ({len(topic.supporting_sources)}):**"
            )
            for source in topic.supporting_sources:
                title = source.get("title", "Untitled")
                link = source.get("link", "")
                lines.append(f"- [{title}]({link})")
                # Add snippet for supporting sources too
                if source.get("snippet"):
                    lines.append(f"  > {source.get('snippet', '')}")

        return "\n".join(lines)

    def _format_topic_graph_as_knowledge(
        self, topics: List[Topic], query: str
    ) -> str:
        """
        Format the topic graph as a structured knowledge representation.
        This becomes the main knowledge output of the strategy.
        """
        if not topics:
            return "No topics identified."

        lines = [
            f"# Topic Graph for: {query}",
            "\n## Overview",
            f"Identified {len(topics)} distinct topic clusters from {sum(len(t.get_all_sources()) for t in topics)} sources.\n",
            "## Topic Structure\n",
        ]

        # Build the graph structure
        for i, topic in enumerate(topics, 1):
            lines.append(f"### Topic {i}: {topic.title}")
            lines.append(f"**Sources:** {len(topic.get_all_sources())}")

            # Lead source details
            lead = topic.lead_source
            lines.append("\n**Lead Source:**")
            lines.append(
                f"[{lead.get('title', 'Untitled')}]({lead.get('link', '')})"
            )
            if lead.get("snippet"):
                lines.append(f"> {lead.get('snippet', '')}")

            # Relationships
            if topic.related_topic_ids:
                lines.append("\n**Related Topics:** ")
                for related_id in topic.related_topic_ids:
                    related_topic = self.topic_graph.get_topic(related_id)
                    if related_topic:
                        lines.append(f"- {related_topic.title}")

            lines.append("")  # Blank line between topics

        # Add graph relationships summary
        if len(topics) > 1:
            lines.append("\n## Topic Relationships\n")
            relationships = []
            for topic in topics:
                for related_id in topic.related_topic_ids:
                    related = self.topic_graph.get_topic(related_id)
                    if (
                        related
                        and (related.title, topic.title) not in relationships
                    ):
                        relationships.append((topic.title, related.title))

            if relationships:
                for t1, t2 in relationships:
                    lines.append(f"- **{t1}** ↔ **{t2}**")

        return "\n".join(lines)

    def _format_topic_findings(self, topics: List[Topic], query: str) -> str:
        """
        Format topic organization results for display with full source listings.
        """
        if not topics:
            return "No topics identified."

        lines = [
            f"# Topic Organization for: {query}",
            f"\nIdentified {len(topics)} distinct topics:\n",
        ]

        for i, topic in enumerate(topics, 1):
            lines.append(f"## Topic {i}: {topic.title}")
            lines.append(
                f"Sources: {len(topic.get_all_sources())} (1 lead + {len(topic.supporting_sources)} supporting)"
            )

            # Lead source with link and snippet
            lead = topic.lead_source
            lines.append("\n**Lead Source:**")
            lines.append(
                f"- [{lead.get('title', 'Untitled')}]({lead.get('link', '')})"
            )
            if lead.get("snippet"):
                lines.append(f"  > {lead.get('snippet', '')}")

            # All supporting sources with links and snippets
            if topic.supporting_sources:
                lines.append(
                    f"\n**Supporting Sources ({len(topic.supporting_sources)}):**"
                )
                for source in topic.supporting_sources:
                    lines.append(
                        f"- [{source.get('title', 'Untitled')}]({source.get('link', '')})"
                    )
                    if source.get("snippet"):
                        lines.append(f"  > {source.get('snippet', '')}")

            # Related topics
            if topic.related_topic_ids:
                related_titles = []
                for related_id in topic.related_topic_ids:
                    related = self.topic_graph.get_topic(related_id)
                    if related:
                        related_titles.append(related.title)
                if related_titles:
                    lines.append(
                        f"\n**Related Topics:** {', '.join(related_titles)}"
                    )

            lines.append("")  # Blank line between topics

        # Add source summary at the end
        total_sources = sum(len(topic.get_all_sources()) for topic in topics)
        lines.append(f"\n---\n**Total Sources Organized:** {total_sources}")

        # Add iteration history if we have refinements
        if self.iteration_history and len(self.iteration_history) > 1:
            lines.append("\n## Research Refinement History")
            for hist in self.iteration_history:
                if hist["type"] == "initial":
                    lines.append(
                        f"\n**Initial Search:** {hist['sources_count']} sources → {hist['topics_count']} topics"
                    )
                else:
                    lines.append(
                        f'\n**Refinement {hist["iteration"]}:** "{hist["question"]}"'
                    )
                    lines.append(f"  - Added {hist['new_sources']} new sources")
                    lines.append(
                        f"  - Topics: {hist['topics_before']} → {hist['topics_after']}"
                    )

        return "\n".join(lines)

    def _generate_topic_based_text(
        self, topics: List[Topic], query: str
    ) -> str:
        """
        Generate text synthesis from topics using citation handler with custom prompts.
        Two-part generation: individual topic paragraphs first, then summary based on them.
        """
        if not topics or not self.citation_handler:
            return ""

        generated_texts = []
        topic_sections = []  # Store generated topic sections for summary

        # Part 1: Generate individual topic paragraphs first
        for i, topic in enumerate(topics):
            # Get lead sources from OTHER topics for context
            other_leads_info = []
            for j, other_topic in enumerate(topics):
                if i != j:
                    other_lead = other_topic.lead_source
                    other_leads_info.append(
                        f"- Topic {j + 1} ({other_topic.title}): {other_lead.get('snippet', '')}"
                    )

            # Get ALL sources from this topic
            topic_sources = []
            topic_sources.append(topic.lead_source)
            topic_sources.extend(topic.supporting_sources)

            # Create documents and format using re-indexed sources
            # Pass nr_of_links=0 since we just re-indexed all sources
            topic_documents = self.citation_handler._create_documents(
                topic_sources, nr_of_links=0
            )
            formatted_topic_sources = self.citation_handler._format_sources(
                topic_documents
            )

            topic_prompt = f"""CURRENT TOPIC: {topic.title}
SOURCE SNIPPETS FOR THIS TOPIC (partial content):
{formatted_topic_sources}

OTHER TOPICS BEING COVERED (for context - avoid repeating):
{chr(10).join(other_leads_info) if other_leads_info else "None"}

Write a SHORT paragraph (2-3 sentences) based on these SNIPPETS that:
1. DIRECTLY ANSWERS aspects of the research question using this topic's sources
2. Focuses on what's UNIQUE about this topic's contribution
3. Works with the partial information available in the snippets
4. Uses citations for claims found in the snippets
5. Acknowledges if key details are not in the snippets but describes what the sources appear to cover

Stay anchored to the research question and highlight this topic's specific insights.
Remember: These are snippets, not full articles.

RESEARCH QUESTION TO ANSWER: {query}"""

            try:
                topic_response = self.model.invoke(topic_prompt)
                topic_text = (
                    topic_response.content
                    if hasattr(topic_response, "content")
                    else str(topic_response)
                )

                # Store section for summary generation
                topic_sections.append(f"{topic.title}: {topic_text}")
                # Add topic header for output
                generated_texts.append(f"\n**{topic.title}:**")
                generated_texts.append(topic_text)
            except Exception:
                logger.exception(
                    f"Error generating text for topic '{topic.title}'"
                )

        # Part 2: Generate comprehensive summary based on the topic sections
        if topic_sections:
            # Get lead sources for citations
            lead_sources = []
            for topic in topics:
                lead_sources.append(topic.lead_source)

            lead_documents = self.citation_handler._create_documents(
                lead_sources
            )
            formatted_lead_sources = self.citation_handler._format_sources(
                lead_documents
            )

            summary_prompt = f"""SOURCE SNIPPETS (for citation reference):
{formatted_lead_sources}

TOPIC SECTIONS ALREADY GENERATED:
{chr(10).join(topic_sections)}

Based on the topic sections above, write ONE comprehensive introductory paragraph that:
1. DIRECTLY ANSWERS the research question by synthesizing insights from ALL topic sections
2. Focuses on the main answer to the question first
3. Integrates key findings from the different topics into a coherent response
4. Uses citations [1], [2], etc. for specific claims

IMPORTANT: Focus on ANSWERING THE QUESTION directly using the information from the topic sections.

RESEARCH QUESTION TO ANSWER: {query}"""

            try:
                summary_response = self.model.invoke(summary_prompt)
                summary_text = (
                    summary_response.content
                    if hasattr(summary_response, "content")
                    else str(summary_response)
                )
                # Insert summary at the beginning
                generated_texts.insert(0, summary_text)
            except Exception:
                logger.exception("Error generating summary synthesis")

        return "\n".join(generated_texts)
