"""
swanki/pipeline/pipeline.py
[[swanki.pipeline.pipeline]]
https://github.com/Mjvolk3/Swanki/tree/main/swanki/pipeline/pipeline.py
Test file: tests/test_pipeline.py

Main processing pipeline orchestrating PDF-to-Anki conversion.
"""

import logging
import os
import random
import re
from pathlib import Path
from typing import Any

from dotenv import load_dotenv
from pydantic import ValidationError

# Import audio utilities
from ..audio import (
    generate_card_audio,
    generate_lecture_audio,
    generate_reading_audio,
    generate_summary_audio,
)
from ..llm.agents import (
    audio_feedback_agent,
    card_feedback_agent,
    card_gen_agent,
    document_summary_agent,
    get_model_string,
    text_agent,
)
from ..models import (
    AudioTranscriptFeedback,
    CardFeedback,
    CardGenerationResponse,
    DocumentSummary,
    ImageSummary,
    PlainCard,
    ProcessingState,
    RefinementHistory,
)

# Import new processing modules
from ..processing import (
    AnkiProcessor,
    ApkgExporter,
    ImageProcessor,
    MarkdownCleaner,
    PDFProcessor,
)
from ..processing.markdown_cleaner import _natural_sort_key

# Import content utilities
from ..utils.content import (
    detect_math_content,
    extract_images_from_markdown,
)
from .segmenter import (
    build_segment_to_page_map,
    combine_markdown_files,
    split_into_segments,
    write_segment_files,
)

logger = logging.getLogger(__name__)


class Pipeline:
    """Main processing pipeline with configuration-driven workflow.

    Orchestrates the complete PDF to Anki card conversion process,
    managing all intermediate steps and outputs. Supports various
    configuration options for customizing the processing behavior.

    Parameters
    ----------
    config : Dict[str, Any]
        Hydra configuration dictionary containing all processing options

    Attributes:
    ----------
    config : Dict[str, Any]
        Configuration dictionary
    state : ProcessingState or None
        Current processing state tracker
    data_dir : Path
        Base directory for output data
    output_base : Path
        Current output directory for this run
    citation_key : str
        Citation key for the current document

    Methods:
    -------
    process_full(pdf_path, citation_key)
        Run the complete processing pipeline
    split_pdf(pdf_path)
        Split PDF into individual pages
    convert_to_markdown(pages)
        Convert PDF pages to markdown
    clean_markdown(markdown_files)
        Clean and format markdown files
    process_images(markdown_files)
        Extract and summarize images
    generate_document_summary(markdown_files, image_summaries)
        Generate comprehensive document summary
    generate_image_cards(markdown_files, doc_summary, ...)
        Generate cards from images
    generate_outputs(cards, summary, output_dir)
        Generate output files in various formats
    generate_audio(cards, summary, outputs, cleaned_files, image_summaries)
        Generate audio files for cards and content
    send_to_anki(cards, outputs, anki_config)
        Send cards to Anki via AnkiConnect

    Examples:
    --------
    >>> config = {
    ...     'pipeline': {'processing': {'window_size': 2}},
    ...     'audio': {'audio': {'generate_complementary': True}},
    ...     'anki': {'anki': {'enabled': True, 'auto_send': True}}
    ... }
    >>> pipeline = Pipeline(config)
    >>> outputs = pipeline.process_full(
    ...     pdf_path=Path("research_paper.pdf"),
    ...     citation_key="Johnson2024"
    ... )
    """

    def __init__(self, config: dict[str, Any]):
        """Initialize the pipeline with configuration.

        Parameters
        ----------
        config : Dict[str, Any]
            Hydra configuration dictionary with processing options

        Notes:
        -----
        Loads environment variables including SWANKI_DATA for output directory.
        Loads environment variables and initializes state.
        """
        self.config = config
        self.state: ProcessingState | None = None

        # Load environment variables
        load_dotenv()
        self.data_dir = Path(os.getenv("SWANKI_DATA", "swanki-out"))

    def process_full(
        self, pdf_path: Path, citation_key: str, output_dir: str | None = None
    ) -> dict[str, Path]:
        """Process PDF through the complete pipeline.

        Executes all processing steps from PDF to final outputs. When
        ``config["mode"]`` is ``"audio_only"``, card generation, output
        writing, APKG export, and Anki sending are skipped — only shared
        stages (PDF split, markdown, images, summary) and audio run.

        Parameters
        ----------
        pdf_path : Path
            Path to the input PDF file
        citation_key : str
            Citation key for naming outputs and referencing
        output_dir : str, optional
            Name for the output directory. If not provided, uses citation_key

        Returns:
        -------
        Dict[str, Path]
            Dictionary mapping output types to file paths:
            - 'cards_plain': Plain markdown cards
            - 'cards_audio': Cards with audio links (if enabled)
            - 'summary': Document summary

        Raises:
        ------
        RuntimeError
            If PDF conversion fails or no markdown content generated

        Examples:
        --------
        >>> pipeline = Pipeline(config)
        >>> outputs = pipeline.process_full(
        ...     pdf_path=Path("/docs/paper.pdf"),
        ...     citation_key="Einstein1905"
        ... )
        >>> print(outputs['cards_plain'])
        PosixPath('swanki-out/Einstein1905/cards-plain.md')
        """
        # Initialize state
        self.state = ProcessingState(
            pdf_path=pdf_path, citation_key=citation_key, current_stage="initialization"
        )
        assert self.state is not None  # Set above; helps mypy narrow type

        # Create output directory based on output_dir or citation key with auto-increment if exists
        base_name = (
            output_dir
            if output_dir
            else (citation_key if citation_key else "swanki-out")
        )
        output_path = self.data_dir / base_name

        # Store base_name for audio file naming
        # Use only the last component if output_dir contains slashes
        if output_dir and "/" in output_dir:
            self.audio_prefix = Path(base_name).name
        else:
            self.audio_prefix = base_name

        # If directory exists, append a number
        if output_path.exists():
            counter = 0
            while True:
                numbered_dir = self.data_dir / f"{base_name}_{counter}"
                if not numbered_dir.exists():
                    output_path = numbered_dir
                    break
                counter += 1

        self.output_base = output_path
        self.output_base.mkdir(parents=True, exist_ok=True)

        # Log output directory for easy navigation
        logger.info(f"Output directory: {self.output_base.absolute()}")

        # 1. Split PDF based on config
        self.state.current_stage = "pdf_split"
        pages = self.split_pdf(pdf_path)

        # 2. Convert to markdown
        self.state.current_stage = "markdown_conversion"
        markdown_files = self.convert_to_markdown(pages)

        # Check if conversion was successful - fail fast if not
        if not markdown_files:
            raise RuntimeError(
                "PDF to markdown conversion failed. Cannot proceed without markdown content."
            )

        # 3. Clean markdown
        self.state.current_stage = "markdown_cleaning"
        cleaned_files = self.clean_markdown(markdown_files)

        # 4. Process images
        self.state.current_stage = "image_processing"
        image_summaries = self.process_images(cleaned_files)

        # 5. Generate document summary (EARLY!)
        self.state.current_stage = "summary_generation"
        doc_summary = self.generate_document_summary(cleaned_files, image_summaries)
        self.state.document_summary = doc_summary

        # Branch based on mode
        mode = self.config.get("mode", "full")

        if mode == "audio_only":
            # Skip card generation stages (5.5-8)
            logger.info("Running in audio_only mode — skipping card generation")
            all_cards: list[PlainCard] = []
            outputs: dict[str, Path] = {}
            self.citation_key = citation_key
        else:
            pipeline_config = self.config.get("pipeline", {})
            processing_config = pipeline_config.get("processing", {})

            # 5.5. Segmentation (char mode recombines pages into uniform segments)
            segmentation_mode = processing_config.get("segmentation", "page")

            if segmentation_mode == "char":
                self.state.current_stage = "segmentation"
                char_config = processing_config.get("char_segmentation", {})
                combined_text, page_offsets = combine_markdown_files(cleaned_files)
                segment_tuples = split_into_segments(
                    combined_text, char_config.get("target_chars", 2000)
                )
                segment_dir = self.output_base / "segments"
                segment_files = write_segment_files(segment_tuples, segment_dir)
                segment_to_pages = build_segment_to_page_map(
                    page_offsets,
                    [(s, e) for _, s, e in segment_tuples],
                    len(cleaned_files),
                )
                text_card_files = segment_files
            else:
                text_card_files = cleaned_files
                segment_to_pages = None

            # 5.6. Estimate card count
            estimated_cards = self.estimate_card_count(
                text_card_files, image_summaries, processing_config
            )

            # Check if user confirmation is required
            confirm_required = processing_config.get("confirm_before_generation", True)
            if confirm_required:
                logger.info(
                    f"Proceeding with card generation (estimated: {estimated_cards} cards)"
                )
                response = (
                    input(
                        f"\nContinue with card generation? ({estimated_cards} cards estimated) [Y/n]: "
                    )
                    .strip()
                    .lower()
                )
                if response and response not in ["y", "yes", ""]:
                    logger.info("Card generation cancelled by user")
                    raise KeyboardInterrupt("User cancelled card generation")
            else:
                logger.info(
                    f"Proceeding with card generation (estimated: {estimated_cards} cards)"
                )

            # 6. Generate cards with document-order interleaving
            self.state.current_stage = "card_generation"
            all_cards = []

            # Get image card config
            image_config = processing_config.get("image_cards", {})
            image_cards_enabled = image_config.get("enabled", True)
            cards_per_seg = processing_config.get("cards_per_segment", 3)

            last_image_page = -1
            for seg_idx, seg_file in enumerate(text_card_files):
                # Text cards for this segment
                seg_cards = self._generate_cards_for_segment(
                    seg_idx,
                    text_card_files,
                    doc_summary,
                    context_radius=processing_config.get("context_radius", 1),
                    num_cards=cards_per_seg,
                )
                all_cards.extend(seg_cards)

                # Image cards — interleaved in document order
                if image_cards_enabled:
                    if segment_to_pages is not None:
                        pages_for_seg = segment_to_pages[seg_idx]
                    else:
                        pages_for_seg = [seg_idx]

                    for page_idx in pages_for_seg:
                        if page_idx > last_image_page:
                            page_image_cards = self._generate_image_cards_for_page(
                                cleaned_files[page_idx],
                                doc_summary,
                                image_summaries,
                                cards_per_image=image_config.get("cards_per_image", 3),
                                image_on_front=image_config.get("image_on_front", True),
                                image_on_back=image_config.get("image_on_back", True),
                                require_math=image_config.get(
                                    "require_math_content", False
                                ),
                                placement_strategy=image_config.get(
                                    "placement_strategy", "smart"
                                ),
                                front_back_ratio=image_config.get(
                                    "front_back_ratio", 0.5
                                ),
                            )
                            all_cards.extend(page_image_cards)
                            last_image_page = page_idx

            # 7. Store citation key for later use
            self.citation_key = citation_key

            self.state.cards_generated = len(all_cards)

            # 8. Generate outputs based on config
            self.state.current_stage = "output_generation"
            outputs = self.generate_outputs(all_cards, doc_summary, self.output_base)

        # 9. Generate audio if configured
        audio_config = self.config.get("audio", {}).get("audio", {})
        any_audio = any(
            [
                audio_config.get("generate_complementary", False),
                audio_config.get("generate_summary", False),
                audio_config.get("generate_reading", False),
                audio_config.get("generate_lecture", False),
            ]
        )
        if mode == "audio_only" and not any_audio:
            logger.warning(
                "audio_only mode with no audio types enabled produces no output."
            )
        if any_audio:
            self.state.current_stage = "audio_generation"
            self.generate_audio(
                all_cards, doc_summary, outputs, cleaned_files, image_summaries
            )

        # 9b. Re-export .apkg with audio if audio cards were generated
        output_config = self.config.get("output", {}).get("output", {})
        if output_config.get("create_anki_deck", False) and "cards_audio" in outputs:
            anki_cfg = self.config.get("anki", {}).get("anki", {})
            deck_template = anki_cfg.get("deck_name", "{deck_name}")
            deck_name_value = (
                self.audio_prefix
                if hasattr(self, "audio_prefix")
                else self.citation_key
            )
            deck_name = self.format_deck_name(deck_template, deck_name_value)
            apkg_path = self.output_base / f"{self.citation_key}.apkg"
            exporter = ApkgExporter(deck_name)
            exporter.export_from_file(outputs["cards_audio"], apkg_path)
            outputs["apkg"] = apkg_path

        # 10. Send to Anki if configured
        anki_config = self.config.get("anki", {}).get("anki", {})
        if anki_config.get("enabled", False) and anki_config.get("auto_send", False):
            self.state.current_stage = "anki_sending"
            self.send_to_anki(all_cards, outputs, anki_config)

        self.state.outputs = outputs
        return outputs

    def split_pdf(self, pdf_path: Path) -> list[Path]:
        """Split PDF into individual pages.

        Uses PDFProcessor to split a multi-page PDF into separate
        single-page PDF files for easier processing.

        Parameters
        ----------
        pdf_path : Path
            Path to the input PDF file

        Returns:
        -------
        List[Path]
            List of paths to individual page PDFs

        See Also:
        --------
        PDFProcessor : Handles PDF splitting operations
        """
        pdf_processor = PDFProcessor(self.output_base)
        pdf_files = pdf_processor.split_pdf(pdf_path)
        return pdf_files

    def convert_to_markdown(self, pages: list[Path]) -> list[Path]:
        """Convert PDF pages to markdown format.

        Converts individual PDF pages to markdown using the Mathpix
        service. Handles conversion errors gracefully and logs progress.

        Parameters
        ----------
        pages : List[Path]
            List of single-page PDF files to convert

        Returns:
        -------
        List[Path]
            List of paths to generated markdown files

        Raises:
        ------
        RuntimeError
            If no pages could be converted successfully

        Notes:
        -----
        Uses os.system for better TTY handling with the mpx command.
        Creates output in 'md-singles' subdirectory.
        """
        # Create output directory
        md_singles_dir = self.output_base / "md-singles"
        md_singles_dir.mkdir(parents=True, exist_ok=True)

        markdown_files = []

        logger.info(f"Converting {len(pages)} PDF pages to markdown")

        # Convert each page individually (like the legacy approach)
        for page_pdf in pages:
            # Create corresponding markdown filename
            md_filename = page_pdf.stem + ".md"  # page-1.pdf -> page-1.md
            md_path = md_singles_dir / md_filename

            # Use os.system approach which handles clearLine errors better
            cmd = f"mpx convert '{page_pdf}' '{md_path}'"

            logger.debug(f"Converting {page_pdf.name} to {md_path.name}")

            try:
                # Use os.system - it handles TTY issues better than subprocess
                exit_code = os.system(cmd)

                if exit_code == 0 and md_path.exists() and md_path.stat().st_size > 0:
                    logger.debug(f"Successfully converted {page_pdf.name}")
                    markdown_files.append(md_path)
                else:
                    logger.warning(
                        f"Failed to convert {page_pdf.name} - exit code: {exit_code}"
                    )

            except Exception as e:
                logger.error(f"Error converting {page_pdf.name}: {e}")

        if not markdown_files:
            raise RuntimeError("Failed to convert any PDF pages to markdown.")

        logger.info(f"Successfully converted {len(markdown_files)} pages to markdown")
        # Use natural sorting to ensure page-2.md comes before page-10.md
        return sorted(markdown_files, key=_natural_sort_key)

    def clean_markdown(self, markdown_files: list[Path]) -> list[Path]:
        """Clean and standardize markdown files.

        Applies various cleaning operations to markdown files including
        removing artifacts, fixing formatting, and standardizing structure.

        Parameters
        ----------
        markdown_files : List[Path]
            List of markdown files to clean

        Returns:
        -------
        List[Path]
            List of paths to cleaned markdown files

        See Also:
        --------
        MarkdownCleaner : Handles markdown cleaning operations
        """
        cleaner = MarkdownCleaner(self.output_base)
        cleaned_files = cleaner.clean_all_markdown_files()
        return cleaned_files

    def process_images(self, markdown_files: list[Path]) -> list[ImageSummary]:
        """Extract and summarize images from markdown.

        Processes all images found in markdown files, generating
        AI-powered summaries and extracting relevant metadata.

        Parameters
        ----------
        markdown_files : List[Path]
            List of markdown files to process

        Returns:
        -------
        List[ImageSummary]
            List of image summaries with metadata

        See Also:
        --------
        ImageProcessor : Handles image extraction and summarization
        ImageSummary : Data model for image information
        """
        # Initialize processor with model string
        models_config = self.config.get("models", {}).get("models", {})
        llm_config = models_config.get("llm", {})
        model_string = get_model_string(llm_config)
        processor = ImageProcessor(self.output_base, model_string)

        # Process all images
        try:
            image_infos = processor.process_all_images()
        except RuntimeError as e:
            logger.error(f"Failed to process images: {e}")
            raise  # Re-raise to fail the pipeline

        # Convert to ImageSummary objects
        image_summaries = []
        images_without_summaries = []

        for idx, info in enumerate(image_infos):
            if "summary" in info:
                try:
                    image_summary = ImageSummary(
                        image_url=info["url"],
                        summary=info["summary"],
                        alt_text=info.get("alt_text", ""),
                        page_idx=idx,
                        context=info.get("context", ""),
                    )
                    image_summaries.append(image_summary)
                except ValidationError as e:
                    logger.warning(
                        f"Image summary validation error for image {idx}: {e}"
                    )
                    # Truncate the summary to fit within the limit
                    words = info["summary"].split()
                    truncated_summary = (
                        " ".join(words[:295]) + "..."
                        if len(words) > 300
                        else info["summary"]
                    )

                    try:
                        image_summary = ImageSummary(
                            image_url=info["url"],
                            summary=truncated_summary,
                            alt_text=info.get("alt_text", ""),
                            page_idx=idx,
                            context=info.get("context", ""),
                        )
                        image_summaries.append(image_summary)
                        logger.info(
                            f"Successfully added truncated summary for image {idx}"
                        )
                    except Exception as e2:
                        logger.error(
                            f"Failed to add image {idx} even with truncation: {e2}"
                        )
                        images_without_summaries.append(info["url"])
            else:
                images_without_summaries.append(info["url"])

        # Fail if any images don't have summaries
        if images_without_summaries:
            error_msg = f"The following {len(images_without_summaries)} images do not have summaries: {', '.join(images_without_summaries)}"
            logger.error(error_msg)
            raise RuntimeError(error_msg)

        return image_summaries

    def generate_document_summary(
        self, markdown_files: list[Path], image_summaries: list[ImageSummary]
    ) -> DocumentSummary:
        """Generate comprehensive document summary.

        Creates a structured summary of the entire document including
        title, authors, key contributions, methodology, and definitions
        of acronyms and technical terms.

        Parameters
        ----------
        markdown_files : List[Path]
            List of cleaned markdown files
        image_summaries : List[ImageSummary]
            List of image summaries to include

        Returns:
        -------
        DocumentSummary
            Structured document summary with metadata

        Notes:
        -----
        Uses configured prompts and LLM model for generation.
        Summary must be 200-500 words as validated by DocumentSummary.

        See Also:
        --------
        DocumentSummary : Data model for document summaries
        """
        # Combine all markdown content
        combined_content = "\\n\\n".join([f.read_text() for f in markdown_files])

        # Format image summaries
        image_summary_text = "\\n".join(
            [f"Image {img.page_idx}: {img.summary}" for img in image_summaries]
        )

        # Get prompts from config (note the nested structure)
        prompts_config = self.config.get("prompts", {}).get("prompts", {})
        summary_prompts = prompts_config.get("summary", {})
        system_prompt = summary_prompts.get(
            "system",
            "You are an expert at creating concise, informative summaries of academic documents.",
        )
        user_prompt = summary_prompts.get(
            "document_summary",
            """Create a comprehensive summary of this document.
Focus on:
1. Main thesis and key contributions
2. All acronyms and their full forms
3. Technical terms that need clear definitions
4. Methodology and approach
5. Key findings

Document content:
{content}

Image summaries:
{image_summaries}""",
        )

        # Generate summary using pydantic-ai
        models_config = self.config.get("models", {}).get("models", {})
        llm_config = models_config.get("llm", {})
        result = document_summary_agent.run_sync(
            user_prompt.format(
                content=combined_content, image_summaries=image_summary_text
            ),
            instructions=system_prompt,
            model=get_model_string(llm_config),
        )

        return result.output

    def _detect_math_density(self, content: str) -> float:
        """Detect the density of mathematical content in text.

        Returns a score from 0.0 to 1.0 indicating math density.
        """
        # Count various math indicators
        math_patterns = [
            r"\$[^$]+\$",  # Inline math
            r"\$\$[^$]+\$\$",  # Display math
            r"\\[a-zA-Z]+\{",  # LaTeX commands
            r"\\frac\{",  # Fractions
            r"\\sum",  # Summations
            r"\\int",  # Integrals
            r"\\mathbf",  # Bold math
            r"[=<>≤≥≠]",  # Math operators
            r"\^{",  # Superscripts
            r"_{",  # Subscripts
            r"\\theta|\\alpha|\\beta|\\gamma",  # Greek letters
            r"theorem|proof|lemma|corollary",  # Math terminology
            r"equation|formula|function",  # More math terms
        ]

        total_matches = 0
        for pattern in math_patterns:
            matches = re.findall(pattern, content, re.IGNORECASE)
            total_matches += len(matches)

        # Calculate density (normalized by content length)
        words = len(content.split())
        if words == 0:
            return 0.0

        # Normalize to 0-1 range (assume 1 math element per 20 words is high density)
        density = min(1.0, total_matches / (words / 20))
        return density

    def _generate_cards_for_segment(
        self,
        seg_idx: int,
        segment_files: list[Path],
        doc_summary: DocumentSummary,
        context_radius: int,
        num_cards: int,
    ) -> list[PlainCard]:
        """Generate cards for a single segment with context from surrounding segments.

        Works identically for page-mode segments (clean-md-singles) and
        char-mode segments (segments/).

        Parameters
        ----------
        seg_idx : int
            Index of the focal segment in segment_files
        segment_files : List[Path]
            All segment files (pages or char-segments)
        doc_summary : DocumentSummary
            Document summary for context
        context_radius : int
            Number of segments before/after focal segment for context
        num_cards : int
            Number of cards to generate for this segment

        Returns:
        -------
        List[PlainCard]
            Generated cards for this segment
        """
        # Determine context window
        start_idx = max(0, seg_idx - context_radius)
        end_idx = min(len(segment_files), seg_idx + context_radius + 1)

        # Get focal segment
        focal_page = segment_files[seg_idx]
        focal_content = focal_page.read_text()

        # Get context segments
        context_pages = []
        for idx in range(start_idx, end_idx):
            if idx != seg_idx:
                context_pages.append(segment_files[idx])

        # Build content with clear separation
        if context_pages:
            context_content = "\n\n".join([f.read_text() for f in context_pages])
            combined_content = f"[CONTEXT FROM SURROUNDING PAGES]\n{context_content}\n\n[FOCAL PAGE CONTENT]\n{focal_content}"
        else:
            combined_content = focal_content

        # Get config values
        models_config = self.config.get("models", {}).get("models", {})
        llm_config = models_config.get("llm", {})
        processing_config = self.config.get("pipeline", {}).get("processing", {})

        # Detect math density and adjust card count
        math_density = self._detect_math_density(focal_content)
        adjusted_num_cards = num_cards
        adjusted_cloze_cards = processing_config.get("cloze_per_segment", 2)

        if math_density > 0.5:  # High math density
            adjusted_num_cards = int(num_cards * 1.5)  # 50% more cards
            adjusted_cloze_cards = int(adjusted_cloze_cards * 1.5)
            logger.info(
                f"High math density ({math_density:.2f}) detected on segment {seg_idx + 1}, increasing cards from {num_cards} to {adjusted_num_cards}"
            )
        elif math_density > 0.3:  # Medium math density
            adjusted_num_cards = int(num_cards * 1.25)  # 25% more cards
            logger.info(
                f"Medium math density ({math_density:.2f}) detected on segment {seg_idx + 1}, increasing cards from {num_cards} to {adjusted_num_cards}"
            )

        # Debug logging
        logger.debug(
            f"Processing segment {seg_idx + 1}/{len(segment_files)} with context radius {context_radius}"
        )
        logger.debug(f"Math density: {math_density:.2f}")
        logger.debug(
            f"Requesting {adjusted_num_cards} regular cards and {adjusted_cloze_cards} cloze cards"
        )

        # Generate regular cards first
        regular_prompt = f"""Generate EXACTLY {adjusted_num_cards} regular Q&A flashcards from the main content.

Context from document summary:
Title: {doc_summary.title}
Acronyms: {doc_summary.acronyms}
Technical terms: {doc_summary.technical_terms}

Content provided:
{combined_content}

IMPORTANT STRUCTURAL NOTES (FOR YOUR REFERENCE ONLY):
- The content may include markers like [CONTEXT FROM SURROUNDING PAGES] and [FOCAL PAGE CONTENT]
- These are ONLY organizational markers to help you understand the content structure
- Generate cards from the main content section (after [FOCAL PAGE CONTENT] if present)
- NEVER mention these markers, "focal page", "surrounding pages", or "context" in your cards
- Cards should be about the actual subject matter, not about document structure

FORMAT RULES:
1. Use ## for the question (front of card)
2. Answer goes on the next line (back of card)
3. Tags go as a single bullet: - #tag1, #tag2, #tag3
4. NO CLOZE CARDS - only regular Q&A format
5. NEVER use generic tags like #equation, #definition - use conceptual tags like #causal-inference.dag

CRITICAL REQUIREMENTS:
1. Cards MUST be self-contained - students won't have access to the paper, figures, or other references
2. NEVER reference external content: "According to [12]", "As shown in Figure 3", "The paper states"
3. NEVER say "in the context of the document" or "as described in the document"
4. NEVER mention "focal page", "surrounding pages", "context", or any structural markers in cards
5. NEVER ask about specific authors: "What is Lachapelle et al.'s method?" - ask about the METHOD itself
6. NEVER ask "What does X stand for?" - ask what X IS or HOW it works instead
7. Each card tests ONE concept and includes all context needed to understand it
8. PRIORITIZE creating cards for EVERY mathematical equation, formula, or algorithm in the content
9. CRITICAL LaTeX FORMATTING:
   - ALL mathematical variables, symbols, and expressions MUST be wrapped in LaTeX delimiters
   - Use $ for inline math (e.g., $W$, $X_j$, $h(W) = 0$)
   - Use $$ for display math equations
   - NEVER write bare mathematical symbols without LaTeX (WRONG: W, X_j, h(W)=0)
   - Common mistakes to AVOID:
     * Writing "W" instead of "$W$" when referring to a matrix
     * Writing "X_j" instead of "$X_j$" for subscripted variables
     * Writing "h(W) = 0" instead of "$h(W) = 0$" for equations
     * Writing "α" instead of "$\\alpha$" for Greek letters
     * Writing "W_{{ji}} ≠ 0" instead of "$W_{{ji}} \\neq 0$"
10. NEVER use LaTeX tables (\\\\begin{{tabular}})
11. For EVERY math symbol ($h$, $F$, $g_j$, etc.), define what it represents IN THE CARD

MATHEMATICAL CONTENT PRIORITY (CRITICAL):
- SCAN the content for ALL mathematical elements: equations, formulas, definitions, proofs, theorems
- Create AT LEAST 2-3 cards for EVERY equation in the content
- Mathematical card types to include:
  * DEFINITION: "What does equation X represent/mean?"
  * COMPONENTS: "In equation X, what does symbol Y represent?"
  * PROPERTIES: "What property does equation X satisfy?"
  * PROOF STEPS: "What is the key insight in step N of the proof?"
  * APPLICATIONS: "How is equation X used in practice?"
  * RELATIONSHIPS: "How does equation X relate to equation Y?"
- For key equations (like $E(\\mathbf{{w}})=\\frac{{1}}{{2}} \\sum_{{n=1}}^N\\{{y(x_n, \\mathbf{{w}})-t_n\\}}^2$), create 4-5 cards
- Even simple equations deserve multiple cards (e.g., for $h(W) = 0$: meaning, purpose, constraints)
- PROOFS: Break down each proof into step-by-step cards
- THEOREMS: Create cards for statement, conditions, implications

Generate {adjusted_num_cards} regular Q&A cards now. Focus on the actual subject matter, not document structure."""

        regular_result = card_gen_agent.run_sync(
            regular_prompt,
            instructions="Generate ONLY regular Q&A flashcards. Do NOT create any cloze deletion cards.\n\nCRITICAL LENGTH REQUIREMENT:\n- Card answers (back) MUST be under 500 characters (HARD LIMIT - validation will fail otherwise)\n- Aim for 200-400 characters for optimal learning\n- Be concise and focus on key points only\n- Remove verbose explanations and unnecessary words\n\nCRITICAL: ALL mathematical variables, symbols, and expressions MUST be wrapped in LaTeX delimiters using $ for inline math. For example: $W$, $X_j$, $h(W) = 0$, $W_{ji} \\neq 0$. NEVER write bare mathematical symbols without LaTeX.",
            model=get_model_string(llm_config),
        )
        regular_response = regular_result.output

        # Generate cloze cards separately (only if requested)
        if adjusted_cloze_cards > 0:
            cloze_prompt = f"""Generate EXACTLY {adjusted_cloze_cards} cloze deletion flashcards from the main content.

Content provided:
{combined_content}

IMPORTANT STRUCTURAL NOTES (FOR YOUR REFERENCE ONLY):
- Markers like [CONTEXT FROM SURROUNDING PAGES] and [FOCAL PAGE CONTENT] are only for organization
- Generate cards from the main content (after [FOCAL PAGE CONTENT] if present)
- NEVER mention these markers or "focal page" in your cards

FORMAT: Each card MUST use {{{{c1::hidden text}}}} syntax for cloze deletions.

CRITICAL RULES:
1. Cards MUST be self-contained - no references to "the paper", figures, or external content
2. CLOZE LENGTH: Hide 1-5 words maximum per cloze deletion (focus on key terms/concepts)
3. For equations: Hide PARTS of equations (variables, operators), not entire equations
4. Hide meaningful concepts, not arbitrary numbers or references
5. EVERY card MUST have AT LEAST 2 meaningful tags on the "- #tag1, #tag2" line
6. PRIORITIZE equations - create cloze cards for key mathematical expressions
7. MATH IN CLOZE: When hiding parts of math expressions, the ENTIRE math notation must stay together
8. PARENTHESES: Minimize parenthetical phrases - use them ONLY when absolutely necessary
   - BAD: "The method (which is efficient) improves (by 50%) the performance"
   - GOOD: "The efficient method improves performance by 50%"
   - ACCEPTABLE: "O(n log n)" or "f(x)" - parentheses are part of notation
   - WRONG: "In {{{{c1::$E[X_j$}}}} | {{{{c2::$X_{{pa(j)}}]$}}}}" (splits the equation)
   - WRONG: "The model {{{{c1::$E[X_j | X_{{}}}} {{{{c2::pa}}}} {{{{c3::(j)}}}}$" (splits subscript)
   - RIGHT: "In the model $E[X_j | {{{{c1::X_{{pa(j)}}}}}}] = {{{{c2::g_j(f_j(X))}}}}$"
   - RIGHT: "The equation {{{{c1::$E[X_j | X_{{pa(j)}}] = g_j(f_j(X))$}}}} represents conditional expectation"

MATHEMATICAL CONTENT FOCUS (CRITICAL):
- PRIORITIZE mathematical content: equations, formulas, theorems, proofs
- For EVERY equation/formula, create at least one cloze card
- CRITICAL CLOZE RULE: Use ONLY {{{{c1::}}}} - NEVER use c2, c3, etc.
- If you need to hide multiple concepts, create SEPARATE cards instead
- Mathematical cloze patterns (ONE cloze per card):
  * Hide key variables: "In $E(\\mathbf{{w}}) = \\frac{{1}}{{2}} \\sum_{{n=1}}^N ...$, {{{{c1::E}}}} represents error function"
  * Hide operators/functions: "The gradient descent rule uses $\\theta = \\theta {{{{c1::- α∇L(θ)}}}}$"
  * Hide mathematical properties: "The constraint $h(W) = 0$ ensures {{{{c1::acyclicity}}}} of the DAG"
  * Hide proof steps: "To prove convergence, we show that {{{{c1::||∇E|| < ε}}}}"
- Keep mathematical notation intact - don't split subscripts, superscripts, or operators
- Examples (SINGLE c1 only): 
  * "The error function {{{{c1::$E(\\mathbf{{w}})$}}}} equals $\\frac{{1}}{{2}} \\sum_{{n=1}}^N \\{{y(x_n, \\mathbf{{w}})-t_n\\}}^2$"
  * "In optimization, {{{{c1::gradient descent}}}} uses the update rule $\\theta = \\theta - α∇L(θ)$"

GOOD Examples:
## The {{{{c1::Pythagorean theorem}}}} states that $a^2 + b^2 = c^2$ for right triangles.

- #mathematics.geometry, #theorems.pythagorean

## In gradient descent, the update rule is $\\theta = \\theta - {{{{c1::α∇L(θ)}}}}$ where α is the learning rate.

- #optimization.gradient-descent, #machine-learning.algorithms

## The error function $E(\\mathbf{{w}}) = \\frac{{1}}{{2}} \\sum_{{n=1}}^N \\{{y(x_n, \\mathbf{{w}}) - t_n\\}}^2$ uses {{{{c1::squared error}}}} to measure {{{{c2::prediction accuracy}}}}.

- #machine-learning.loss-functions, #optimization.least-squares

BAD Examples (AVOID):
- "The original method uses {{{{c1::algorithm X}}}}" (What original method? AND missing tags!)
- "{{{{c1::$E[X|Y] = g(f(X))$}}}}" (Don't hide entire equations AND missing tags!)
- "In the context of the document, {{{{c1::h(W) = 0}}}} ensures acyclicity" (Remove "in the context"!)
- "{{{{c1::NAS}}}} is used for model optimization" (Undefined acronym!)
- Tags: #equation, #definition (Generic tags - use conceptual ones!)

REMEMBER: EVERY cloze card MUST have tags just like regular cards!
Generate {adjusted_cloze_cards} cloze cards now. Focus on key definitions, formulas, and facts from the actual content."""

            cloze_result = card_gen_agent.run_sync(
                cloze_prompt,
                instructions="Generate ONLY cloze deletion flashcards using {{c1::text}} syntax.\n\nCRITICAL LENGTH REQUIREMENT:\n- Cloze card BACKS should be MINIMAL (ideally empty, max 100 characters)\n- Front content (full text with cloze) must stay under 500 characters total\n- Keep content concise - cloze cards are meant to be brief\n\nCRITICAL LaTeX RULES:\n1. ALL mathematical variables and expressions MUST use LaTeX with $ delimiters\n2. When math is NOT inside cloze markers, wrap it properly: $W$, $X_j$, $h(W) = 0$\n3. When math IS inside cloze markers, still use LaTeX: {{c1::$E = mc^2$}}\n4. NEVER write bare math symbols without LaTeX (WRONG: W, X_j, W_{ji})",
                model=get_model_string(llm_config),
            )
            cloze_response = cloze_result.output
        else:
            # No cloze cards requested - set to None
            cloze_response = None

        # Apply self-refine if enabled
        refinement_config = self.config.get("refinement", {}).get("refinement", {})
        if refinement_config.get("enabled", False):
            logger.info("Applying self-refine to improve card quality...")

            # Refine regular cards
            if regular_response.cards and "regular" in refinement_config.get(
                "content_types", ["regular", "cloze"]
            ):
                regular_response = self._self_refine_cards(
                    regular_response, doc_summary, "regular"
                )

            # Refine cloze cards (only if they were generated)
            if (
                cloze_response
                and cloze_response.cards
                and "cloze"
                in refinement_config.get("content_types", ["regular", "cloze"])
            ):
                cloze_response = self._self_refine_cards(
                    cloze_response, doc_summary, "cloze"
                )

        # Combine responses
        all_cards = regular_response.cards + (
            cloze_response.cards if cloze_response else []
        )

        # Debug: Check what cards were generated
        regular_cards = [c for c in all_cards if "{{c" not in c.front.text]
        cloze_cards = [c for c in all_cards if "{{c" in c.front.text]
        logger.debug(
            f"Generated: {len(regular_cards)} regular cards, {len(cloze_cards)} cloze cards"
        )

        # Check for math content
        math_cards = [c for c in all_cards if "$" in c.front.text or "$" in c.back.text]
        logger.debug(f"Math cards: {len(math_cards)}")

        # Check for references
        ref_cards = [
            c
            for c in all_cards
            if any(
                ref in c.front.text.lower() or ref in c.back.text.lower()
                for ref in ["ref.", "reference", "according to", "["]
            )
        ]
        if ref_cards:
            logger.debug(
                f"{len(ref_cards)} cards contain references that should have been removed"
            )

        return all_cards

    def _generate_image_cards_for_page(
        self,
        markdown_file: Path,
        doc_summary: DocumentSummary,
        all_image_summaries: list[ImageSummary],
        cards_per_image: int = 3,
        image_on_front: bool = True,
        image_on_back: bool = True,
        require_math: bool = False,
        placement_strategy: str = "smart",
        front_back_ratio: float = 0.5,
    ) -> list[PlainCard]:
        """Generate image cards for a single page.

        This is a helper method that processes images from one page at a time,
        used by the main pipeline to interleave text and image cards.

        Parameters
        ----------
        markdown_file : Path
            The markdown file to process
        doc_summary : DocumentSummary
            Document summary for context
        all_image_summaries : List[ImageSummary]
            All image summaries from the document
        cards_per_image : int, optional
            Number of cards per image (default 3)
        image_on_front : bool, optional
            Allow images on card fronts (default True)
        image_on_back : bool, optional
            Allow images on card backs (default True)
        require_math : bool, optional
            Only process images with math content (default False)
        placement_strategy : str, optional
            Strategy for image placement (default "smart")
        front_back_ratio : float, optional
            Ratio for random placement (default 0.5)

        Returns:
        -------
        List[PlainCard]
            Generated image cards for this page
        """
        image_cards = []

        content = markdown_file.read_text()

        # Extract images from this file
        images = extract_images_from_markdown(content, markdown_file.parent)

        for image_info in images:
            # Check if math content is required and present
            if require_math and not detect_math_content(image_info["context"]):
                continue

            logger.debug(
                f"Generating {cards_per_image} cards for image: {image_info['path']}"
            )

            # Get surrounding content (more context around the image)
            file_content = content

            # Generate prompts for image cards
            prompts_config = self.config.get("prompts", {}).get("prompts", {})
            cards_prompts = prompts_config.get("cards", {})
            models_config = self.config.get("models", {}).get("models", {})
            llm_config = models_config.get("llm", {})

            # Find matching image summary if available
            image_summary_text = None
            if all_image_summaries:
                # Try to match by URL/path
                for img_summary in all_image_summaries:
                    # Check if paths match (handle different path formats)
                    if (
                        image_info["path"] in img_summary.image_url
                        or img_summary.image_url in image_info["path"]
                        or Path(image_info["path"]).name
                        == Path(img_summary.image_url).name
                    ):
                        image_summary_text = img_summary.summary
                        logger.debug(
                            f"Found image summary for {image_info['path']}: {image_summary_text[:50]}..."
                        )
                        break

                if not image_summary_text:
                    logger.debug(
                        f"No matching image summary found for {image_info['path']}"
                    )
                    logger.debug(
                        f"  Available summaries: {[img.image_url for img in all_image_summaries]}"
                    )

            # Create a specialized prompt for image cards
            image_prompt = f"""Create {cards_per_image} educational flashcards based on this image and its context.

Image Information:
- Path: {image_info["path"]}
- Alt text: {image_info["alt"]}
- Context: {image_info["context"]}
{f"- Image Description: {image_summary_text}" if image_summary_text else ""}

Document Summary Context:
- Title: {doc_summary.title}
- Acronyms: {doc_summary.acronyms}
- Technical terms: {doc_summary.technical_terms}

Full content from this page:
{file_content}

CRITICAL RULES:
- DO NOT include ![Image](...) markdown in your response - the image will be added automatically
- DO NOT put image links in the question or answer text
- Just write the question and answer text referring to "the image" or "this figure"

Requirements:
- Each card should test UNDERSTANDING of concepts, not just ask to describe the image
- Questions should focus on WHY, HOW, or WHAT implications the image shows
- BAD: "How are X and Y represented in this image?" (just asks for description)
- GOOD: "What relationship does the image show between X and Y?" (tests understanding)
- GOOD: "Why does the pathway split into two branches in the diagram?" (tests conceptual understanding)
- Cards can reference specific parts of the image (e.g., "What does the upper pathway represent?")
- Include mathematical notation if present in the image
- Make questions test conceptual understanding, not just visual identification
- {"Image can appear on front or back of card as appropriate" if image_on_front and image_on_back else "Image should appear on front of card" if image_on_front else "Image should appear on back of card" if image_on_back else "No image placement preference"}

FORMAT RULES:
1. Use ## for the question (front of card) - NO IMAGE MARKDOWN
2. Answer goes on the next line (back of card) - NO IMAGE MARKDOWN
3. Tags go as a single bullet: - #tag1, #tag2, #tag3
4. Use period-delimited tags from broad to narrow (e.g., #biology.cell-biology, #machine-learning.neural-networks)
5. DO NOT include figure/image tags like #figures, #diagrams, #graphs - the card already has an image
6. NEVER include ![Image](...) in your text - images are handled separately
7. CRITICAL: Image cards must NEVER be cloze cards - use Q&A format only
   - WRONG: "The diagram shows {{c1::three stages}} of cell division"
   - RIGHT: "How many stages of cell division are shown in the diagram?"

Example card for an image:
## What relationship between learning rate and convergence is shown in this graph?

The graph demonstrates that smaller learning rates lead to slower but more stable convergence, while larger learning rates can cause oscillations or divergence. The optimal learning rate (shown in green) balances speed and stability.

- #optimization.learning-rate, #machine-learning.training, #convergence-analysis"""

            # Generate cards using pydantic-ai
            try:
                img_result = card_gen_agent.run_sync(
                    image_prompt,
                    instructions=cards_prompts.get(
                        "system",
                        "You are an expert at creating educational flashcards that test understanding of visual content.",
                    ),
                    model=get_model_string(llm_config),
                )
                response = img_result.output

                # Apply self-refine if enabled for image cards
                refinement_config = self.config.get("refinement", {}).get(
                    "refinement", {}
                )
                if refinement_config.get(
                    "enabled", False
                ) and "image" in refinement_config.get(
                    "content_types", ["regular", "cloze", "image"]
                ):
                    logger.info(
                        f"Applying self-refine to image cards for {image_info['path']}..."
                    )
                    response = self._self_refine_cards(response, doc_summary, "image")

                # Add image paths to the generated cards
                for idx, card in enumerate(response.cards):
                    # Clean up any image markdown that might have slipped through
                    # Remove ![Image](...) patterns from both front and back text
                    import re

                    image_pattern = r"!\[.*?\]\([^)]+\)"
                    card.front.text = re.sub(image_pattern, "", card.front.text).strip()
                    card.back.text = re.sub(image_pattern, "", card.back.text).strip()

                    # CRITICAL: Ensure image cards are not cloze cards
                    if "{{c" in card.front.text or "{{c" in card.back.text:
                        logger.error(
                            f"Image card generated as cloze card, skipping: {card.front.text[:50]}"
                        )
                        continue  # Skip this card entirely

                    # Decide where to place the image based on config
                    if image_on_front and not image_on_back:
                        card.front.image_path = image_info["original_path"]
                        if image_summary_text:
                            card.front.image_summary = image_summary_text
                        else:
                            logger.error(
                                f"No image summary found for {image_info['path']} - skipping this image card"
                            )
                            continue  # Skip this card entirely
                    elif image_on_back and not image_on_front:
                        card.back.image_path = image_info["original_path"]
                        if image_summary_text:
                            card.back.image_summary = image_summary_text
                        else:
                            logger.error(
                                f"No image summary found for {image_info['path']} - skipping this image card"
                            )
                            continue  # Skip this card entirely
                    elif image_on_front and image_on_back:
                        # Both allowed, but NEVER put the same image on both sides
                        place_on_front = False

                        if placement_strategy == "smart":
                            # Place on front if question references the image
                            place_on_front = any(
                                word in card.front.text.lower()
                                for word in [
                                    "figure",
                                    "image",
                                    "graph",
                                    "chart",
                                    "diagram",
                                    "show",
                                    "illustrate",
                                    "depict",
                                    "visual",
                                ]
                            )
                        elif placement_strategy == "alternate":
                            # Alternate between front and back
                            place_on_front = idx % 2 == 0
                        elif placement_strategy == "random":
                            # Random with specified ratio
                            place_on_front = random.random() < front_back_ratio
                        elif placement_strategy == "prefer_front":
                            place_on_front = True
                        elif placement_strategy == "prefer_back":
                            place_on_front = False

                        # Place image on ONLY ONE side
                        if place_on_front:
                            card.front.image_path = image_info["original_path"]
                            if image_summary_text:
                                card.front.image_summary = image_summary_text
                            else:
                                logger.error(
                                    f"No image summary found for {image_info['path']} - skipping this image card"
                                )
                                continue  # Skip this card entirely
                            # Explicitly ensure IMAGE is not on the back
                            card.back.image_path = None
                        else:
                            card.back.image_path = image_info["original_path"]
                            if image_summary_text:
                                card.back.image_summary = image_summary_text
                            else:
                                logger.error(
                                    f"No image summary found for {image_info['path']} - skipping this image card"
                                )
                                continue  # Skip this card entirely
                            # Explicitly ensure IMAGE is not on the front
                            card.front.image_path = None

                image_cards.extend(response.cards)

                # Debug output for image placement
                front_count = sum(1 for card in response.cards if card.front.image_path)
                back_count = sum(1 for card in response.cards if card.back.image_path)
                logger.debug(
                    f"Generated {len(response.cards)} cards for image {image_info['path']}"
                )
                logger.debug(
                    f"  Image placement - Front: {front_count}, Back: {back_count} (strategy: {placement_strategy})"
                )

            except Exception as e:
                logger.error(
                    f"Error generating cards for image {image_info['path']}: {e}"
                )
                continue

        return image_cards

    def generate_image_cards(
        self,
        markdown_files: list[Path],
        doc_summary: DocumentSummary,
        cards_per_image: int = 3,
        image_on_front: bool = True,
        image_on_back: bool = True,
        require_math: bool = False,
        placement_strategy: str = "smart",
        front_back_ratio: float = 0.5,
        image_summaries: list[ImageSummary] | None = None,
    ) -> list[PlainCard]:
        """Generate flashcards from document images.

        Creates cards that test understanding of visual content like
        figures, graphs, and diagrams. Supports various strategies
        for placing images on card fronts or backs.

        Parameters
        ----------
        markdown_files : List[Path]
            Markdown files containing image references
        doc_summary : DocumentSummary
            Document summary for context
        cards_per_image : int, optional
            Number of cards to generate per image (default is 3)
        image_on_front : bool, optional
            Allow images on card fronts (default is True)
        image_on_back : bool, optional
            Allow images on card backs (default is True)
        require_math : bool, optional
            Only process images with math content (default is False)
        placement_strategy : {'smart', 'alternate', 'random', 'prefer_front', 'prefer_back'}, optional
            Strategy for image placement (default is 'smart')
        front_back_ratio : float, optional
            Ratio for random placement strategy (default is 0.5)

        Returns:
        -------
        List[PlainCard]
            Generated image-based flashcards

        Notes:
        -----
        Image placement strategies:
        - 'smart': Place on front if question references the image
        - 'alternate': Alternate between front and back
        - 'random': Random placement with specified ratio
        - 'prefer_front': Always place on front
        - 'prefer_back': Always place on back

        Images are never placed on both sides of the same card.

        Examples:
        --------
        >>> # Generate 2 cards per image, smart placement
        >>> image_cards = pipeline.generate_image_cards(
        ...     markdown_files=files,
        ...     doc_summary=summary,
        ...     cards_per_image=2,
        ...     placement_strategy='smart'
        ... )
        """
        image_cards = []

        for file_path in markdown_files:
            content = file_path.read_text()

            # Extract images from this file
            images = extract_images_from_markdown(content, file_path.parent)

            for image_info in images:
                # Check if math content is required and present
                if require_math and not detect_math_content(image_info["context"]):
                    continue

                logger.debug(
                    f"Generating {cards_per_image} cards for image: {image_info['path']}"
                )

                # Get surrounding content (more context around the image)
                file_content = content

                # Generate prompts for image cards
                prompts_config = self.config.get("prompts", {}).get("prompts", {})
                cards_prompts = prompts_config.get("cards", {})
                models_config = self.config.get("models", {}).get("models", {})
                llm_config = models_config.get("llm", {})

                # Find matching image summary if available
                image_summary_text = None
                if image_summaries:
                    # Try to match by URL/path
                    for img_summary in image_summaries:
                        # Check if paths match (handle different path formats)
                        if (
                            image_info["path"] in img_summary.image_url
                            or img_summary.image_url in image_info["path"]
                            or Path(image_info["path"]).name
                            == Path(img_summary.image_url).name
                        ):
                            image_summary_text = img_summary.summary
                            logger.debug(
                                f"Found image summary for {image_info['path']}: {image_summary_text[:50]}..."
                            )
                            break

                    if not image_summary_text:
                        logger.debug(
                            f"No matching image summary found for {image_info['path']}"
                        )
                        logger.debug(
                            f"  Available summaries: {[img.image_url for img in image_summaries]}"
                        )

                # Create a specialized prompt for image cards
                image_prompt = f"""Create {cards_per_image} educational flashcards based on this image and its context.

Image Information:
- Path: {image_info["path"]}
- Alt text: {image_info["alt"]}
- Context: {image_info["context"]}
{f"- Image Description: {image_summary_text}" if image_summary_text else ""}

Document Summary Context:
- Title: {doc_summary.title}
- Acronyms: {doc_summary.acronyms}
- Technical terms: {doc_summary.technical_terms}

Full content from this page:
{file_content}

CRITICAL RULES:
- DO NOT include ![Image](...) markdown in your response - the image will be added automatically
- DO NOT put image links in the question or answer text
- Just write the question and answer text referring to "the image" or "this figure"

Requirements:
- Each card should test UNDERSTANDING of concepts, not just ask to describe the image
- Questions should focus on WHY, HOW, or WHAT implications the image shows
- BAD: "How are X and Y represented in this image?" (just asks for description)
- GOOD: "What relationship does the image show between X and Y?" (tests understanding)
- GOOD: "Why does the pathway split into two branches in the diagram?" (tests conceptual understanding)
- Cards can reference specific parts of the image (e.g., "What does the upper pathway represent?")
- Include mathematical notation if present in the image
- Make questions test conceptual understanding, not just visual identification
- {"Image can appear on front or back of card as appropriate" if image_on_front and image_on_back else "Image should appear on front of card" if image_on_front else "Image should appear on back of card" if image_on_back else "No image placement preference"}

FORMAT RULES:
1. Use ## for the question (front of card) - NO IMAGE MARKDOWN
2. Answer goes on the next line (back of card) - NO IMAGE MARKDOWN
3. Tags go as a single bullet: - #tag1, #tag2, #tag3
4. Use period-delimited tags from broad to narrow (e.g., #biology.cell-biology, #machine-learning.neural-networks)
5. DO NOT include figure/image tags like #figures, #diagrams, #graphs - the card already has an image
6. NEVER include ![Image](...) in your text - images are handled separately
7. CRITICAL: Image cards must NEVER be cloze cards - use Q&A format only
   - WRONG: "The diagram shows {{c1::three stages}} of cell division"
   - RIGHT: "How many stages of cell division are shown in the diagram?"

Example card for an image:
## What relationship between learning rate and convergence is shown in this graph?

The graph demonstrates that smaller learning rates lead to slower but more stable convergence, while larger learning rates can cause oscillations or divergence. The optimal learning rate (shown in green) balances speed and stability.

- #optimization.learning-rate, #machine-learning.training, #convergence-analysis"""

                # Generate cards using pydantic-ai
                try:
                    img_result2 = card_gen_agent.run_sync(
                        image_prompt,
                        instructions=cards_prompts.get(
                            "system",
                            "You are an expert at creating educational flashcards that test understanding of visual content.",
                        ),
                        model=get_model_string(llm_config),
                    )
                    response = img_result2.output

                    # Apply self-refine if enabled for image cards
                    refinement_config = self.config.get("refinement", {}).get(
                        "refinement", {}
                    )
                    if refinement_config.get(
                        "enabled", False
                    ) and "image" in refinement_config.get(
                        "content_types", ["regular", "cloze", "image"]
                    ):
                        logger.info(
                            f"Applying self-refine to image cards for {image_info['path']}..."
                        )
                        response = self._self_refine_cards(
                            response, doc_summary, "image"
                        )

                    # Add image paths to the generated cards
                    for idx, card in enumerate(response.cards):
                        # Clean up any image markdown that might have slipped through
                        # Remove ![Image](...) patterns from both front and back text
                        import re

                        image_pattern = r"!\[.*?\]\([^)]+\)"
                        card.front.text = re.sub(
                            image_pattern, "", card.front.text
                        ).strip()
                        card.back.text = re.sub(
                            image_pattern, "", card.back.text
                        ).strip()

                        # Decide where to place the image based on config
                        if image_on_front and not image_on_back:
                            card.front.image_path = image_info["original_path"]
                            if image_summary_text:
                                card.front.image_summary = image_summary_text
                            else:
                                logger.error(
                                    f"No image summary found for {image_info['path']} - skipping this image card"
                                )
                                continue  # Skip this card entirely
                        elif image_on_back and not image_on_front:
                            card.back.image_path = image_info["original_path"]
                            if image_summary_text:
                                card.back.image_summary = image_summary_text
                            else:
                                logger.error(
                                    f"No image summary found for {image_info['path']} - skipping this image card"
                                )
                                continue  # Skip this card entirely
                        elif image_on_front and image_on_back:
                            # Both allowed, but NEVER put the same image on both sides
                            place_on_front = False

                            if placement_strategy == "smart":
                                # Place on front if question references the image
                                place_on_front = any(
                                    word in card.front.text.lower()
                                    for word in [
                                        "figure",
                                        "image",
                                        "graph",
                                        "chart",
                                        "diagram",
                                        "show",
                                        "illustrate",
                                        "depict",
                                        "visual",
                                    ]
                                )
                            elif placement_strategy == "alternate":
                                # Alternate between front and back
                                place_on_front = idx % 2 == 0
                            elif placement_strategy == "random":
                                # Random with specified ratio
                                place_on_front = random.random() < front_back_ratio
                            elif placement_strategy == "prefer_front":
                                place_on_front = True
                            elif placement_strategy == "prefer_back":
                                place_on_front = False

                            # Place image on ONLY ONE side
                            if place_on_front:
                                card.front.image_path = image_info["original_path"]
                                if image_summary_text:
                                    card.front.image_summary = image_summary_text
                                else:
                                    logger.error(
                                        f"No image summary found for {image_info['path']} - skipping this image card"
                                    )
                                    continue  # Skip this card entirely
                                # Explicitly ensure IMAGE is not on the back
                                card.back.image_path = None
                            else:
                                card.back.image_path = image_info["original_path"]
                                if image_summary_text:
                                    card.back.image_summary = image_summary_text
                                else:
                                    logger.error(
                                        f"No image summary found for {image_info['path']} - skipping this image card"
                                    )
                                    continue  # Skip this card entirely
                                # Explicitly ensure IMAGE is not on the front
                                card.front.image_path = None

                    image_cards.extend(response.cards)

                    # Debug output for image placement
                    front_count = sum(
                        1 for card in response.cards if card.front.image_path
                    )
                    back_count = sum(
                        1 for card in response.cards if card.back.image_path
                    )
                    logger.debug(
                        f"Generated {len(response.cards)} cards for image {image_info['path']}"
                    )
                    logger.debug(
                        f"  Image placement - Front: {front_count}, Back: {back_count} (strategy: {placement_strategy})"
                    )

                except Exception as e:
                    logger.error(
                        f"Error generating cards for image {image_info['path']}: {e}"
                    )
                    continue

        return image_cards

    def estimate_card_count(
        self,
        segment_files: list[Path],
        image_summaries: list[ImageSummary],
        processing_config: dict[str, Any],
    ) -> int:
        """Estimate the total number of cards that will be generated.

        Args:
            segment_files: Segment files (pages in page mode, char-segments
                in char mode).
            image_summaries: Image summaries from the document.
            processing_config: Processing configuration dict.

        Returns:
            Estimated total number of cards.
        """
        num_segments = len(segment_files)
        num_images = len(image_summaries)

        # Get configuration values
        context_radius = processing_config.get("context_radius", 1)
        cards_per_seg = processing_config.get("cards_per_segment", 3)
        cloze_per_seg = processing_config.get("cloze_per_segment", 2)

        # Image card settings
        image_config = processing_config.get("image_cards", {})
        image_cards_enabled = image_config.get("enabled", True)
        cards_per_image = image_config.get("cards_per_image", 3)

        # Calculate cards from text content
        cards_per_focal = cards_per_seg + cloze_per_seg
        text_cards = num_segments * cards_per_focal

        # Calculate cards from images
        image_cards = num_images * cards_per_image if image_cards_enabled else 0

        # Total estimate
        total_estimated = text_cards + image_cards

        # Log the estimation details
        logger.info("=" * 50)
        logger.info("CARD ESTIMATION:")
        logger.info(f"  Segments: {num_segments}")
        logger.info(f"  Images: {num_images}")
        logger.info("  Processing configuration:")
        logger.info(f"    - Context radius: ±{context_radius} segments")
        logger.info("  Cards per segment:")
        logger.info(f"    - Regular cards: {cards_per_seg}")
        logger.info(f"    - Cloze cards: {cloze_per_seg}")
        logger.info(f"    - Total per segment: {cards_per_focal}")
        logger.info(
            f"  Estimated text cards: {num_segments} segments × {cards_per_focal} cards = {text_cards}"
        )
        if image_cards_enabled:
            logger.info(
                f"  Estimated image cards: {num_images} images × {cards_per_image} cards = {image_cards}"
            )
        else:
            logger.info("  Image cards: disabled")
        logger.info(f"  TOTAL ESTIMATED CARDS: {total_estimated}")
        logger.info("=" * 50)

        return total_estimated

    def generate_outputs(
        self, cards: list[PlainCard], summary: DocumentSummary, output_dir: Path
    ) -> dict[str, Path]:
        """Generate output files in configured formats.

        Creates various output files including plain cards, cards with
        audio links, and document summary based on configuration.

        Parameters
        ----------
        cards : List[PlainCard]
            Generated flashcards
        summary : DocumentSummary
            Document summary
        output_dir : Path
            Directory for output files

        Returns:
        -------
        Dict[str, Path]
            Mapping of output types to file paths:
            - 'cards_plain': Plain markdown cards
            - 'cards_audio': Cards with audio (if enabled)
            - 'summary': Document summary

        Notes:
        -----
        Audio card output is only generated if complementary audio
        generation is enabled in configuration.
        """
        outputs = {}

        # Get output config
        output_config = self.config.get("output", {}).get("output", {})
        formats = output_config.get("formats", {})
        tag_format = output_config.get("tag_format", "slugified")

        # Plain cards (no audio)
        plain_path = output_dir / formats.get("cards_plain", "cards-plain.md")
        with open(plain_path, "w") as f:
            for card in cards:
                f.write(
                    card.to_md(
                        include_audio=False,
                        citation_key=self.citation_key,
                        tag_format=tag_format,
                    )
                )
        outputs["cards_plain"] = plain_path

        # Export .apkg alongside the markdown if configured
        if output_config.get("create_anki_deck", False):
            anki_config = self.config.get("anki", {}).get("anki", {})
            deck_template = anki_config.get("deck_name", "{deck_name}")
            deck_name_value = (
                self.audio_prefix
                if hasattr(self, "audio_prefix")
                else self.citation_key
            )
            deck_name = self.format_deck_name(deck_template, deck_name_value)
            apkg_path = output_dir / f"{self.citation_key}.apkg"
            exporter = ApkgExporter(deck_name)
            exporter.export_from_file(plain_path, apkg_path)
            outputs["apkg"] = apkg_path

        # Note: Cards with audio will be written after audio generation
        # This ensures the audio URIs are properly set on the cards
        # See generate_audio method for audio card generation

        # Document summary
        summary_path = output_dir / formats.get("summary", "document-summary.md")
        with open(summary_path, "w") as f:
            f.write(f"# {summary.title}\\n\\n")
            f.write(f"**Authors:** {', '.join(summary.authors)}\\n\\n")
            f.write(f"**Main Topic:** {summary.main_topic}\\n\\n")
            f.write(f"**Methodology:** {summary.methodology}\\n\\n")

            # Learning objectives if present
            if summary.learning_objectives:
                f.write("## Learning Objectives\\n\\n")
                for objective in summary.learning_objectives:
                    f.write(f"- {objective}\\n")
                f.write("\\n")

            f.write(f"## Summary\\n\\n{summary.summary}\\n\\n")

            # Key equations if present
            if summary.key_equations:
                f.write("## Key Equations\\n\\n")
                for equation in summary.key_equations:
                    f.write(f"- {equation}\\n")
                f.write("\\n")

            # Conceptual framework if present
            if summary.conceptual_framework:
                f.write(
                    f"## Conceptual Framework\\n\\n{summary.conceptual_framework}\\n\\n"
                )

            f.write("## Key Contributions\\n\\n")
            for contrib in summary.key_contributions:
                f.write(f"- {contrib}\\n")

            # Acronyms
            if summary.acronyms:
                f.write("\\n## Acronyms\\n\\n")
                for acronym, definition in summary.acronyms.items():
                    f.write(f"- **{acronym}**: {definition}\\n")

            # Technical terms
            if summary.technical_terms:
                f.write("\\n## Technical Terms\\n\\n")
                for term, definition in summary.technical_terms.items():
                    f.write(f"- **{term}**: {definition}\\n")
        outputs["summary"] = summary_path

        return outputs

    def generate_audio(
        self,
        cards: list[PlainCard],
        summary: DocumentSummary,
        outputs: dict[str, Path],
        cleaned_files: list[Path],
        image_summaries: list[ImageSummary],
    ):
        """Generate audio files for various content types.

        Creates audio files based on configuration including card audio,
        summary narration, full document reading, and educational lectures.

        Parameters
        ----------
        cards : List[PlainCard]
            Flashcards for audio generation
        summary : DocumentSummary
            Document summary for narration
        outputs : Dict[str, Path]
            Output file paths
        cleaned_files : List[Path]
            Cleaned markdown files for reading
        image_summaries : List[ImageSummary]
            Image summaries for lecture content

        Raises:
        ------
        RuntimeError
            If required API keys are not set in environment

        Notes:
        -----
        Requires ELEVEN_LABS_API_KEY and OPENAI_API_KEY environment
        variables. Generates different audio types based on config:
        - Complementary: Front/back audio for each card
        - Summary: Narration of document summary
        - Reading: Full document text-to-speech
        - Lecture: Educational presentation style

        See Also:
        --------
        utils.audio : Audio generation utility functions
        """
        audio_config = self.config.get("audio", {}).get("audio", {})

        # Get API keys
        elevenlabs_api_key = os.getenv("ELEVEN_LABS_API_KEY")
        if not elevenlabs_api_key:
            raise RuntimeError("ELEVEN_LABS_API_KEY not set in environment.")

        # Get model config
        models_config = self.config.get("models", {}).get("models", {})
        llm_config = models_config.get("llm", {})
        model = get_model_string(llm_config)

        # Get voice_id from TTS config in models, not from audio config
        tts_config = models_config.get("tts", {})
        voice_id = tts_config.get("voice_id")

        # Generate complementary audio (card audio)
        if audio_config.get("generate_complementary", False):
            if not cards:
                logger.warning(
                    "Complementary audio requires cards. Skipping in audio_only mode."
                )
            else:
                logger.info("Generating complementary audio...")
                audio_dir = self.output_base / "gen-md-complementary-audio"
                audio_dir.mkdir(exist_ok=True)

                for i, card in enumerate(cards):
                    # Use consistent card numbering instead of misleading page numbers
                    # This ensures audio files match the card order
                    card_base = "card"

                    # Get complementary audio speed from config
                    complementary_speed = audio_config.get("complementary_speed", 1.0)
                    # Get force regenerate option from config
                    force_regenerate_citation = audio_config.get(
                        "force_regenerate_citation", False
                    )

                    front_filename, back_filename = generate_card_audio(
                        card=card,
                        card_index=i + 1,
                        page_base=card_base,
                        audio_dir=audio_dir,
                        elevenlabs_api_key=elevenlabs_api_key,
                        voice_id=voice_id,
                        model=model,
                        citation_key=self.citation_key,
                        speed=complementary_speed,
                        force_regenerate_citation=force_regenerate_citation,
                    )

                    # Set audio URIs on the card for robust pairing
                    if front_filename:
                        # Use relative path from output directory
                        card.audio_front_uri = (
                            f"gen-md-complementary-audio/{front_filename}"
                        )
                    if back_filename:
                        card.audio_back_uri = (
                            f"gen-md-complementary-audio/{back_filename}"
                        )

                    # Validate audio transcript matches card content
                    if not card.validate_audio_match():
                        print(f"WARNING: Audio transcript mismatch for card {i + 1}")

                    if back_filename:
                        print(
                            f"Generated audio for card {i + 1}: {front_filename}, {back_filename}"
                        )
                    else:
                        print(
                            f"Generated audio for card {i + 1}: {front_filename} (front only)"
                        )

        # Generate summary audio
        if audio_config.get("generate_summary", False):
            logger.info("Generating summary audio...")
            summary_text = f"{summary.summary}\n\nKey Contributions:\n" + "\n".join(
                [f"- {contrib}" for contrib in summary.key_contributions]
            )
            summary_audio_path = (
                self.output_base / f"{self.audio_prefix}-summary-audio.mp3"
            )

            # Get summary audio speed from config
            summary_speed = audio_config.get("summary_speed", 1.0)

            generate_summary_audio(
                summary_text=summary_text,
                output_path=summary_audio_path,
                elevenlabs_api_key=elevenlabs_api_key,
                voice_id=voice_id,
                model=model,
                citation_key=self.citation_key,
                speed=summary_speed,
            )
            print(f"Generated summary audio: {summary_audio_path.name}")

        # Generate reading audio (full document)
        if audio_config.get("generate_reading", False):
            logger.info("Generating reading audio...")
            # Combine all cleaned markdown content
            full_content = "\n\n".join([f.read_text() for f in cleaned_files])
            reading_audio_path = (
                self.output_base / f"{self.audio_prefix}-reading-audio.mp3"
            )

            # Get reading audio speed from config
            reading_speed = audio_config.get("reading_speed", 1.0)

            generate_reading_audio(
                full_content=full_content,
                output_path=reading_audio_path,
                elevenlabs_api_key=elevenlabs_api_key,
                voice_id=voice_id,
                model=model,
                citation_key=self.citation_key,
                speed=reading_speed,
            )
            print(f"Generated reading audio: {reading_audio_path.name}")

        # Generate lecture audio (educational style)
        if audio_config.get("generate_lecture", False):
            logger.info("Generating lecture audio...")
            lecture_audio_path = (
                self.output_base / f"{self.audio_prefix}-lecture-audio.mp3"
            )

            # Get lecture prompt configuration
            prompts_config = self.config.get("prompts", {}).get("prompts", {})
            audio_prompts = prompts_config.get("audio", {})
            lecture_prompt_config = {
                "lecture_system": audio_prompts.get("lecture_system"),
                "lecture_generation": audio_prompts.get("lecture_generation"),
            }

            # Extract image summaries as strings
            image_summary_strings = [img.summary for img in image_summaries]

            # Get lecture audio speed from config (defaults to same as summary)
            lecture_speed = audio_config.get(
                "lecture_speed", audio_config.get("summary_speed", 1.0)
            )

            # Read SI boundary from _meta.json if available
            assert self.state is not None
            meta_path = self.state.pdf_path.parent / f"{self.citation_key}_meta.json"
            si_start_page = None
            if meta_path.exists():
                import json

                meta = json.loads(meta_path.read_text())
                si_start_page = meta.get("si_start_page")

            generate_lecture_audio(
                markdown_files=cleaned_files,
                image_summaries=image_summary_strings,
                output_path=lecture_audio_path,
                elevenlabs_api_key=elevenlabs_api_key,
                voice_id=voice_id,
                model=model,
                citation_key=self.citation_key,
                lecture_prompt_config=lecture_prompt_config,
                speed=lecture_speed,
                si_start_page=si_start_page,
            )
            print(f"Generated lecture audio: {lecture_audio_path.name}")

        # After all audio generation is complete, write cards with audio links
        if audio_config.get("generate_complementary", False) and cards:
            # Get output config
            output_config = self.config.get("output", {}).get("output", {})
            formats = output_config.get("formats", {})
            tag_format = output_config.get("tag_format", "slugified")

            # Write cards with audio using the URIs stored in each card
            audio_path = self.output_base / formats.get(
                "cards_audio", "cards-with-audio.md"
            )
            with open(audio_path, "w") as f:
                for card in cards:
                    f.write(
                        card.to_md(
                            include_audio=True,
                            citation_key=self.citation_key,
                            tag_format=tag_format,
                        )
                    )
            outputs["cards_audio"] = audio_path
            logger.debug(f"Written cards with audio: {audio_path.name}")

            # Create a summary of audio transcripts for debugging
            transcripts_dir = self.output_base / "audio-transcripts"
            if transcripts_dir.exists():
                summary_path = transcripts_dir / "transcript-summary.md"
                with open(summary_path, "w") as f:
                    f.write("# Audio Transcript Summary\n\n")
                    f.write(f"Generated transcripts for {len(cards)} cards\n\n")

                    # Count cards with images
                    cards_with_images = sum(
                        1
                        for card in cards
                        if card.front.image_path or card.back.image_path
                    )
                    cards_with_summaries = sum(
                        1
                        for card in cards
                        if card.front.image_summary or card.back.image_summary
                    )

                    f.write(f"- Total cards: {len(cards)}\n")
                    f.write(f"- Cards with images: {cards_with_images}\n")
                    f.write(f"- Cards with image summaries: {cards_with_summaries}\n\n")

                    # List any cards missing summaries
                    missing_summaries = [
                        card
                        for card in cards
                        if (card.front.image_path or card.back.image_path)
                        and not (card.front.image_summary or card.back.image_summary)
                    ]

                    if missing_summaries:
                        f.write(
                            f"## Cards Missing Image Summaries ({len(missing_summaries)} cards)\n\n"
                        )
                        for card in missing_summaries:
                            f.write(f"- Card ID: {card.card_id}\n")
                            f.write(f"  - Front image: {card.front.image_path}\n")
                            f.write(f"  - Back image: {card.back.image_path}\n")
                            f.write(
                                f"  - Front text preview: {card.front.text[:50]}...\n\n"
                            )

                    f.write("\n## Transcript Files Generated\n\n")
                    transcript_files = sorted(transcripts_dir.glob("*.md"))
                    # Exclude the summary file itself
                    transcript_files = [
                        f for f in transcript_files if f.name != "transcript-summary.md"
                    ]
                    for tf in transcript_files:
                        f.write(f"- {tf.name}\n")

                print(f"Written transcript summary: {summary_path.name}")

    def format_deck_name(self, template: str, deck_name: str) -> str:
        """Format deck name template with variables.

        Replaces template variables in deck name with actual values.

        Parameters
        ----------
        template : str
            Deck name template (e.g., '{deck_name}_cards')
        deck_name : str
            Deck name to substitute (output_dir if specified, otherwise citation_key)

        Returns:
        -------
        str
            Formatted deck name

        Examples:
        --------
        >>> pipeline.format_deck_name("{deck_name}_2024", "Smith")
        'Smith_2024'
        """
        # Support template variables
        variables = {
            "deck_name": deck_name or "default",
            "citation_key": self.citation_key
            or "default",  # Still support citation_key for backward compatibility
        }

        # Replace template variables
        formatted = template
        for var, value in variables.items():
            formatted = formatted.replace(f"{{{var}}}", value)

        return formatted

    def prepare_anki_file(
        self,
        cards: list[PlainCard],
        deck_name: str,
        output_path: Path,
        include_audio: bool = True,
    ) -> Path:
        """Prepare markdown file for Anki import.

        Creates a properly formatted markdown file with deck header
        required by the Anki markdown importer.

        Parameters
        ----------
        cards : List[PlainCard]
            Cards to include
        deck_name : str
            Name of the Anki deck
        output_path : Path
            Base output file path
        include_audio : bool, optional
            Whether to include audio links (default is True)

        Returns:
        -------
        Path
            Path to prepared Anki file

        Notes:
        -----
        Adds '# DeckName' header required by md_to_anki.py.
        Preserves exact card formatting from original output.
        """
        anki_file = output_path.parent / f"anki-{output_path.name}"

        # First, read the existing output file to use its exact format
        # This ensures we preserve the exact card format that was generated
        with open(output_path) as f:
            content = f.read()

        with open(anki_file, "w") as f:
            # Write deck name header (required by md_to_anki.py)
            f.write(f"# {deck_name}\n\n")

            # Write the rest of the content as-is
            # This preserves the exact tag format from the original file
            f.write(content)

        return anki_file

    def send_to_anki(
        self,
        cards: list[PlainCard],
        outputs: dict[str, Path],
        anki_config: dict[str, Any],
    ):
        """Send generated cards to Anki via AnkiConnect.

        Uploads cards to Anki using the AnkiConnect API, supporting
        deck creation, card updates, and media file uploads.

        Parameters
        ----------
        cards : List[PlainCard]
            Flashcards to send
        outputs : Dict[str, Path]
            Output file paths
        anki_config : Dict[str, Any]
            Anki configuration options:
            - deck_name: Template for deck name (supports {citation_key})
            - card_format: 'plain' or 'with_audio'
            - update_existing: Update existing cards
            - media_upload: Upload media files
            - sync_after: Sync after upload
            - host: AnkiConnect host
            - port: AnkiConnect port

        Raises:
        ------
        Exception
            If connection to Anki fails or upload errors occur

        Notes:
        -----
        Requires Anki to be running with AnkiConnect addon installed.
        Creates deck if it doesn't exist. Handles both new cards and
        updates to existing cards based on configuration.

        See Also:
        --------
        AnkiProcessor : Handles AnkiConnect communication
        """
        try:
            # Format deck name with template variables
            # Use output_dir (stored in audio_prefix) if it was specified, otherwise use citation_key
            deck_template = anki_config.get("deck_name", "{deck_name}")
            # Use audio_prefix which contains output_dir if specified, otherwise citation_key
            deck_name_value = (
                self.audio_prefix
                if hasattr(self, "audio_prefix")
                else self.citation_key
            )
            deck_name = self.format_deck_name(deck_template, deck_name_value)

            logger.info(f"Sending cards to Anki deck: {deck_name}")

            # Choose which output file to use as base
            # Important: Only use audio cards if they actually exist
            card_format = anki_config.get("card_format", "with_audio")
            if card_format == "with_audio" and "cards_audio" in outputs:
                base_file = outputs["cards_audio"]
            else:
                # Fall back to plain cards if audio cards don't exist or not requested
                base_file = outputs["cards_plain"]

            # Prepare Anki file with proper deck header
            # Only include audio if it was actually generated
            use_audio = card_format == "with_audio" and "cards_audio" in outputs
            anki_file = self.prepare_anki_file(
                cards, deck_name, base_file, include_audio=use_audio
            )

            # Initialize AnkiProcessor
            host = anki_config.get("host", "127.0.0.1")
            port = anki_config.get("port", 8765)
            anki_processor = AnkiProcessor(host, port)

            # Send cards to Anki
            cards_added, cards_updated = anki_processor.send_cards_from_file(
                file_path=anki_file,
                deck_name=deck_name,
                update_existing=anki_config.get("update_existing", True),
                upload_media=anki_config.get("media_upload", True),
                sync_after=anki_config.get("sync_after", False),
            )

            logger.info(
                f"Successfully sent to Anki deck '{deck_name}': {cards_added} added, {cards_updated} updated"
            )

        except Exception as e:
            logger.error(f"Failed to send cards to Anki: {e}")
            raise

    def _self_refine_cards(
        self,
        response: CardGenerationResponse,
        doc_summary: DocumentSummary,
        card_type: str,
    ) -> CardGenerationResponse:
        """Apply self-refine pattern to improve card quality.

        Iteratively generates feedback and refines cards until they meet
        quality standards or reach maximum iterations.

        Parameters
        ----------
        response : CardGenerationResponse
            Initial card generation response
        doc_summary : DocumentSummary
            Document summary for context
        card_type : str
            Type of cards ('regular', 'cloze', or 'image')

        Returns:
        -------
        CardGenerationResponse
            Refined cards response
        """
        refinement_config = self.config.get("refinement", {}).get("refinement", {})
        max_iterations = refinement_config.get("max_iterations", 3)

        # Track refinement history
        history = RefinementHistory()

        for iteration in range(max_iterations):
            try:
                # Generate feedback
                feedback = self._generate_card_feedback(
                    response.cards, doc_summary, card_type
                )

                # Check if cards are good enough
                if feedback.done:
                    logger.info(
                        f"{card_type.capitalize()} cards passed quality check after {iteration} iterations"
                    )
                    break

                # Log feedback
                logger.info(
                    f"Iteration {iteration + 1} feedback for {card_type} cards:"
                )
                for issue in feedback.feedback:
                    logger.info(f"  - {issue}")

                # Refine cards based on feedback
                refined_response = self._refine_cards(
                    response, feedback, doc_summary, card_type
                )

                # Save to history
                history.add_iteration(response.cards, feedback, refined_response.cards)

                # Update response for next iteration
                response = refined_response

            except ValidationError as e:
                logger.warning(
                    f"Validation error during refinement iteration {iteration + 1}: {e}"
                )
                logger.info(f"Continuing with current cards for {card_type} type")
                # Don't update response, keep current version
                break
            except Exception as e:
                logger.error(f"Unexpected error during refinement: {e}")
                break

        return response

    def _generate_card_feedback(
        self, cards: list[PlainCard], doc_summary: DocumentSummary, card_type: str
    ) -> CardFeedback:
        """Generate quality feedback for cards.

        Parameters
        ----------
        cards : List[PlainCard]
            Cards to evaluate
        doc_summary : DocumentSummary
            Document context
        card_type : str
            Type of cards being evaluated

        Returns:
        -------
        CardFeedback
            Structured feedback about card quality
        """
        refinement_config = self.config.get("refinement", {}).get("refinement", {})
        feedback_prompts = refinement_config.get("feedback_prompts", {})

        # Format cards for evaluation
        cards_text = self._format_cards_for_feedback(cards)

        # Get appropriate feedback prompt
        feedback_prompt = feedback_prompts.get(f"{card_type}_cards", "")

        # Get LLM config
        models_config = self.config.get("models", {}).get("models", {})
        llm_config = models_config.get("llm", {})

        system_prompt = f"""You are an expert teacher evaluating flashcards for educational value.
Your goal is to ensure cards help students learn KEY CONCEPTS from the document.

Check for these issues in {card_type} cards:
1. External references ([1], "According to X", "Figure 3")
2. Insufficient context ("this", "the model") 
3. Generic tags (#equation vs #calculus.derivatives)
4. Rote memorization ("What does X stand for?")
5. Math formatting issues
6. Cloze problems:
   - CRITICAL: Cloze cards that ARE questions (end with "?")
   - CRITICAL: Cloze deletions longer than 5 words
   - Extra questions after cloze deletion
   - Split equations across cloze markers
   - {"CRITICAL: Image cards should NEVER be cloze cards" if card_type == "image" else ""}
7. Undefined acronyms
8. Author-centric questions
9. EDUCATIONAL VALUE - does this help students learn fundamental concepts?
   - BAD: "In sin(2πx), what does π represent?" (trivial notation)
   - GOOD: "How does Fourier analysis decompose signals?" (key concept)
10. Answer revealed in question (for regular cards)
11. Focus on trivial details instead of important principles
12. META-CONTENT LEAKAGE - cards must NEVER mention:
    - "focal page", "surrounding pages", "context pages"
    - "CONTEXT FROM SURROUNDING PAGES", "FOCAL PAGE CONTENT"
    - References to document structure or organization markers
    - Any meta-instructions or processing hints
    - Examples of bad cards:
      * "What is discussed in the focal page content?"
      * "According to the context from surrounding pages..."
      * "The focal page presents..."
13. WRITING STYLE AND FLOW:
    - Choppy, terse sentences that don't flow well
    - Excessive parenthetical statements
    - Poor readability when spoken aloud
    - BAD: "Neural networks learn. They use backprop. Weights update."
    - GOOD: "Neural networks learn through backpropagation, which updates weights based on error gradients."
14. Check that content reads naturally and flows smoothly
    - Would this sound natural when converted to audio?
    - Are ideas connected with transitional phrases?
    - Is the writing engaging rather than robotic?

If ALL cards are high quality AND educationally valuable, set done=True.
Otherwise provide specific, actionable feedback focused on what matters for learning."""

        eval_result = card_feedback_agent.run_sync(
            f"""
Document context:
Title: {doc_summary.title}
Summary: {doc_summary.summary[:500]}...
Key concepts: {doc_summary.technical_terms}
Acronyms: {doc_summary.acronyms}

Cards to evaluate:
{cards_text}

Use the document summary to understand what KEY CONCEPTS students should learn.
Cards should focus on these fundamental ideas, not trivial notation or random details.

{feedback_prompt}
""",
            instructions=system_prompt,
            model=get_model_string(llm_config),
        )
        return eval_result.output

    def _refine_cards(
        self,
        response: CardGenerationResponse,
        feedback: CardFeedback,
        doc_summary: DocumentSummary,
        card_type: str,
    ) -> CardGenerationResponse:
        """Refine cards based on feedback.

        Parameters
        ----------
        response : CardGenerationResponse
            Original cards
        feedback : CardFeedback
            Feedback to address
        doc_summary : DocumentSummary
            Document context
        card_type : str
            Type of cards

        Returns:
        -------
        CardGenerationResponse
            Refined cards
        """
        refinement_config = self.config.get("refinement", {}).get("refinement", {})
        refinement_prompts = refinement_config.get("refinement_prompts", {})

        # Format cards and feedback
        cards_text = self._format_cards_for_feedback(response.cards)
        feedback_text = "\n".join([f"- {f}" for f in feedback.feedback])

        # Get refinement prompt
        refinement_prompt = refinement_prompts.get(f"{card_type}_cards", "")

        # Add specific guidance for cloze cards with math
        if card_type == "cloze":
            refinement_prompt += """

IMPORTANT for cloze cards with math:
- If an equation has multiple parts, you can hide different parts in separate cloze markers
- Numeric values like {{c1::0.1}} are valid - don't avoid them
- Example with multiple parts: "In {{c1::\\alpha = 0.1}}, the learning rate is {{c2::0.1}}"
- Example with full equation: "The equation {{c1::\\(E = mc^2\\)}} relates energy to mass"
"""

        # Add specific guidance for meta-content issues
        if any(
            "focal page" in f.lower()
            or "surrounding pages" in f.lower()
            or "meta-content" in f.lower()
            for f in feedback.feedback
        ):
            refinement_prompt += """

CRITICAL META-CONTENT FIX REQUIRED:
The feedback indicates cards are referencing document structure instead of content.
You MUST rewrite these cards to:
1. Remove ALL mentions of "focal page content", "surrounding pages", "context"
2. Focus on the actual technical content and concepts
3. Ask about the subject matter directly, not about how it's presented

Example fixes:
- BAD: "What is discussed in the focal page content?"
- GOOD: "What is the key principle of backpropagation?"

- BAD: "According to the focal page, what is the error function?"
- GOOD: "What is the error function E(w) in terms of network output and target values?"
"""

        # Get LLM config
        models_config = self.config.get("models", {}).get("models", {})
        llm_config = models_config.get("llm", {})

        try:
            refine_result = card_gen_agent.run_sync(
                f"""
Original cards:
{cards_text}

Feedback to address:
{feedback_text}

Document context:
Title: {doc_summary.title}
Acronyms: {doc_summary.acronyms}
Technical terms: {doc_summary.technical_terms}

{refinement_prompt}

Generate {len(response.cards)} improved {card_type} cards addressing all feedback.
""",
                instructions=f"""You are an expert flashcard creator.
Refine the {card_type} cards to address ALL feedback issues.
Maintain the same number and types of cards.

CRITICAL: If feedback mentions meta-content leakage, you MUST:
1. Remove ALL mentions of "focal page", "surrounding pages", "context pages"
2. Remove ALL structural markers like "FOCAL PAGE CONTENT" or "CONTEXT FROM"
3. Rewrite questions to focus on the actual content, not document structure
4. Never reference how the content is organized or presented

WRITING STYLE IMPROVEMENTS:
1. Write in complete, flowing sentences that read naturally
2. Connect ideas with smooth transitions
3. Avoid choppy, terse statements
4. Minimize parenthetical phrases - integrate information naturally
5. Ensure content sounds good when read aloud
6. Use varied sentence structure for better engagement""",
                model=get_model_string(llm_config),
            )
            return refine_result.output
        except (ValidationError, Exception) as e:
            logger.error(f"Error during refinement: {e}")
            logger.warning("Returning original cards due to refinement error")
            return response

    def _format_cards_for_feedback(self, cards: list[PlainCard]) -> str:
        """Format cards for feedback evaluation.

        Parameters
        ----------
        cards : List[PlainCard]
            Cards to format

        Returns:
        -------
        str
            Formatted card text
        """
        formatted = []
        for i, card in enumerate(cards):
            formatted.append(f"Card {i + 1}:")
            formatted.append(f"Front: {card.front.text}")
            if "{{c" not in card.front.text:  # Regular card
                formatted.append(f"Back: {card.back.text}")
            formatted.append(f"Tags: {', '.join(card.tags)}")
            formatted.append("")  # Empty line between cards

        return "\n".join(formatted)

    def _self_refine_audio_transcript(
        self, transcript: str, card_type: str, doc_summary: DocumentSummary
    ) -> str:
        """Apply self-refine to audio transcripts.

        Parameters
        ----------
        transcript : str
            Original audio transcript
        card_type : str
            Type of card ('regular', 'cloze', etc.)
        doc_summary : DocumentSummary
            Document context

        Returns:
        -------
        str
            Refined transcript suitable for TTS
        """
        refinement_config = self.config.get("refinement", {}).get("refinement", {})

        if not refinement_config.get("include_audio", False):
            return transcript

        max_iterations = refinement_config.get("max_iterations", 3)

        for iteration in range(max_iterations):
            # Generate feedback for transcript
            feedback = self._generate_audio_feedback(transcript, card_type, doc_summary)

            if feedback.done:
                logger.info(
                    f"Audio transcript passed quality check after {iteration} iterations"
                )
                break

            # Refine transcript
            transcript = self._refine_audio_transcript(
                transcript, feedback, card_type, doc_summary
            )

        return transcript

    def _generate_audio_feedback(
        self, transcript: str, card_type: str, doc_summary: DocumentSummary
    ) -> AudioTranscriptFeedback:
        """Generate feedback for audio transcript.

        Parameters
        ----------
        transcript : str
            Transcript to evaluate
        card_type : str
            Type of content
        doc_summary : DocumentSummary
            Document context

        Returns:
        -------
        AudioTranscriptFeedback
            Feedback about transcript quality
        """
        refinement_config = self.config.get("refinement", {}).get("refinement", {})
        feedback_prompts = refinement_config.get("feedback_prompts", {})

        # Get LLM config
        models_config = self.config.get("models", {}).get("models", {})
        llm_config = models_config.get("llm", {})

        af_result = audio_feedback_agent.run_sync(
            f"""
Transcript to evaluate:
{transcript}

Card type: {card_type}
Citation key: {doc_summary.citation_key if hasattr(doc_summary, "citation_key") else "Unknown"}
Acronyms in document: {doc_summary.acronyms}

{feedback_prompts.get(f"{card_type}_audio", feedback_prompts.get("audio_transcript", ""))}
""",
            instructions="You are an expert in audio transcript quality for text-to-speech.",
            model=get_model_string(llm_config),
        )
        return af_result.output

    def _refine_audio_transcript(
        self,
        transcript: str,
        feedback: AudioTranscriptFeedback,
        card_type: str,
        doc_summary: DocumentSummary,
    ) -> str:
        """Refine audio transcript based on feedback.

        Parameters
        ----------
        transcript : str
            Original transcript
        feedback : AudioTranscriptFeedback
            Feedback to address
        card_type : str
            Type of content
        doc_summary : DocumentSummary
            Document context

        Returns:
        -------
        str
            Refined transcript
        """
        refinement_config = self.config.get("refinement", {}).get("refinement", {})
        refinement_prompts = refinement_config.get("refinement_prompts", {})

        # Get LLM config
        models_config = self.config.get("models", {}).get("models", {})
        llm_config = models_config.get("llm", {})

        feedback_text = "\n".join([f"- {f}" for f in feedback.feedback])

        messages = [
            {
                "role": "system",
                "content": "You are an expert at creating clear audio transcripts for text-to-speech.",
            },
            {
                "role": "user",
                "content": f"""
Original transcript:
{transcript}

Issues to fix:
{feedback_text}

Context:
- Card type: {card_type}
- Document acronyms: {doc_summary.acronyms}

{refinement_prompts.get(f"{card_type}_audio", refinement_prompts.get("audio_transcript", ""))}

Rewrite the transcript addressing all issues.
""",
            },
        ]

        rat_result = text_agent.run_sync(
            messages[1]["content"],
            instructions=messages[0]["content"],
            model=get_model_string(llm_config),
        )

        return rat_result.output
