//! Text extraction from PDF using lopdf
//!
//! This module extracts text with position information for structure detection.

mod content_stream;
mod fonts;
mod interpreter;
mod layout;
mod links;
mod xobjects;

use crate::pdf_inspector::PdfError;
use crate::pdf_inspector::text_utils::{expand_ligatures, is_rtl_text};
use crate::pdf_inspector::tounicode::FontCMaps;
use crate::pdf_inspector::types::{PdfRect, TextItem};
use log::debug;
use lopdf::{Document, Object, ObjectId};
use std::collections::{HashMap, HashSet};
use std::path::Path;

use content_stream::extract_page_text_items;
use links::{extract_form_fields, extract_page_links};

// Re-export public types so existing `crate::pdf_inspector::extractor::X` paths keep working.
pub use crate::pdf_inspector::text_utils::{is_bold_font, is_italic_font};
pub use crate::pdf_inspector::types::{ItemType, TextLine};
pub(crate) use layout::detect_columns;
pub use layout::group_into_lines;

// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------

/// Extract text from PDF file as plain string
pub fn extract_text<P: AsRef<Path>>(path: P) -> Result<String, PdfError> {
    crate::pdf_inspector::validate_pdf_file(&path)?;
    let doc = match Document::load(&path) {
        Ok(d) => d,
        Err(ref e) if crate::pdf_inspector::is_encrypted_lopdf_error(e) => {
            Document::load_with_password(&path, "")?
        }
        Err(e) => return Err(e.into()),
    };
    extract_text_from_doc(&doc)
}

/// Extract text from PDF memory buffer
pub fn extract_text_mem(buffer: &[u8]) -> Result<String, PdfError> {
    crate::pdf_inspector::validate_pdf_bytes(buffer)?;
    let doc = match Document::load_mem(buffer) {
        Ok(d) => d,
        Err(ref e) if crate::pdf_inspector::is_encrypted_lopdf_error(e) => {
            Document::load_mem_with_password(buffer, "")?
        }
        Err(e) => return Err(e.into()),
    };
    extract_text_from_doc(&doc)
}

/// Extract text from loaded document
fn extract_text_from_doc(doc: &Document) -> Result<String, PdfError> {
    let pages = doc.get_pages();
    let page_nums: Vec<u32> = pages.keys().cloned().collect();

    doc.extract_text(&page_nums)
        .map_err(|e| PdfError::Parse(e.to_string()))
}

/// Extract text with position information from PDF file
pub fn extract_text_with_positions<P: AsRef<Path>>(path: P) -> Result<Vec<TextItem>, PdfError> {
    extract_text_with_positions_pages(path, None)
}

/// Extract text with positions from a file, limited to specific pages.
///
/// `page_filter` is an optional set of 1-indexed page numbers to process.
/// When `None`, all pages are processed.
pub fn extract_text_with_positions_pages<P: AsRef<Path>>(
    path: P,
    page_filter: Option<&HashSet<u32>>,
) -> Result<Vec<TextItem>, PdfError> {
    let (items, _rects) = extract_text_with_positions_and_rects(path, page_filter)?;
    Ok(items)
}

/// Extract text with positions and rectangles from a file.
pub(crate) fn extract_text_with_positions_and_rects<P: AsRef<Path>>(
    path: P,
    page_filter: Option<&HashSet<u32>>,
) -> Result<(Vec<TextItem>, Vec<PdfRect>), PdfError> {
    crate::pdf_inspector::validate_pdf_file(&path)?;
    let doc = match Document::load(&path) {
        Ok(d) => d,
        Err(ref e) if crate::pdf_inspector::is_encrypted_lopdf_error(e) => {
            Document::load_with_password(&path, "")?
        }
        Err(e) => return Err(e.into()),
    };
    let font_cmaps = FontCMaps::from_doc(&doc);
    extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)
}

/// Extract text with positions from memory buffer
pub fn extract_text_with_positions_mem(buffer: &[u8]) -> Result<Vec<TextItem>, PdfError> {
    extract_text_with_positions_mem_pages(buffer, None)
}

/// Extract text with positions from memory buffer, limited to specific pages.
pub fn extract_text_with_positions_mem_pages(
    buffer: &[u8],
    page_filter: Option<&HashSet<u32>>,
) -> Result<Vec<TextItem>, PdfError> {
    let (items, _rects) = extract_text_with_positions_mem_and_rects(buffer, page_filter)?;
    Ok(items)
}

/// Extract text with positions and rectangles from memory buffer.
pub(crate) fn extract_text_with_positions_mem_and_rects(
    buffer: &[u8],
    page_filter: Option<&HashSet<u32>>,
) -> Result<(Vec<TextItem>, Vec<PdfRect>), PdfError> {
    crate::pdf_inspector::validate_pdf_bytes(buffer)?;
    let doc = match Document::load_mem(buffer) {
        Ok(d) => d,
        Err(ref e) if crate::pdf_inspector::is_encrypted_lopdf_error(e) => {
            Document::load_mem_with_password(buffer, "")?
        }
        Err(e) => return Err(e.into()),
    };
    let font_cmaps = FontCMaps::from_doc(&doc);
    extract_positioned_text_from_doc(&doc, &font_cmaps, page_filter)
}

// ---------------------------------------------------------------------------
// Orchestration
// ---------------------------------------------------------------------------

/// Extract positioned text and rectangles from a pre-loaded document.
pub(crate) fn extract_positioned_text_from_doc(
    doc: &Document,
    font_cmaps: &FontCMaps,
    page_filter: Option<&HashSet<u32>>,
) -> Result<(Vec<TextItem>, Vec<PdfRect>), PdfError> {
    let pages = doc.get_pages();
    let mut all_items = Vec::new();
    let mut all_rects = Vec::new();

    // Build page ObjectId → page number map for form field extraction
    let page_id_to_num: HashMap<ObjectId, u32> =
        pages.iter().map(|(num, &id)| (id, *num)).collect();

    for (page_num, &page_id) in pages.iter() {
        if let Some(filter) = page_filter {
            if !filter.contains(page_num) {
                continue;
            }
        }
        let (items, rects) = extract_page_text_items(doc, page_id, *page_num, font_cmaps)?;
        debug!(
            "page {}: {} text items, {} rects",
            page_num,
            items.len(),
            rects.len()
        );
        if log::log_enabled!(log::Level::Trace) {
            for item in &items {
                log::trace!(
                    "  p={} x={:7.1} y={:7.1} w={:7.1} fs={:5.1} font={:6} {:?}",
                    page_num,
                    item.x,
                    item.y,
                    item.width,
                    item.font_size,
                    item.font,
                    if item.text.len() > 80 {
                        &item.text[..80]
                    } else {
                        &item.text
                    }
                );
            }
        }
        all_items.extend(items);
        all_rects.extend(rects);

        // Extract hyperlinks from page annotations
        let links = extract_page_links(doc, page_id, *page_num);
        all_items.extend(links);
    }

    // Extract AcroForm field values
    let form_items = extract_form_fields(doc, &page_id_to_num);
    all_items.extend(form_items);

    Ok((all_items, all_rects))
}

// ---------------------------------------------------------------------------
// Shared helpers (used by submodules via `super::`)
// ---------------------------------------------------------------------------

/// Multiply two 2D transformation matrices
/// Matrix format: [a, b, c, d, e, f] representing:
/// | a  b  0 |
/// | c  d  0 |
/// | e  f  1 |
pub(crate) fn multiply_matrices(m1: &[f32; 6], m2: &[f32; 6]) -> [f32; 6] {
    [
        m1[0] * m2[0] + m1[1] * m2[2],
        m1[0] * m2[1] + m1[1] * m2[3],
        m1[2] * m2[0] + m1[3] * m2[2],
        m1[2] * m2[1] + m1[3] * m2[3],
        m1[4] * m2[0] + m1[5] * m2[2] + m2[4],
        m1[4] * m2[1] + m1[5] * m2[3] + m2[5],
    ]
}

/// Merge adjacent text items on the same line into single items.
///
/// Groups items by (page, Y-position) with a 5pt tolerance, sorts within each
/// group by X, then merges consecutive items that share a similar font size
/// and are close horizontally.
pub(crate) fn merge_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
    if items.is_empty() {
        return items;
    }

    // Group items by (page, Y position) with 5pt tolerance
    let y_tolerance = 5.0;
    let mut line_groups: Vec<(u32, f32, Vec<&TextItem>)> = Vec::new();

    for item in &items {
        let found = line_groups
            .iter_mut()
            .find(|(pg, y, _)| *pg == item.page && (item.y - *y).abs() < y_tolerance);
        if let Some((_, _, group)) = found {
            group.push(item);
        } else {
            line_groups.push((item.page, item.y, vec![item]));
        }
    }

    // Sort each group by X position (direction-aware)
    for (_, _, group) in &mut line_groups {
        let rtl = is_rtl_text(group.iter().map(|i| &i.text));
        if rtl {
            group.sort_by(|a, b| b.x.partial_cmp(&a.x).unwrap_or(std::cmp::Ordering::Equal));
        } else {
            group.sort_by(|a, b| a.x.partial_cmp(&b.x).unwrap_or(std::cmp::Ordering::Equal));
        }
    }

    // Sort groups by page then Y descending (top of page first)
    line_groups.sort_by(|a, b| {
        a.0.cmp(&b.0)
            .then_with(|| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal))
    });

    let mut merged = Vec::new();

    for (_, _, group) in &line_groups {
        let mut i = 0;
        while i < group.len() {
            let first = group[i];
            if first.is_rotated {
                merged.push(first.clone());
                i += 1;
                continue;
            }
            let mut text = first.text.clone();
            let mut end_x = first.x + first.width;
            let x_gap_max = first.font_size * 0.75;

            let mut j = i + 1;
            while j < group.len() {
                let next = group[j];
                if next.is_rotated {
                    break;
                }
                // Must be similar font size (within 20%)
                if (next.font_size - first.font_size).abs() > first.font_size * 0.20 {
                    break;
                }
                let gap = next.x - end_x;
                if gap > x_gap_max {
                    break;
                }
                if gap < -first.font_size * 0.5 {
                    break;
                }
                // Insert space at word boundaries
                if gap > first.font_size * 0.13 {
                    text.push(' ');
                }
                text.push_str(&next.text);
                end_x = next.x + next.width;
                j += 1;
            }

            merged.push(TextItem {
                text: expand_ligatures(&text),
                x: first.x,
                y: first.y,
                width: end_x - first.x,
                height: first.height,
                font: first.font.clone(),
                font_size: first.font_size,
                page: first.page,
                is_rotated: false,
                is_bold: first.is_bold,
                is_italic: first.is_italic,
                item_type: first.item_type.clone(),
            });

            i = j;
        }
    }

    merged
}

fn same_item_type(a: &ItemType, b: &ItemType) -> bool {
    matches!(
        (a, b),
        (ItemType::Text, ItemType::Text)
            | (ItemType::Image, ItemType::Image)
            | (ItemType::FormField, ItemType::FormField)
            | (ItemType::Link(_), ItemType::Link(_))
    )
}

fn is_overlapping_duplicate_text_item(existing: &TextItem, candidate: &TextItem) -> bool {
    if existing.page != candidate.page
        || existing.is_rotated != candidate.is_rotated
        || existing.font != candidate.font
        || !same_item_type(&existing.item_type, &candidate.item_type)
        || existing.text.trim() != candidate.text.trim()
    {
        return false;
    }

    let x_tol = existing.font_size.max(candidate.font_size) * 0.18 + 1.0;
    let y_tol = existing.font_size.max(candidate.font_size) * 0.15 + 1.0;
    let width_tol = existing.width.max(candidate.width) * 0.25 + 1.0;
    let height_tol = existing.height.max(candidate.height) * 0.25 + 1.0;

    (existing.x - candidate.x).abs() <= x_tol
        && (existing.y - candidate.y).abs() <= y_tol
        && (existing.width - candidate.width).abs() <= width_tol
        && (existing.height - candidate.height).abs() <= height_tol
}

fn prefer_duplicate_replacement(existing: &TextItem, candidate: &TextItem) -> bool {
    let existing_has_width = existing.width > 0.0;
    let candidate_has_width = candidate.width > 0.0;
    if candidate_has_width != existing_has_width {
        return candidate_has_width;
    }

    if (candidate.width - existing.width).abs() > 0.5 {
        return candidate.width > existing.width;
    }

    candidate.height > existing.height
}

pub(crate) fn deduplicate_overlapping_text_items(items: Vec<TextItem>) -> Vec<TextItem> {
    if items.len() < 2 {
        return items;
    }

    let mut kept: Vec<TextItem> = Vec::with_capacity(items.len());
    'outer: for item in items {
        for existing in kept.iter_mut().rev() {
            if existing.page != item.page {
                break;
            }
            if (existing.y - item.y).abs() > existing.font_size.max(item.font_size) * 0.6 + 2.0 {
                continue;
            }
            if is_overlapping_duplicate_text_item(existing, &item) {
                if prefer_duplicate_replacement(existing, &item) {
                    *existing = item;
                }
                continue 'outer;
            }
        }
        kept.push(item);
    }

    kept
}

/// Check whether the combined text matrix represents rotated text (> 15 degrees).
pub(crate) fn is_rotated_text_transform(matrix: &[f32; 6]) -> bool {
    let angle = matrix[1]
        .atan2(matrix[0])
        .to_degrees()
        .abs()
        .rem_euclid(180.0);
    let deviation = angle.min((180.0 - angle).abs());
    deviation > 15.0
}

/// Parse a 6-element matrix from operator operands (used by `cm`).
pub(crate) fn parse_matrix_from_operands(operands: &[Object]) -> Option<[f32; 6]> {
    if operands.len() < 6 {
        return None;
    }
    Some([
        get_number(&operands[0]).unwrap_or(1.0),
        get_number(&operands[1]).unwrap_or(0.0),
        get_number(&operands[2]).unwrap_or(0.0),
        get_number(&operands[3]).unwrap_or(1.0),
        get_number(&operands[4]).unwrap_or(0.0),
        get_number(&operands[5]).unwrap_or(0.0),
    ])
}

/// Parse a rectangle from `re` operator operands and transform to device space.
pub(crate) fn parse_rect_operator(
    operands: &[Object],
    ctm: &[f32; 6],
    page: u32,
) -> Option<crate::pdf_inspector::types::PdfRect> {
    if operands.len() < 4 {
        return None;
    }
    let rx = get_number(&operands[0]).unwrap_or(0.0);
    let ry = get_number(&operands[1]).unwrap_or(0.0);
    let rw = get_number(&operands[2]).unwrap_or(0.0);
    let rh = get_number(&operands[3]).unwrap_or(0.0);
    let x_dev = rx * ctm[0] + ry * ctm[2] + ctm[4];
    let y_dev = rx * ctm[1] + ry * ctm[3] + ctm[5];
    let w_dev = rw * ctm[0];
    let h_dev = rh * ctm[3];
    Some(crate::pdf_inspector::types::PdfRect {
        x: x_dev,
        y: y_dev,
        width: w_dev,
        height: h_dev,
        page,
    })
}

/// Helper to get f32 from Object
pub(crate) fn get_number(obj: &Object) -> Option<f32> {
    match obj {
        Object::Integer(i) => Some(*i as f32),
        Object::Real(r) => Some(*r),
        _ => None,
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::pdf_inspector::text_utils::{
        is_cjk_char, is_rtl_char, is_rtl_text, sort_line_items,
    };
    use crate::pdf_inspector::types::{ItemType, TextLine};
    use layout::{detect_columns, is_newspaper_layout};

    #[test]
    fn test_group_into_lines() {
        let items = vec![
            TextItem {
                text: "Hello".into(),
                x: 100.0,
                y: 700.0,
                width: 50.0,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
            TextItem {
                text: "World".into(),
                x: 160.0,
                y: 700.0,
                width: 50.0,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
            TextItem {
                text: "Next line".into(),
                x: 100.0,
                y: 680.0,
                width: 80.0,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
        ];

        let lines = group_into_lines(items);
        assert_eq!(lines.len(), 2);
        assert_eq!(lines[0].text(), "Hello World");
        assert_eq!(lines[1].text(), "Next line");
    }

    #[test]
    fn test_deduplicate_overlapping_text_items() {
        let items = vec![
            TextItem {
                text: "Exercice".into(),
                x: 100.0,
                y: 700.0,
                width: 50.0,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
            TextItem {
                text: "Exercice".into(),
                x: 100.4,
                y: 700.2,
                width: 50.5,
                height: 12.1,
                font: "F1".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
        ];

        let deduped = deduplicate_overlapping_text_items(items);
        assert_eq!(deduped.len(), 1);
        assert_eq!(deduped[0].text, "Exercice");
    }

    #[test]
    fn test_bold_italic_detection() {
        // Test bold detection
        assert!(is_bold_font("Arial-Bold"));
        assert!(is_bold_font("TimesNewRoman-Bold"));
        assert!(is_bold_font("Helvetica-BoldOblique"));
        assert!(is_bold_font("ABCDEF+ArialMT-Bold"));
        assert!(is_bold_font("NotoSans-Black"));
        assert!(is_bold_font("Roboto-SemiBold"));
        assert!(!is_bold_font("Arial"));
        assert!(!is_bold_font("TimesNewRoman-Italic"));

        // Test italic detection
        assert!(is_italic_font("Arial-Italic"));
        assert!(is_italic_font("TimesNewRoman-Italic"));
        assert!(is_italic_font("Helvetica-Oblique"));
        assert!(is_italic_font("ABCDEF+ArialMT-Italic"));
        assert!(is_italic_font("Helvetica-BoldOblique"));
        assert!(!is_italic_font("Arial"));
        assert!(!is_italic_font("TimesNewRoman-Bold"));

        // Test bold-italic detection
        assert!(is_bold_font("Arial-BoldItalic"));
        assert!(is_italic_font("Arial-BoldItalic"));
        assert!(is_bold_font("Helvetica-BoldOblique"));
        assert!(is_italic_font("Helvetica-BoldOblique"));
    }

    #[test]
    fn test_word_level_items_get_spaces() {
        // Simulate CID font per-word items touching with gap=0
        let items = vec![
            TextItem {
                text: "the".into(),
                x: 100.0,
                y: 500.0,
                width: 19.5,
                height: 12.0,
                font: "C2_0".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
            TextItem {
                text: "Prague".into(),
                x: 119.5,
                y: 500.0,
                width: 42.0,
                height: 12.0,
                font: "C2_0".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
            TextItem {
                text: "Rules".into(),
                x: 161.5,
                y: 500.0,
                width: 35.0,
                height: 12.0,
                font: "C2_0".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
        ];

        let lines = group_into_lines(items);
        assert_eq!(lines.len(), 1);
        assert_eq!(lines[0].text(), "the Prague Rules");
    }

    #[test]
    fn test_single_char_items_still_join() {
        // Per-glyph positioning: single chars should join into words
        let items = vec![
            TextItem {
                text: "N".into(),
                x: 100.0,
                y: 500.0,
                width: 8.0,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
            TextItem {
                text: "A".into(),
                x: 108.0,
                y: 500.0,
                width: 8.0,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
            TextItem {
                text: "V".into(),
                x: 116.0,
                y: 500.0,
                width: 8.0,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
        ];

        let lines = group_into_lines(items);
        assert_eq!(lines.len(), 1);
        assert_eq!(lines[0].text(), "NAV");
    }

    #[test]
    fn test_per_glyph_word_boundaries() {
        // Per-character PDF rendering (e.g. SEC filings): each glyph is a
        // separate TextItem. Intra-word gaps are ≈ 0, word gaps ≈ 2.0 at
        // font_size 13.3 (ratio 0.15). Must detect word boundaries correctly.
        fn char_item(ch: &str, x: f32, width: f32) -> TextItem {
            TextItem {
                text: ch.into(),
                x,
                y: 719.3,
                width,
                height: 13.3,
                font: "F4".into(),
                font_size: 13.3,
                page: 1,
                is_rotated: false,
                is_bold: true,
                is_italic: false,
                item_type: ItemType::Text,
            }
        }

        // "Item 2" — gap of 2.0 between 'm' and '2' at font_size 13.3
        let items = vec![
            char_item("I", 24.3, 3.1),
            char_item("t", 27.5, 2.7),
            char_item("e", 30.1, 3.5),
            char_item("m", 33.7, 6.7),
            char_item("2", 42.3, 4.0), // gap = 42.3 - 40.4 = 1.9
        ];

        let lines = group_into_lines(items);
        assert_eq!(lines.len(), 1);
        assert_eq!(lines[0].text(), "Item 2");
    }

    #[test]
    fn test_per_glyph_words_not_merged() {
        // Verify multiple words from per-character rendering get spaces between them
        fn char_item(ch: &str, x: f32, width: f32) -> TextItem {
            TextItem {
                text: ch.into(),
                x,
                y: 705.5,
                width,
                height: 13.3,
                font: "F5".into(),
                font_size: 13.3,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            }
        }

        // "of the" — three words, each with ~2px word gaps
        let items = vec![
            char_item("o", 100.0, 4.0),
            char_item("f", 104.0, 2.7),
            // word gap: 108.7 → 110.7 (gap = 4.0)
            char_item("t", 110.7, 2.7),
            char_item("h", 113.4, 4.4),
            char_item("e", 117.8, 3.5),
        ];

        let lines = group_into_lines(items);
        assert_eq!(lines.len(), 1);
        assert_eq!(lines[0].text(), "of the");
    }

    #[test]
    fn test_cjk_items_join_without_spaces() {
        // Japanese text items touching at gap=0 should join without spaces
        let items = vec![
            TextItem {
                text: "である".into(),
                x: 100.0,
                y: 500.0,
                width: 24.0,
                height: 12.0,
                font: "C2_0".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
            TextItem {
                text: "履行義務".into(),
                x: 124.0,
                y: 500.0,
                width: 32.0,
                height: 12.0,
                font: "C2_0".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
            TextItem {
                text: "を識別す".into(),
                x: 156.0,
                y: 500.0,
                width: 32.0,
                height: 12.0,
                font: "C2_0".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
        ];

        let lines = group_into_lines(items);
        assert_eq!(lines.len(), 1);
        assert_eq!(lines[0].text(), "である履行義務を識別す");
    }

    fn make_item(text: &str, x: f32, y: f32, width: f32) -> TextItem {
        TextItem {
            text: text.into(),
            x,
            y,
            width,
            height: 12.0,
            font: "F1".into(),
            font_size: 12.0,
            page: 1,
            is_rotated: false,
            is_bold: false,
            is_italic: false,
            item_type: ItemType::Text,
        }
    }

    #[test]
    fn test_detect_two_columns() {
        let mut items = Vec::new();
        // Left column at x=72, right column at x=350, gutter ~278-350
        for i in 0..30 {
            let y = 700.0 - (i as f32) * 14.0;
            items.push(make_item("Left text here", 72.0, y, 200.0));
            items.push(make_item("Right text here", 350.0, y, 200.0));
        }
        let cols = detect_columns(&items, 1);
        assert_eq!(cols.len(), 2, "Expected 2 columns, got {:?}", cols);
        assert!(cols[0].x_min < cols[1].x_min);
    }

    #[test]
    fn test_detect_three_columns() {
        let mut items = Vec::new();
        // Three columns at x=50, x=220, x=390
        for i in 0..30 {
            let y = 700.0 - (i as f32) * 14.0;
            items.push(make_item("Col one", 50.0, y, 140.0));
            items.push(make_item("Col two", 220.0, y, 140.0));
            items.push(make_item("Col three", 390.0, y, 140.0));
        }
        let cols = detect_columns(&items, 1);
        assert_eq!(cols.len(), 3, "Expected 3 columns, got {:?}", cols);
    }

    #[test]
    fn test_width_bleed_tolerance() {
        let mut items = Vec::new();
        // Two columns with a clear gutter
        for i in 0..30 {
            let y = 700.0 - (i as f32) * 14.0;
            items.push(make_item("Left text", 72.0, y, 200.0));
            items.push(make_item("Right text", 350.0, y, 200.0));
        }
        // Add a few items that bleed across the gutter
        for i in 0..3 {
            let y = 700.0 - (i as f32) * 14.0;
            items.push(make_item("wide", 72.0, y, 320.0));
        }
        let cols = detect_columns(&items, 1);
        assert!(
            cols.len() >= 2,
            "Width bleed should not prevent column detection, got {:?}",
            cols
        );
    }

    #[test]
    fn test_single_column_no_false_split() {
        let mut items = Vec::new();
        // Single column: items spanning full width
        for i in 0..30 {
            let y = 700.0 - (i as f32) * 14.0;
            items.push(make_item(
                "This is a full-width paragraph of text",
                72.0,
                y,
                468.0,
            ));
        }
        let cols = detect_columns(&items, 1);
        assert!(
            cols.len() <= 1,
            "Full-width text should not be split into columns, got {:?}",
            cols
        );
    }

    #[test]
    fn test_is_rtl_char() {
        // Hebrew alef
        assert!(is_rtl_char('\u{05D0}'));
        // Arabic alif
        assert!(is_rtl_char('\u{0627}'));
        // Latin 'A' is not RTL
        assert!(!is_rtl_char('A'));
        // CJK is not RTL
        assert!(!is_rtl_char('\u{4E00}'));
    }

    #[test]
    fn test_is_rtl_text() {
        // Majority Hebrew with digits → RTL
        assert!(is_rtl_text(["\u{05E9}\u{05DC}\u{05D5}\u{05DD} 123"].iter()));
        // Majority Latin → not RTL
        assert!(!is_rtl_text(["Hello world"].iter()));
        // Empty → not RTL
        assert!(!is_rtl_text(std::iter::empty::<&str>()));
    }

    #[test]
    fn test_rtl_line_sorting() {
        let mut items = vec![
            TextItem {
                text: "\u{05D0}".into(), // alef at x=100
                x: 100.0,
                y: 700.0,
                width: 10.0,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
            TextItem {
                text: "\u{05D1}".into(), // bet at x=200 (rightmost)
                x: 200.0,
                y: 700.0,
                width: 10.0,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
        ];
        sort_line_items(&mut items);
        // RTL: rightmost (higher X) comes first
        assert_eq!(items[0].x, 200.0);
        assert_eq!(items[1].x, 100.0);
    }

    #[test]
    fn test_ltr_unaffected() {
        let mut items = vec![
            TextItem {
                text: "Hello".into(),
                x: 100.0,
                y: 700.0,
                width: 50.0,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
            TextItem {
                text: "World".into(),
                x: 200.0,
                y: 700.0,
                width: 50.0,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
        ];
        sort_line_items(&mut items);
        // LTR: leftmost comes first
        assert_eq!(items[0].x, 100.0);
        assert_eq!(items[1].x, 200.0);
    }

    #[test]
    fn test_hangul_is_cjk() {
        // Hangul Jamo
        assert!(is_cjk_char('\u{1100}'));
        // Hangul Compatibility Jamo
        assert!(is_cjk_char('\u{3131}'));
        // Hangul Syllable '가'
        assert!(is_cjk_char('\u{AC00}'));
        // Latin is not CJK
        assert!(!is_cjk_char('A'));
    }

    #[test]
    fn test_merge_items_french_letter_spacing() {
        // French financial PDFs use wider letter-spacing (1-1.5pt gaps at 12pt).
        // These intra-word gaps should NOT produce spaces.
        fn item(ch: &str, x: f32, width: f32) -> TextItem {
            TextItem {
                text: ch.into(),
                x,
                y: 500.0,
                width,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            }
        }

        // "BILAN" with ~1.0-1.5pt gaps between chars (ratio 0.083-0.125 at 12pt)
        let items = vec![
            item("B", 100.0, 7.0),
            item("I", 108.0, 3.5), // gap = 108.0 - 107.0 = 1.0
            item("L", 112.5, 6.0), // gap = 112.5 - 111.5 = 1.0
            item("A", 119.5, 7.0), // gap = 119.5 - 118.5 = 1.0
            item("N", 128.0, 7.0), // gap = 128.0 - 126.5 = 1.5
        ];

        let merged = merge_text_items(items);
        assert_eq!(merged.len(), 1);
        assert_eq!(
            merged[0].text, "BILAN",
            "French letter-spaced chars should merge without spaces"
        );
    }

    #[test]
    fn test_merge_items_preserves_word_boundary() {
        // Word boundaries (~2pt gap at 13.3pt, ratio 0.15) must still get spaces.
        fn item(ch: &str, x: f32, width: f32) -> TextItem {
            TextItem {
                text: ch.into(),
                x,
                y: 500.0,
                width,
                height: 13.3,
                font: "F1".into(),
                font_size: 13.3,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            }
        }

        // "AB CD" with 0pt intra-word gaps and 2pt inter-word gap
        let items = vec![
            item("A", 100.0, 7.0),
            item("B", 107.0, 7.0), // gap = 0
            item("C", 116.0, 7.0), // gap = 116.0 - 114.0 = 2.0 (ratio 0.15)
            item("D", 123.0, 7.0), // gap = 0
        ];

        let merged = merge_text_items(items);
        assert_eq!(merged.len(), 1);
        assert_eq!(
            merged[0].text, "AB CD",
            "Word boundary should produce a space"
        );
    }

    #[test]
    fn test_merge_items_touching_words() {
        // Touching items (gap = 0) should merge without spaces.
        let items = vec![
            TextItem {
                text: "Hello".into(),
                x: 100.0,
                y: 500.0,
                width: 30.0,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
            TextItem {
                text: "World".into(),
                x: 130.0,
                y: 500.0,
                width: 30.0,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page: 1,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            },
        ];

        let merged = merge_text_items(items);
        assert_eq!(merged.len(), 1);
        assert_eq!(
            merged[0].text, "HelloWorld",
            "Touching items merge without space"
        );
    }

    #[test]
    fn test_newspaper_layout_detection() {
        // Two dense columns (>15 lines each) with matching Y positions → newspaper
        let make_line = |y: f32, x: f32, page: u32| TextLine {
            y,
            page,
            items: vec![TextItem {
                text: "text".into(),
                x,
                y,
                width: 100.0,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            }],
        };

        let col1: Vec<TextLine> = (0..20)
            .map(|i| make_line(700.0 - i as f32 * 14.0, 50.0, 1))
            .collect();
        let col2: Vec<TextLine> = (0..20)
            .map(|i| make_line(700.0 - i as f32 * 14.0, 350.0, 1))
            .collect();

        assert!(is_newspaper_layout(&[col1, col2]));
    }

    #[test]
    fn test_newspaper_layout_misaligned_baselines() {
        // Two dense balanced columns with non-aligned Y positions (e.g. government gazettes
        // where columns are independently typeset) → should still be newspaper
        let make_line = |y: f32, x: f32, page: u32| TextLine {
            y,
            page,
            items: vec![TextItem {
                text: "text".into(),
                x,
                y,
                width: 100.0,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            }],
        };

        // Col1 starts at Y=700, col2 starts at Y=685 (15pt offset — no Y-collision)
        let col1: Vec<TextLine> = (0..20)
            .map(|i| make_line(700.0 - i as f32 * 14.0, 50.0, 1))
            .collect();
        let col2: Vec<TextLine> = (0..20)
            .map(|i| make_line(685.0 - i as f32 * 14.0, 350.0, 1))
            .collect();

        assert!(is_newspaper_layout(&[col1, col2]));
    }

    #[test]
    fn test_tabular_layout_detection() {
        // Sparse columns (<15 lines) → tabular, not newspaper
        let make_line = |y: f32, x: f32, page: u32| TextLine {
            y,
            page,
            items: vec![TextItem {
                text: "text".into(),
                x,
                y,
                width: 100.0,
                height: 12.0,
                font: "F1".into(),
                font_size: 12.0,
                page,
                is_rotated: false,
                is_bold: false,
                is_italic: false,
                item_type: ItemType::Text,
            }],
        };

        let col1: Vec<TextLine> = (0..5)
            .map(|i| make_line(700.0 - i as f32 * 14.0, 50.0, 1))
            .collect();
        let col2: Vec<TextLine> = (0..5)
            .map(|i| make_line(700.0 - i as f32 * 14.0, 350.0, 1))
            .collect();

        assert!(!is_newspaper_layout(&[col1, col2]));
    }
}
