// wasot: Core Wasot compiler and format converter
//
// Core commands:
// - pretty: Pretty print a Wasot file
// - to-sqlite/from-sqlite: SQLite database conversion
// - to-binary: Binary protobuf format conversion
// - to-jsonl: JSONL format conversion (auto-detects binary input)
// - to-xml: XML format conversion
// - to-wasot: Convert to Wasot format (auto-detects binary input)
// - convert: Universal format converter
// - completions: Generate shell completions
//
// For demo commands, see: wasot-demo
// For utility tools, see: wasot-tools

// Use mimalloc allocator for optimized builds
#[cfg(all(feature = "optimized-alloc", not(feature = "dhat-heap")))]
use mimalloc::MiMalloc;

#[cfg(all(feature = "optimized-alloc", not(feature = "dhat-heap")))]
#[global_allocator]
static GLOBAL: MiMalloc = MiMalloc;

// Use dhat allocator for heap profiling
#[cfg(feature = "dhat-heap")]
#[global_allocator]
static ALLOC: dhat::Alloc = dhat::Alloc;

use clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum};
use clap_complete::{generate, Shell};
use std::io::{self, BufWriter, Read};
use std::process;
use std::sync::mpsc;

use wasot_compiler::events::events::Emitter;
#[cfg(feature = "sexp")]
use wasot_compiler::ToSexpConsumer;
#[cfg(feature = "binary")]
use wasot_compiler::{BinaryProtobufConsumer, MarkdownEmitter, ProtobufBinaryEmitter};
use wasot_compiler::{
    Consumer, JsonlEmitter, SelectorConsumer, SqlFileConsumer, ToJsonlConsumer, WasotFileEmitter,
    WasotReconstructor, XmlEmitter, XmlListener,
};
#[cfg(feature = "sqlite")]
use wasot_compiler::{SqlFileEmitter, SqliteConsumer, SqliteEmitter};

/// Exit codes following Unix conventions
#[allow(dead_code)]
mod exit_codes {
    /// Success
    pub const SUCCESS: i32 = 0;
    /// Invalid input (parse error, validation failure)
    pub const INVALID_INPUT: i32 = 1;
    /// I/O error (file not found, permission denied)
    pub const IO_ERROR: i32 = 2;
    /// Invalid arguments (unknown format, bad selector)
    pub const INVALID_ARGS: i32 = 3;
}

/// Determine appropriate exit code from error message
fn exit_code_for_error(err: &str) -> i32 {
    let err_lower = err.to_lowercase();
    if err_lower.contains("not found")
        || err_lower.contains("permission denied")
        || err_lower.contains("no such file")
        || err_lower.contains("i/o error")
    {
        exit_codes::IO_ERROR
    } else if err_lower.contains("not yet implemented")
        || err_lower.contains("invalid format")
        || err_lower.contains("unknown")
        || err_lower.contains("invalid argument")
    {
        exit_codes::INVALID_ARGS
    } else {
        // Default to invalid input for parse errors etc.
        exit_codes::INVALID_INPUT
    }
}

#[derive(Parser)]
#[command(name = "wasot")]
#[command(version)]
#[command(about = "Convert and transform Wasot markup")]
#[command(
    long_about = "Wasot is an event-driven markup language. Human-readable like Markdown, \
    structured like XML, streams like neither. This tool converts between formats, \
    validates syntax, and analyzes documents."
)]
#[command(after_help = "EXAMPLES:
    wasot doc.wasot                      Normalize/pretty-print
    wasot doc.wasot -t xml               Convert to XML
    wasot doc.wasot -t jsonl             Convert to JSON Lines
    wasot doc.wasot -o doc.xml           Output format from extension
    wasot -f binary data.wasotb          Explicit input format
    wasot readme.md                      Convert Markdown to Wasot
    cat doc.wasot | wasot -t xml         Read from stdin
    wasot check *.wasot                  Validate multiple files
    wasot check doc.wasot && echo ok     Check in scripts
    wasot info doc.wasot                 Show block counts and depth
    wasot -t jsonl doc.wasot | jq .      Pipe to jq for JSON processing
    wasot -t xml doc.wasot | xmllint -   Pipe to xmllint for XML tools")]
#[command(args_conflicts_with_subcommands = true)]
#[command(disable_help_subcommand = true)]
struct Cli {
    #[command(flatten)]
    opts: GlobalOpts,

    #[command(subcommand)]
    command: Option<Commands>,
}

/// Options for conversion commands (default behavior)
#[derive(Args)]
struct GlobalOpts {
    /// Input format (auto-detected if not specified)
    #[arg(short = 'f', long = "from", value_enum)]
    from: Option<Format>,

    /// Output format (default: wasot)
    #[arg(short = 't', long = "to", value_enum)]
    to: Option<Format>,

    /// Output file (default: stdout)
    #[arg(short = 'o', long = "output")]
    output: Option<String>,

    /// Suppress warnings
    #[arg(short = 'q', long = "quiet")]
    quiet: bool,

    /// Verbose output (debug info to stderr)
    #[arg(short = 'v', long = "verbose")]
    verbose: bool,

    /// When to use colors
    #[arg(long = "color", value_enum, default_value = "auto")]
    color: ColorChoice,

    /// Filter blocks matching CSS-like selector
    #[arg(long = "select")]
    select: Option<String>,

    /// Skip loading .wasot-ids index for resolving external document references
    #[arg(long = "no-id-index")]
    no_id_index: bool,

    /// Disable auto-generation of IDs for blocks without explicit IDs
    ///
    /// By default, the parser pre-scans the file to find the maximum existing ID,
    /// then assigns auto-generated IDs to blocks without explicit IDs.
    /// This flag disables both the pre-scan and auto-ID generation.
    #[arg(long = "no-auto-ids")]
    no_auto_ids: bool,

    /// Input file(s) - reads from stdin if not provided
    files: Vec<String>,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)]
enum Format {
    /// Wasot format
    Wasot,
    /// XML format
    Xml,
    /// HTML format
    Html,
    /// Binary protobuf format
    Binary,
    /// JSON Lines format
    Jsonl,
    /// SQL file format (SQL statements to create a database)
    Sql,
    /// SQLite database format
    #[cfg(feature = "sqlite")]
    Sqlite,
    /// Markdown format (input only)
    Markdown,
    /// S-expression format (tree-sitter style)
    #[cfg(feature = "sexp")]
    Sexp,
}

#[derive(Clone, Copy, Debug, Default, ValueEnum)]
enum ColorChoice {
    /// Auto-detect based on terminal
    #[default]
    Auto,
    /// Always use colors
    Always,
    /// Never use colors
    Never,
}

#[derive(Subcommand)]
enum Commands {
    /// Convert a Wasot file to SQLite database
    #[cfg(feature = "sqlite")]
    ToSqlite {
        /// Path to the Wasot file to convert (optional, reads from stdin if not provided)
        file_path: Option<String>,

        /// Path to the SQLite database file
        #[arg(short = 'd', long)]
        database: String,
    },

    /// Convert a SQLite database back to Wasot format
    #[cfg(feature = "sqlite")]
    FromSqlite {
        /// Path to the SQLite database file
        #[arg(short = 'd', long)]
        database: String,

        /// Output file path (optional, writes to stdout if not provided)
        #[arg(short = 'o', long)]
        output: Option<String>,
    },

    /// Document processing tools
    Tools {
        #[command(subcommand)]
        command: ToolCommands,
    },

    /// Start the Language Server Protocol (LSP) server
    ///
    /// The LSP server provides IDE features like go-to-definition for links,
    /// hover information, and document indexing. Run this command and connect
    /// your editor to stdin/stdout.
    Lsp,
}

#[derive(Subcommand)]
enum ToolCommands {
    /// Validate wasot file syntax (exit 0 if valid, 1 if invalid)
    Check {
        /// Files to validate (reads from stdin if not provided)
        files: Vec<String>,
    },

    /// Show document statistics (block counts, depth, size)
    Info {
        /// File to analyze (reads from stdin if not provided)
        file: Option<String>,
    },

    /// Pretty-print wasot files in place
    ///
    /// Accepts files or directories. For directories, recursively finds all .wasot files.
    /// Files are reformatted in place with consistent indentation and auto-generated IDs.
    Pretty {
        /// Path(s) to files or directories to process (default: current directory)
        paths: Vec<String>,

        /// Print to stdout instead of modifying files
        #[arg(long)]
        stdout: bool,

        /// Show which files are being processed
        #[arg(short, long)]
        verbose: bool,

        /// Skip confirmation prompt for multiple files
        #[arg(short, long)]
        yes: bool,
    },

    /// Generate shell completion scripts
    Completions {
        /// The shell to generate completions for
        #[arg(value_enum)]
        shell: Shell,
    },

    /// Manage document IDs and the .wasot-ids index file
    #[command(subcommand)]
    DocumentIds(DocumentIdsCommands),
}

#[derive(Subcommand)]
enum DocumentIdsCommands {
    /// Scan files and print document IDs to stdout
    ///
    /// Accepts files or directories. For directories, recursively finds all .wasot files.
    /// Output format: <document-id> <relative-path>
    /// Files without an ID produce no output.
    Scan {
        /// Path(s) to files or directories to process (default: current directory)
        paths: Vec<String>,
    },

    /// Update the .wasot-ids index file
    ///
    /// Scans all .wasot files in the specified paths and writes the index to .wasot-ids.
    /// Format: <document-id> <relative-path-from-index-file>
    /// For duplicate IDs, last write wins (no error).
    /// If no paths are given, scans the directory containing the output file.
    Update {
        /// Path(s) to files or directories to scan (default: output file's directory)
        paths: Vec<String>,

        /// Output file path (default: .wasot-ids in current directory)
        #[arg(short, long, default_value = ".wasot-ids")]
        output: String,
    },

    /// Get the path for a document ID from the index
    ///
    /// Looks up the ID in .wasot-ids and prints the path if found.
    /// Exits with code 1 if not found.
    Get {
        /// The document ID to look up
        id: String,

        /// Index file path (default: .wasot-ids in current directory)
        #[arg(short, long, default_value = ".wasot-ids")]
        index: String,
    },
}

/// Detect input format from content by examining magic bytes and content
/// Detection priority:
/// 1. Check for Wasot binary magic bytes
/// 2. Check UTF-8 validity - invalid UTF-8 means binary
/// 3. For valid UTF-8, detect format from first meaningful character
fn detect_format_from_content(data: &[u8]) -> Format {
    // First check for Wasot binary magic bytes
    #[cfg(feature = "binary")]
    {
        use wasot_compiler::binary_format;
        if binary_format::check_header(data).is_some() {
            return Format::Binary;
        }
    }

    // Check UTF-8 validity - if not valid, assume binary
    let text = match std::str::from_utf8(data.get(..data.len().min(256)).unwrap_or(data)) {
        Ok(t) => t,
        Err(_) => return Format::Binary,
    };

    // Skip whitespace to find first meaningful character
    let first_char = match text.chars().find(|c| !c.is_whitespace()) {
        Some(c) => c,
        None => return Format::Wasot, // Empty or whitespace-only = assume wasot
    };

    // Detect format from content - all valid UTF-8 text is either wasot, XML, HTML, or JSONL
    match first_char {
        '<' => {
            // Distinguish HTML from XML by checking for HTML markers
            let text_lower = text.to_lowercase();
            if text_lower.starts_with("<!doctype html")
                || text_lower.trim_start().starts_with("<html")
            {
                Format::Html
            } else {
                Format::Xml
            }
        }
        '{' | '[' => Format::Jsonl, // JSON object or array
        _ => Format::Wasot,         // Everything else is wasot (text content)
    }
}

/// Check if data appears to be binary (wrapper for backwards compat)
fn is_likely_binary(data: &[u8]) -> bool {
    detect_format_from_content(data) == Format::Binary
}

/// Detect format from file extension
fn detect_format_from_extension(path: &str) -> Option<Format> {
    use std::path::Path;
    let ext = Path::new(path).extension().and_then(|e| e.to_str())?;
    match ext {
        "wasot" => Some(Format::Wasot),
        "xml" => Some(Format::Xml),
        "html" | "htm" => Some(Format::Html),
        "jsonl" => Some(Format::Jsonl),
        "wasotb" => Some(Format::Binary),
        "sql" => Some(Format::Sql),
        "md" | "markdown" => Some(Format::Markdown),
        #[cfg(feature = "sexp")]
        "sexp" => Some(Format::Sexp),
        _ => None,
    }
}

/// Default command: convert/normalize files based on global options
async fn process_default(opts: &GlobalOpts) -> Result<(), Box<dyn std::error::Error>> {
    // Get input: first file from args or stdin
    let file_path = opts.files.first().map(String::as_str);

    // Determine output format:
    // 1. Explicit -t/--to flag
    // 2. Detect from -o/--output extension
    // 3. Default to wasot (normalization)
    let output_format = opts.to.unwrap_or_else(|| {
        opts.output
            .as_deref()
            .and_then(detect_format_from_extension)
            .unwrap_or(Format::Wasot)
    });

    // For stdin input without explicit format, we need to detect from content
    // Buffer stdin content and create a temp file for processing
    let (effective_file_path, _temp_file): (Option<String>, Option<tempfile::NamedTempFile>) =
        if file_path.is_none() && opts.from.is_none() {
            // Read stdin to detect format
            let mut buffer = Vec::new();
            io::stdin().read_to_end(&mut buffer)?;
            let detected_format = detect_format_from_content(&buffer);

            // If not wasot, write to temp file for processing
            if detected_format != Format::Wasot && detected_format != Format::Binary {
                let extension = match detected_format {
                    Format::Html => "html",
                    Format::Xml => "xml",
                    Format::Jsonl => "jsonl",
                    _ => "tmp",
                };
                let temp = tempfile::Builder::new()
                    .suffix(&format!(".{extension}"))
                    .tempfile()?;
                std::fs::write(temp.path(), &buffer)?;
                (Some(temp.path().to_string_lossy().to_string()), Some(temp))
            } else {
                // For wasot/binary, write to temp file too so stdin isn't consumed twice
                let temp = tempfile::Builder::new().suffix(".wasot").tempfile()?;
                std::fs::write(temp.path(), &buffer)?;
                (Some(temp.path().to_string_lossy().to_string()), Some(temp))
            }
        } else {
            (file_path.map(String::from), None)
        };

    let file_path_ref = effective_file_path.as_deref();

    // Determine input format:
    // 1. Explicit -f/--from flag
    // 2. Detect from input file extension
    // 3. Default to wasot
    let input_format = opts.from.unwrap_or_else(|| {
        file_path_ref
            .and_then(detect_format_from_extension)
            .unwrap_or(Format::Wasot)
    });

    // Handle --select if provided
    if let Some(selector) = &opts.select {
        return match input_format {
            Format::Wasot => process_with_select(
                file_path_ref,
                selector,
                output_format,
                opts.output.as_deref(),
                opts.no_auto_ids,
            ),
            _ => Err(
                format!("--select is only supported for wasot input (got {input_format:?})").into(),
            ),
        };
    }

    // Route to appropriate conversion based on formats
    match (input_format, output_format) {
        (Format::Wasot, Format::Wasot) => {
            // Normalize wasot (same as pretty)
            process_pretty(file_path_ref, opts.output.as_deref(), opts.no_auto_ids)
        }
        (Format::Wasot, Format::Xml) => {
            // Use existing to-xml logic
            process_to_xml(
                file_path_ref,
                opts.output.as_deref(),
                true,
                opts.no_auto_ids,
            )
        }
        (Format::Wasot, Format::Html) => {
            // Convert wasot to HTML
            process_to_html(file_path_ref, true, opts.no_auto_ids)
        }
        #[cfg(feature = "binary")]
        (Format::Wasot, Format::Binary) => {
            process_to_binary(file_path_ref, opts.output.as_deref(), opts.no_auto_ids)
        }
        (Format::Wasot, Format::Jsonl) => {
            process_to_jsonl(file_path_ref, opts.output.as_deref(), opts.no_auto_ids)
        }
        (Format::Wasot, Format::Sql) => {
            process_to_sql(file_path_ref, opts.output.as_deref(), opts.no_auto_ids)
        }
        #[cfg(feature = "sexp")]
        (Format::Wasot, Format::Sexp) => process_to_sexp(
            file_path_ref,
            opts.output.as_deref(),
            true,
            opts.no_auto_ids,
        ),
        #[cfg(feature = "binary")]
        (Format::Binary, Format::Wasot) | (Format::Binary, Format::Jsonl) => {
            // Binary input - use auto-detecting functions
            if output_format == Format::Jsonl {
                process_to_jsonl(file_path_ref, opts.output.as_deref(), opts.no_auto_ids)
            } else {
                process_to_wasot(file_path_ref, opts.output.as_deref(), false)
            }
        }
        #[cfg(all(feature = "binary", feature = "sexp"))]
        (Format::Binary, Format::Sexp) => {
            process_binary_to_sexp(file_path_ref, opts.output.as_deref())
        }
        (Format::Jsonl, Format::Wasot) => {
            process_convert(
                Format::Jsonl,
                Format::Wasot,
                file_path_ref,
                opts.output.as_deref(),
            )
            .await
        }
        (Format::Xml, Format::Wasot) => {
            process_convert(
                Format::Xml,
                Format::Wasot,
                file_path_ref,
                opts.output.as_deref(),
            )
            .await
        }
        #[cfg(feature = "sqlite")]
        (Format::Wasot, Format::Sqlite) => {
            if let Some(out) = &opts.output {
                process_to_sqlite(file_path_ref, out, opts.no_auto_ids).await
            } else {
                Err("SQLite output requires -o/--output file path".into())
            }
        }
        #[cfg(feature = "sqlite")]
        (Format::Sqlite, Format::Wasot) => {
            if let Some(path) = file_path_ref {
                process_from_sqlite(path, opts.output.as_deref())
            } else {
                Err("SQLite input requires a file path".into())
            }
        }
        #[cfg(feature = "sqlite")]
        (Format::Sql, Format::Wasot) => {
            process_from_sql(file_path_ref, opts.output.as_deref()).await
        }
        #[cfg(feature = "sqlite")]
        (Format::Sql, _) => {
            process_from_sql_to_format(file_path_ref, output_format, opts.output.as_deref()).await
        }
        // HTML input conversions
        #[cfg(feature = "html")]
        (Format::Html, Format::Wasot) => process_from_html(file_path_ref, opts.output.as_deref()),
        #[cfg(feature = "html")]
        (Format::Html, _) => {
            process_from_html_to_format(file_path_ref, output_format, opts.output.as_deref())
        }
        // Markdown input conversions
        (Format::Markdown, Format::Wasot) => {
            process_from_markdown(file_path_ref, opts.output.as_deref())
        }
        (Format::Markdown, _) => {
            process_from_markdown_to_format(file_path_ref, output_format, opts.output.as_deref())
        }
        _ => Err(format!(
            "Conversion from {input_format:?} to {output_format:?} is not yet implemented"
        )
        .into()),
    }
}

fn process_pretty(
    file_path: Option<&str>,
    output_path: Option<&str>,
    no_auto_ids: bool,
) -> Result<(), Box<dyn std::error::Error>> {
    use wasot_compiler::events::run;

    let emitter = match file_path {
        Some(path) => {
            use std::path::Path;
            if !Path::new(path).exists() {
                return Err(format!("File not found: {path}").into());
            }
            WasotFileEmitter::from_file(path).with_auto_generate_ids(!no_auto_ids)
        }
        None => {
            let mut buffer = String::new();
            io::stdin().read_to_string(&mut buffer)?;
            WasotFileEmitter::from_string(buffer).with_auto_generate_ids(!no_auto_ids)
        }
    };

    let result = match output_path {
        Some(path) => {
            let file = std::fs::File::create(path)?;
            let writer = BufWriter::new(file);
            let reconstructor = WasotReconstructor::new_with_writer(writer);
            run(emitter, reconstructor)
        }
        None => {
            let reconstructor = WasotReconstructor::new();
            run(emitter, reconstructor)
        }
    };

    result.map_err(|e| e.to_string())?;
    Ok(())
}

/// Process wasot input with selector filter, outputting in specified format
fn process_with_select(
    file_path: Option<&str>,
    selector: &str,
    output_format: Format,
    output_path: Option<&str>,
    no_auto_ids: bool,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::thread;
    use wasot_compiler::context::Context;

    // Create the input emitter
    let emitter = match file_path {
        Some(path) => {
            use std::path::Path;
            if !Path::new(path).exists() {
                return Err(format!("File not found: {path}").into());
            }
            WasotFileEmitter::from_file(path).with_auto_generate_ids(!no_auto_ids)
        }
        None => {
            let mut buffer = String::new();
            io::stdin().read_to_string(&mut buffer)?;
            WasotFileEmitter::from_string(buffer).with_auto_generate_ids(!no_auto_ids)
        }
    };

    // Create the selector consumer
    let mut selector_consumer = SelectorConsumer::new(vec![selector.to_string()]);

    // Get receivers
    let source_receiver = emitter.subscribe();
    let filtered_receiver = selector_consumer.subscribe();

    // Spawn emitter thread
    let emitter_handle = thread::spawn(move || {
        emitter.run();
    });

    // Spawn selector filter thread
    let selector_handle = thread::spawn(move || {
        let mut context = Context::new();
        for event in source_receiver {
            selector_consumer.handle_with_context(event, &mut context);
        }
    });

    // Consume filtered events and output in requested format
    match output_format {
        Format::Wasot => {
            // Output as wasot
            match output_path {
                Some(path) => {
                    let file = std::fs::File::create(path)?;
                    let writer = BufWriter::new(file);
                    let mut reconstructor = WasotReconstructor::new_with_writer(writer);
                    for event in filtered_receiver {
                        reconstructor.handle(event);
                    }
                }
                None => {
                    let mut reconstructor = WasotReconstructor::new();
                    for event in filtered_receiver {
                        reconstructor.handle(event);
                    }
                }
            }
        }
        Format::Jsonl => {
            // Output as JSONL - use temp file approach like process_to_jsonl
            use std::fs;
            use std::io::{BufRead, Write};

            let temp_file = tempfile::NamedTempFile::new()?;
            let temp_path = temp_file.path().to_string_lossy().to_string();
            let mut exporter = ToJsonlConsumer::new(&temp_path)?;

            for event in filtered_receiver {
                exporter.process_event(&event);
            }
            exporter.finish()?;

            match output_path {
                Some(path) => {
                    // Copy temp file to output path
                    fs::copy(&temp_path, path)?;
                }
                None => {
                    // Write to stdout
                    let file = fs::File::open(&temp_path)?;
                    let reader = io::BufReader::new(file);
                    let stdout = io::stdout();
                    let mut handle = stdout.lock();
                    for line in reader.lines() {
                        writeln!(handle, "{}", line?)?;
                    }
                }
            }
        }
        Format::Xml => {
            // Output as XML
            let mut xml_listener = XmlListener::new(true);
            for event in filtered_receiver {
                xml_listener.handle(event);
            }
        }
        #[cfg(feature = "binary")]
        Format::Binary => {
            // Output as binary protobuf
            match output_path {
                Some(path) => {
                    let file = std::fs::File::create(path)?;
                    let writer = BufWriter::new(file);
                    let mut consumer = BinaryProtobufConsumer::new(writer);
                    for event in filtered_receiver {
                        consumer.convert_and_write(&event)?;
                    }
                }
                None => {
                    let stdout = io::stdout();
                    let writer = BufWriter::new(stdout.lock());
                    let mut consumer = BinaryProtobufConsumer::new(writer);
                    for event in filtered_receiver {
                        consumer.convert_and_write(&event)?;
                    }
                }
            }
        }
        Format::Sql => {
            // Output as SQL
            use std::fs;
            use std::io::Write;

            let temp_file = tempfile::NamedTempFile::new()?;
            let temp_path = temp_file.path().to_string_lossy().to_string();
            let mut consumer = SqlFileConsumer::new(&temp_path)?;

            for event in filtered_receiver {
                consumer.process_event(&event);
            }
            consumer.finish()?;

            match output_path {
                Some(path) => {
                    fs::copy(&temp_path, path)?;
                }
                None => {
                    let contents = fs::read_to_string(&temp_path)?;
                    let stdout = io::stdout();
                    let mut handle = stdout.lock();
                    write!(handle, "{contents}")?;
                }
            }
        }
        #[cfg(feature = "sqlite")]
        Format::Sqlite => {
            return Err("--select with SQLite output is not supported".into());
        }
        Format::Html => {
            // Output as HTML
            use wasot_compiler::HtmlListener;
            let mut html_listener = HtmlListener::new(true);
            for event in filtered_receiver {
                html_listener.handle(event);
            }
        }
        #[cfg(feature = "sexp")]
        Format::Sexp => {
            // Output as S-expression
            match output_path {
                Some(path) => {
                    let file = std::fs::File::create(path)?;
                    let writer = BufWriter::new(file);
                    let mut sexp_consumer = ToSexpConsumer::new(writer, true);
                    for event in filtered_receiver {
                        sexp_consumer.handle(event);
                    }
                    sexp_consumer.finish()?;
                }
                None => {
                    let mut sexp_consumer = ToSexpConsumer::new(std::io::stdout(), true);
                    for event in filtered_receiver {
                        sexp_consumer.handle(event);
                    }
                }
            }
        }
        #[cfg(not(feature = "binary"))]
        Format::Binary => {
            return Err("Binary format requires the 'binary' feature".into());
        }
        Format::Markdown => {
            return Err("Markdown is only supported as an input format, not output".into());
        }
    }

    // Wait for threads to finish
    emitter_handle
        .join()
        .map_err(|_| "Emitter thread panicked")?;
    selector_handle
        .join()
        .map_err(|_| "Selector thread panicked")?;

    Ok(())
}

/// Validate wasot files. Returns Ok if all valid, Err if any invalid.
fn process_check(files: &[String]) -> Result<(), Box<dyn std::error::Error>> {
    use std::path::Path;
    use std::thread;
    use wasot_compiler::events::events::Emitter;

    let mut all_valid = true;
    let mut files_checked = 0;

    // If no files provided, read from stdin
    let files_to_check: Vec<String> = if files.is_empty() {
        vec!["-".to_string()]
    } else {
        files.to_vec()
    };

    for file_path in &files_to_check {
        let result: Result<(), Box<dyn std::error::Error>> = if file_path == "-" {
            // Read from stdin
            let mut buffer = String::new();
            io::stdin().read_to_string(&mut buffer)?;
            let emitter = WasotFileEmitter::from_string(buffer);
            let receiver = emitter.subscribe();

            let emitter_handle = thread::spawn(move || {
                emitter.run();
            });

            // Just consume all events - if parsing succeeds, file is valid
            for _event in receiver {
                // Valid event received
            }

            emitter_handle
                .join()
                .map_err(|_| "Parser thread panicked")?;
            Ok(())
        } else {
            // Check file exists
            if !Path::new(file_path).exists() {
                eprintln!("{file_path}: file not found");
                all_valid = false;
                continue;
            }

            let emitter = WasotFileEmitter::from_file(file_path);
            let receiver = emitter.subscribe();

            let emitter_handle = thread::spawn(move || {
                emitter.run();
            });

            // Just consume all events
            for _event in receiver {
                // Valid event received
            }

            emitter_handle
                .join()
                .map_err(|_| "Parser thread panicked")?;
            Ok(())
        };

        match result {
            Ok(()) => {
                files_checked += 1;
            }
            Err(e) => {
                eprintln!("{file_path}: {e}");
                all_valid = false;
            }
        }
    }

    if all_valid {
        if files_checked > 1 {
            eprintln!("{files_checked} files validated successfully");
        }
        Ok(())
    } else {
        Err("validation failed".into())
    }
}

/// Pretty-print wasot files in place (batch mode)
/// Accepts files or directories. For directories, recursively finds all .wasot files.
/// Respects .gitignore by default.
fn process_pretty_batch(
    paths: &[String],
    to_stdout: bool,
    verbose: bool,
    skip_confirm: bool,
) -> Result<(), Box<dyn std::error::Error>> {
    use ignore::WalkBuilder;
    use std::io::{self, BufRead, Write};
    use std::path::Path;
    use std::thread;
    use wasot_compiler::events::events::Emitter;

    // Default to current directory if no paths given
    let paths = if paths.is_empty() {
        vec![".".to_string()]
    } else {
        paths.to_vec()
    };

    // Collect all .wasot files from all paths, respecting .gitignore
    let mut all_files = Vec::new();
    for path_str in &paths {
        let path = Path::new(path_str);

        if !path.exists() {
            return Err(format!("Path not found: {path_str}").into());
        }

        if path.is_file() {
            // Single file - add directly (even if in .gitignore, user explicitly specified it)
            all_files.push(path.to_path_buf());
        } else {
            // Directory - walk with .gitignore support
            let walker = WalkBuilder::new(path)
                .hidden(false) // Don't skip hidden files by default
                .git_ignore(true) // Respect .gitignore
                .git_global(true) // Respect global gitignore
                .git_exclude(true) // Respect .git/info/exclude
                .build();

            for entry in walker.flatten() {
                let entry_path = entry.path();
                if entry_path.is_file() && entry_path.extension().is_some_and(|ext| ext == "wasot")
                {
                    all_files.push(entry_path.to_path_buf());
                }
            }
        }
    }

    if all_files.is_empty() {
        eprintln!("No .wasot files found");
        return Ok(());
    }

    // Ask for confirmation if multiple files and not writing to stdout
    if all_files.len() > 1 && !to_stdout && !skip_confirm {
        eprintln!(
            "{} .wasot files will be modified in place.",
            all_files.len()
        );
        eprint!("Do you want to continue? [Y/n] ");
        io::stderr().flush()?;

        let mut input = String::new();
        io::stdin().lock().read_line(&mut input)?;
        let input = input.trim().to_lowercase();

        // Default is yes (empty input or 'y'/'yes')
        if input == "n" || input == "no" {
            eprintln!("Aborted.");
            return Ok(());
        }
    }

    let mut processed = 0;
    let mut errors = 0;

    for file_path in &all_files {
        let file_str = file_path.to_string_lossy();

        if verbose {
            eprintln!("Processing: {file_str}");
        }

        // Read file content
        let content = match std::fs::read_to_string(file_path) {
            Ok(c) => c,
            Err(e) => {
                eprintln!("{file_str}: {e}");
                errors += 1;
                continue;
            }
        };

        // Parse and reconstruct
        let emitter = WasotFileEmitter::from_string(content);
        let receiver = emitter.subscribe();

        // Create output buffer
        let mut output = Vec::new();

        let emitter_handle = thread::spawn(move || {
            emitter.run();
        });

        // Reconstruct with pretty printing
        {
            let mut reconstructor = WasotReconstructor::new_with_writer(&mut output);
            for event in receiver {
                reconstructor.on_element_event(&event);
            }
        }

        if let Err(e) = emitter_handle.join() {
            eprintln!("{file_str}: parser thread panicked: {e:?}");
            errors += 1;
            continue;
        }

        // Output result
        if to_stdout {
            // Print separator for multiple files
            if all_files.len() > 1 {
                println!("--- {file_str} ---");
            }
            print!("{}", String::from_utf8_lossy(&output));
        } else {
            // Write back to file
            if let Err(e) = std::fs::write(file_path, &output) {
                eprintln!("{file_str}: failed to write: {e}");
                errors += 1;
                continue;
            }
        }

        processed += 1;
    }

    if verbose || all_files.len() > 1 {
        eprintln!(
            "Processed {processed} file{}{}",
            if processed == 1 { "" } else { "s" },
            if errors > 0 {
                format!(", {errors} error{}", if errors == 1 { "" } else { "s" })
            } else {
                String::new()
            }
        );
    }

    if errors > 0 {
        Err(format!("{errors} file(s) failed").into())
    } else {
        Ok(())
    }
}

/// Show document statistics
fn process_info(file_path: Option<&str>) -> Result<(), Box<dyn std::error::Error>> {
    use std::collections::HashMap;
    use std::path::Path;
    use std::thread;
    use wasot_compiler::events::events::Emitter;
    use wasot_compiler::events::Event;

    let (emitter, file_size, line_count) = match file_path {
        Some(path) => {
            if !Path::new(path).exists() {
                return Err(format!("File not found: {path}").into());
            }
            let metadata = std::fs::metadata(path)?;
            let content = std::fs::read_to_string(path)?;
            let lines = content.lines().count();
            (WasotFileEmitter::from_file(path), metadata.len(), lines)
        }
        None => {
            let mut buffer = String::new();
            io::stdin().read_to_string(&mut buffer)?;
            let bytes = buffer.len() as u64;
            let lines = buffer.lines().count();
            (WasotFileEmitter::from_string(buffer), bytes, lines)
        }
    };

    let receiver = emitter.subscribe();

    let emitter_handle = thread::spawn(move || {
        emitter.run();
    });

    // Collect statistics
    let mut block_counts: HashMap<String, usize> = HashMap::new();
    let mut total_blocks: usize = 0;
    let mut max_depth: usize = 0;

    for event in receiver {
        if let Event::BlockStarted { meta_line } = event {
            total_blocks += 1;
            let depth = meta_line.level;
            if depth > max_depth {
                max_depth = depth;
            }
            *block_counts.entry(meta_line.type_name.clone()).or_insert(0) += 1;
        }
    }

    emitter_handle
        .join()
        .map_err(|_| "Parser thread panicked")?;

    // Output statistics
    println!("blocks: {total_blocks}");
    let mut sorted_types: Vec<_> = block_counts.iter().collect();
    sorted_types.sort_by(|a, b| b.1.cmp(a.1)); // Sort by count descending
    for (type_name, count) in sorted_types {
        println!("  {type_name}: {count}");
    }
    println!("depth: {max_depth}");
    println!("lines: {line_count}");
    println!("bytes: {file_size}");

    Ok(())
}

#[cfg(feature = "sqlite")]
async fn process_to_sqlite(
    file_path: Option<&str>,
    database_path: &str,
    no_auto_ids: bool,
) -> Result<(), Box<dyn std::error::Error>> {
    std::fs::File::create(database_path)?;
    let database_url = format!("sqlite:{database_path}");
    let mut sqlite_consumer = SqliteConsumer::new(&database_url).await?;

    match file_path {
        Some(path) => {
            let emitter = WasotFileEmitter::from_file(path).with_auto_generate_ids(!no_auto_ids);
            let events = emitter.subscribe();

            let emitter_handle = std::thread::spawn(move || {
                emitter.run();
            });

            sqlite_consumer.run(events);
            emitter_handle.join().unwrap();
        }
        None => {
            let mut buffer = String::new();
            io::stdin().read_to_string(&mut buffer)?;

            let emitter =
                WasotFileEmitter::from_string(buffer).with_auto_generate_ids(!no_auto_ids);
            let events = emitter.subscribe();

            let emitter_handle = std::thread::spawn(move || {
                emitter.run();
            });

            sqlite_consumer.run(events);
            emitter_handle.join().unwrap();
        }
    }

    Ok(())
}

#[cfg(feature = "sqlite")]
fn process_from_sqlite(
    database_path: &str,
    output_path: Option<&str>,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::thread;

    let database_url = format!("sqlite:{database_path}");

    let emitter = if let Ok(handle) = tokio::runtime::Handle::try_current() {
        tokio::task::block_in_place(|| {
            handle.block_on(SqliteEmitter::from_database_url(&database_url))
        })?
    } else {
        let rt = tokio::runtime::Runtime::new()?;
        rt.block_on(SqliteEmitter::from_database_url(&database_url))?
    };
    let receiver = emitter.subscribe();

    let emitter_handle = thread::spawn(move || {
        emitter.run();
    });

    match output_path {
        Some(path) => {
            let file = std::fs::File::create(path)?;
            let writer = BufWriter::new(file);
            let mut reconstructor = WasotReconstructor::new_with_writer(writer);

            for event in receiver {
                reconstructor.on_element_event(&event);
            }

            emitter_handle
                .join()
                .map_err(|_| "Emitter thread panicked")?;
        }
        None => {
            let mut reconstructor = WasotReconstructor::new();

            for event in receiver {
                reconstructor.on_element_event(&event);
            }

            emitter_handle
                .join()
                .map_err(|_| "Emitter thread panicked")?;
        }
    }

    Ok(())
}

#[cfg(feature = "sqlite")]
async fn process_from_sql(
    file_path: Option<&str>,
    output_path: Option<&str>,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::thread;

    let sql_content = match file_path {
        Some(path) => std::fs::read_to_string(path)?,
        None => {
            let mut buffer = String::new();
            io::stdin().read_to_string(&mut buffer)?;
            buffer
        }
    };

    let emitter = SqlFileEmitter::from_string(sql_content).await?;
    let receiver = emitter.subscribe();

    let emitter_handle = thread::spawn(move || {
        emitter.run();
    });

    match output_path {
        Some(path) => {
            let file = std::fs::File::create(path)?;
            let writer = BufWriter::new(file);
            let mut reconstructor = WasotReconstructor::new_with_writer(writer);

            for event in receiver {
                reconstructor.on_element_event(&event);
            }

            emitter_handle
                .join()
                .map_err(|_| "Emitter thread panicked")?;
        }
        None => {
            let mut reconstructor = WasotReconstructor::new();

            for event in receiver {
                reconstructor.on_element_event(&event);
            }

            emitter_handle
                .join()
                .map_err(|_| "Emitter thread panicked")?;
        }
    }

    Ok(())
}

#[cfg(feature = "sqlite")]
async fn process_from_sql_to_format(
    file_path: Option<&str>,
    output_format: Format,
    output_path: Option<&str>,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::fs;
    use std::io::Write;
    use std::thread;

    let sql_content = match file_path {
        Some(path) => std::fs::read_to_string(path)?,
        None => {
            let mut buffer = String::new();
            io::stdin().read_to_string(&mut buffer)?;
            buffer
        }
    };

    let emitter = SqlFileEmitter::from_string(sql_content).await?;
    let receiver = emitter.subscribe();

    let emitter_handle = thread::spawn(move || {
        emitter.run();
    });

    match output_format {
        Format::Xml => {
            let mut xml_listener = XmlListener::new(true);
            xml_listener.run(receiver);
        }
        Format::Jsonl => {
            let temp_file = tempfile::NamedTempFile::new()?;
            let temp_path = temp_file.path().to_string_lossy().to_string();
            let mut consumer = ToJsonlConsumer::new(&temp_path)?;

            for event in receiver {
                consumer.process_event(&event);
            }
            consumer.finish()?;

            match output_path {
                Some(path) => {
                    fs::copy(&temp_path, path)?;
                }
                None => {
                    let contents = fs::read_to_string(&temp_path)?;
                    let stdout = io::stdout();
                    let mut handle = stdout.lock();
                    write!(handle, "{contents}")?;
                }
            }
        }
        #[cfg(feature = "binary")]
        Format::Binary => match output_path {
            Some(path) => {
                let file = std::fs::File::create(path)?;
                let mut consumer = BinaryProtobufConsumer::new(file);
                for event in receiver {
                    consumer.convert_and_write(&event)?;
                }
            }
            None => {
                let mut consumer = BinaryProtobufConsumer::new(std::io::stdout());
                consumer.run(receiver);
            }
        },
        Format::Sql => {
            // SQL to SQL - just copy or output the original
            let sql_content = match file_path {
                Some(path) => std::fs::read_to_string(path)?,
                None => return Err("SQL to SQL conversion requires file input".into()),
            };
            match output_path {
                Some(path) => {
                    fs::write(path, sql_content)?;
                }
                None => {
                    print!("{sql_content}");
                }
            }
            // Consume receiver to avoid blocking
            for _ in receiver {}
        }
        _ => {
            return Err(
                format!("Conversion from Sql to {output_format:?} is not yet implemented").into(),
            );
        }
    }

    emitter_handle
        .join()
        .map_err(|_| "Emitter thread panicked")?;

    Ok(())
}

#[cfg(feature = "binary")]
fn process_to_binary(
    file_path: Option<&str>,
    output_path: Option<&str>,
    no_auto_ids: bool,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::path::Path;
    use std::thread;

    let emitter = match file_path {
        Some(path) => {
            if !Path::new(path).exists() {
                return Err(format!("File not found: {path}").into());
            }
            WasotFileEmitter::from_file(path).with_auto_generate_ids(!no_auto_ids)
        }
        None => {
            let mut buffer = String::new();
            io::stdin().read_to_string(&mut buffer)?;
            WasotFileEmitter::from_string(buffer).with_auto_generate_ids(!no_auto_ids)
        }
    };
    let receiver = emitter.subscribe();

    let emitter_handle = thread::spawn(move || {
        emitter.run();
    });

    match output_path {
        Some(path) => {
            let file = std::fs::File::create(path)?;
            let mut consumer = BinaryProtobufConsumer::new(file);
            for event in receiver {
                consumer.convert_and_write(&event)?;
            }
        }
        None => {
            let mut consumer = BinaryProtobufConsumer::new(std::io::stdout());
            consumer.run(receiver);
        }
    }

    emitter_handle
        .join()
        .map_err(|_| "Emitter thread panicked")?;

    Ok(())
}

fn process_to_jsonl(
    file_path: Option<&str>,
    output_path: Option<&str>,
    no_auto_ids: bool,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::fs;
    use std::io::{BufRead, Write};
    use std::thread;

    let (events_tx, events_rx) = mpsc::sync_channel(1024);

    let emitter_handle = match file_path {
        Some(path) => {
            // Read first few bytes to detect format
            // Note: 256 bytes to avoid truncating multi-byte UTF-8 sequences
            let mut file = std::fs::File::open(path)?;
            let mut header = [0u8; 256];
            let bytes_read = file.read(&mut header)?;

            if is_likely_binary(&header[..bytes_read]) {
                // Binary protobuf input
                #[cfg(feature = "binary")]
                {
                    let emitter = ProtobufBinaryEmitter::from_file(path);
                    let receiver = emitter.subscribe();
                    let emitter_thread = thread::spawn(move || {
                        emitter.run();
                    });
                    thread::spawn(move || {
                        for event in receiver {
                            if events_tx.send(event).is_err() {
                                break;
                            }
                        }
                        let _ = emitter_thread.join();
                    })
                }
                #[cfg(not(feature = "binary"))]
                {
                    return Err("Binary protobuf format requires the 'binary' feature".into());
                }
            } else {
                // Wasot text input
                let emitter =
                    WasotFileEmitter::from_file(path).with_auto_generate_ids(!no_auto_ids);
                let receiver = emitter.subscribe();
                // Spawn emitter in its own thread
                let emitter_thread = thread::spawn(move || {
                    emitter.run();
                });
                // Spawn receiver in another thread to avoid deadlock
                // (emitter blocks if bus fills up before receiver starts)
                thread::spawn(move || {
                    for event in receiver {
                        if events_tx.send(event).is_err() {
                            break;
                        }
                    }
                    let _ = emitter_thread.join();
                })
            }
        }
        None => {
            // Read all stdin to detect format
            let mut buffer = Vec::new();
            io::stdin().read_to_end(&mut buffer)?;

            if is_likely_binary(&buffer) {
                // Binary protobuf from stdin
                #[cfg(feature = "binary")]
                {
                    let emitter = ProtobufBinaryEmitter::from_bytes(buffer);
                    let receiver = emitter.subscribe();
                    let emitter_thread = thread::spawn(move || {
                        emitter.run();
                    });
                    thread::spawn(move || {
                        for event in receiver {
                            if events_tx.send(event).is_err() {
                                break;
                            }
                        }
                        let _ = emitter_thread.join();
                    })
                }
                #[cfg(not(feature = "binary"))]
                {
                    return Err("Binary protobuf format requires the 'binary' feature".into());
                }
            } else {
                // Wasot text from stdin
                let text = String::from_utf8(buffer).map_err(|e| e.to_string())?;
                let emitter =
                    WasotFileEmitter::from_string(text).with_auto_generate_ids(!no_auto_ids);
                let receiver = emitter.subscribe();
                let emitter_thread = thread::spawn(move || {
                    emitter.run();
                });
                thread::spawn(move || {
                    for event in receiver {
                        if events_tx.send(event).is_err() {
                            break;
                        }
                    }
                    let _ = emitter_thread.join();
                })
            }
        }
    };

    match output_path {
        Some(path) => {
            let mut exporter = ToJsonlConsumer::new(path)?;
            for event in events_rx {
                exporter.process_event(&event);
            }
            exporter.finish()?;
        }
        None => {
            let temp_file = tempfile::NamedTempFile::new()?;
            let temp_path = temp_file.path().to_string_lossy().to_string();
            let mut exporter = ToJsonlConsumer::new(&temp_path)?;

            for event in events_rx {
                exporter.process_event(&event);
            }
            exporter.finish()?;

            let file = fs::File::open(&temp_path)?;
            let reader = io::BufReader::new(file);
            let stdout = io::stdout();
            let mut handle = stdout.lock();
            for line in reader.lines() {
                writeln!(handle, "{}", line?)?;
            }
        }
    }

    emitter_handle.join().unwrap();

    Ok(())
}

fn process_to_xml(
    file_path: Option<&str>,
    output_path: Option<&str>,
    pretty: bool,
    no_auto_ids: bool,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::path::Path;
    use std::thread;
    use wasot_compiler::Event;

    let emitter = match file_path {
        Some(path) => {
            if !Path::new(path).exists() {
                return Err(format!("File not found: {path}").into());
            }
            WasotFileEmitter::from_file(path).with_auto_generate_ids(!no_auto_ids)
        }
        None => {
            let mut buffer = String::new();
            io::stdin().read_to_string(&mut buffer)?;
            WasotFileEmitter::from_string(buffer).with_auto_generate_ids(!no_auto_ids)
        }
    };

    let receiver = emitter.subscribe();

    let emitter_handle = thread::spawn(move || {
        emitter.run();
    });

    // Helper function to escape XML special characters
    fn escape_xml(s: &str) -> String {
        s.replace('&', "&amp;")
            .replace('<', "&lt;")
            .replace('>', "&gt;")
            .replace('"', "&quot;")
            .replace('\'', "&apos;")
    }

    match output_path {
        Some(path) => {
            use std::io::Write;
            // Write to file
            let file = std::fs::File::create(path)?;
            let mut writer = std::io::BufWriter::new(file);

            for event in receiver {
                // Write XML output based on events
                match &event {
                    Event::BlockStarted { meta_line, .. } => {
                        write!(
                            writer,
                            "<block type=\"{}\"",
                            escape_xml(&meta_line.type_name)
                        )?;
                        write!(writer, " level=\"{}\"", meta_line.level)?;
                        if !meta_line.id.value.is_empty() {
                            write!(writer, " id=\"{}\"", escape_xml(&meta_line.id.value))?;
                        }
                        writeln!(writer, ">")?;

                        if !meta_line.inline_text.content.is_empty() {
                            if meta_line.inline_text.quoted {
                                writeln!(
                                    writer,
                                    "  <inline-text quoted=\"true\">{}</inline-text>",
                                    escape_xml(&meta_line.inline_text.content)
                                )?;
                            } else {
                                writeln!(
                                    writer,
                                    "  <inline-text>{}</inline-text>",
                                    escape_xml(&meta_line.inline_text.content)
                                )?;
                            }
                        }

                        for arg in &meta_line.arguments.args {
                            if arg.quoted {
                                writeln!(
                                    writer,
                                    "  <arg quoted=\"true\">{}</arg>",
                                    escape_xml(&arg.content)
                                )?;
                            } else {
                                writeln!(writer, "  <arg>{}</arg>", escape_xml(&arg.content))?;
                            }
                        }

                        for (key, value) in &meta_line.arguments.kwargs {
                            write!(writer, "  <kwarg>")?;
                            if key.quoted {
                                write!(
                                    writer,
                                    "<key quoted=\"true\">{}</key>",
                                    escape_xml(&key.content)
                                )?;
                            } else {
                                write!(writer, "<key>{}</key>", escape_xml(&key.content))?;
                            }
                            if value.quoted {
                                write!(
                                    writer,
                                    "<value quoted=\"true\">{}</value>",
                                    escape_xml(&value.content)
                                )?;
                            } else {
                                write!(writer, "<value>{}</value>", escape_xml(&value.content))?;
                            }
                            writeln!(writer, "</kwarg>")?;
                        }
                    }
                    Event::TextFinished { lines, .. } => {
                        // Use CDATA to preserve whitespace exactly
                        // Each line is separated by newline; a trailing blank line means
                        // we end with \n\n (two newlines)
                        write!(writer, "  <text><![CDATA[")?;
                        for line in lines {
                            writeln!(writer, "{line}")?;
                        }
                        write!(writer, "]]></text>")?;
                        writeln!(writer)?;
                    }
                    Event::BlockFinished => {
                        writeln!(writer, "</block>")?;
                    }
                    Event::ContextLine { meta_line } => {
                        write!(
                            writer,
                            "<context-line type=\"{}\"",
                            escape_xml(&meta_line.type_name)
                        )?;
                        write!(writer, " level=\"{}\"", meta_line.level)?;
                        if !meta_line.id.value.is_empty() {
                            write!(writer, " id=\"{}\"", escape_xml(&meta_line.id.value))?;
                        }
                        if !meta_line.inline_text.content.is_empty() {
                            writeln!(
                                writer,
                                ">{}</context-line>",
                                escape_xml(&meta_line.inline_text.content)
                            )?;
                        } else {
                            writeln!(writer, "/>")?;
                        }
                    }
                    Event::VoidElement { meta_line } => {
                        write!(
                            writer,
                            "<block type=\"{}\"",
                            escape_xml(&meta_line.type_name)
                        )?;
                        write!(writer, " level=\"{}\"", meta_line.level)?;
                        if !meta_line.id.value.is_empty() {
                            write!(writer, " id=\"{}\"", escape_xml(&meta_line.id.value))?;
                        }
                        write!(writer, " void=\"true\"")?;

                        let has_content = !meta_line.inline_text.content.is_empty()
                            || !meta_line.arguments.args.is_empty()
                            || meta_line
                                .arguments
                                .kwargs
                                .iter()
                                .any(|(k, _)| k.content != "id");

                        if has_content {
                            writeln!(writer, ">")?;
                            if !meta_line.inline_text.content.is_empty() {
                                if meta_line.inline_text.quoted {
                                    writeln!(
                                        writer,
                                        "  <inline-text quoted=\"true\">{}</inline-text>",
                                        escape_xml(&meta_line.inline_text.content)
                                    )?;
                                } else {
                                    writeln!(
                                        writer,
                                        "  <inline-text>{}</inline-text>",
                                        escape_xml(&meta_line.inline_text.content)
                                    )?;
                                }
                            }
                            for arg in &meta_line.arguments.args {
                                if arg.quoted {
                                    writeln!(
                                        writer,
                                        "  <arg quoted=\"true\">{}</arg>",
                                        escape_xml(&arg.content)
                                    )?;
                                } else {
                                    writeln!(writer, "  <arg>{}</arg>", escape_xml(&arg.content))?;
                                }
                            }
                            for (key, value) in &meta_line.arguments.kwargs {
                                write!(writer, "  <kwarg>")?;
                                if key.quoted {
                                    write!(
                                        writer,
                                        "<key quoted=\"true\">{}</key>",
                                        escape_xml(&key.content)
                                    )?;
                                } else {
                                    write!(writer, "<key>{}</key>", escape_xml(&key.content))?;
                                }
                                if value.quoted {
                                    write!(
                                        writer,
                                        "<value quoted=\"true\">{}</value>",
                                        escape_xml(&value.content)
                                    )?;
                                } else {
                                    write!(
                                        writer,
                                        "<value>{}</value>",
                                        escape_xml(&value.content)
                                    )?;
                                }
                                writeln!(writer, "</kwarg>")?;
                            }
                            writeln!(writer, "</block>")?;
                        } else {
                            writeln!(writer, "/>")?;
                        }
                    }
                }
            }
            writer.flush()?;
        }
        None => {
            // Write to stdout
            let mut xml_listener = XmlListener::new(pretty);
            xml_listener.run(receiver);
        }
    }

    emitter_handle
        .join()
        .map_err(|_| "Emitter thread panicked")?;

    Ok(())
}

fn process_to_html(
    file_path: Option<&str>,
    pretty: bool,
    no_auto_ids: bool,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::path::Path;
    use std::thread;
    use wasot_compiler::HtmlListener;

    let emitter = match file_path {
        Some(path) => {
            if !Path::new(path).exists() {
                return Err(format!("File not found: {path}").into());
            }
            WasotFileEmitter::from_file(path).with_auto_generate_ids(!no_auto_ids)
        }
        None => {
            let mut buffer = String::new();
            io::stdin().read_to_string(&mut buffer)?;
            WasotFileEmitter::from_string(buffer).with_auto_generate_ids(!no_auto_ids)
        }
    };

    let receiver = emitter.subscribe();

    let emitter_handle = thread::spawn(move || {
        emitter.run();
    });

    let mut html_listener = HtmlListener::new(pretty);
    html_listener.run(receiver);

    emitter_handle
        .join()
        .map_err(|_| "Emitter thread panicked")?;

    Ok(())
}

/// Convert wasot to S-expression format
#[cfg(feature = "sexp")]
fn process_to_sexp(
    file_path: Option<&str>,
    output_path: Option<&str>,
    pretty: bool,
    no_auto_ids: bool,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::io::BufWriter;
    use std::path::Path;
    use std::thread;

    let emitter = match file_path {
        Some(path) => {
            if !Path::new(path).exists() {
                return Err(format!("File not found: {path}").into());
            }
            WasotFileEmitter::from_file(path).with_auto_generate_ids(!no_auto_ids)
        }
        None => {
            let mut buffer = String::new();
            io::stdin().read_to_string(&mut buffer)?;
            WasotFileEmitter::from_string(buffer).with_auto_generate_ids(!no_auto_ids)
        }
    };

    let receiver = emitter.subscribe();

    let emitter_handle = thread::spawn(move || {
        emitter.run();
    });

    match output_path {
        Some(path) => {
            let file = std::fs::File::create(path)?;
            let writer = BufWriter::new(file);
            let mut sexp_consumer = ToSexpConsumer::new(writer, pretty);
            sexp_consumer.run(receiver);
            sexp_consumer.finish()?;
        }
        None => {
            let mut sexp_consumer = ToSexpConsumer::new(std::io::stdout(), pretty);
            sexp_consumer.run(receiver);
        }
    };

    emitter_handle
        .join()
        .map_err(|_| "Emitter thread panicked")?;

    Ok(())
}

/// Convert binary format to S-expression format
#[cfg(all(feature = "binary", feature = "sexp"))]
fn process_binary_to_sexp(
    file_path: Option<&str>,
    output_path: Option<&str>,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::io::BufWriter;
    use std::path::Path;
    use std::thread;

    let emitter = match file_path {
        Some(path) => {
            if !Path::new(path).exists() {
                return Err(format!("File not found: {path}").into());
            }
            ProtobufBinaryEmitter::from_file(path)
        }
        None => ProtobufBinaryEmitter::from_stdin(),
    };

    let receiver = emitter.subscribe();

    let emitter_handle = thread::spawn(move || {
        emitter.run();
    });

    match output_path {
        Some(path) => {
            let file = std::fs::File::create(path)?;
            let writer = BufWriter::new(file);
            let mut sexp_consumer = ToSexpConsumer::new(writer, true);
            sexp_consumer.run(receiver);
            sexp_consumer.finish()?;
        }
        None => {
            let mut sexp_consumer = ToSexpConsumer::new(std::io::stdout(), true);
            sexp_consumer.run(receiver);
        }
    };

    emitter_handle
        .join()
        .map_err(|_| "Emitter thread panicked")?;

    Ok(())
}

#[cfg(feature = "html")]
fn process_from_html(
    file_path: Option<&str>,
    output_path: Option<&str>,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::io::BufWriter;
    use std::path::Path;
    use std::thread;
    use wasot_compiler::{HtmlEmitter, WasotReconstructor};

    let emitter = match file_path {
        Some(path) => {
            if !Path::new(path).exists() {
                return Err(format!("File not found: {path}").into());
            }
            HtmlEmitter::from_file(path)?
        }
        None => {
            let mut buffer = String::new();
            io::stdin().read_to_string(&mut buffer)?;
            HtmlEmitter::from_string(buffer)
        }
    };

    let receiver = emitter.subscribe();

    let emitter_handle = thread::spawn(move || {
        emitter.run();
    });

    match output_path {
        Some(path) => {
            let file = std::fs::File::create(path)?;
            let writer = BufWriter::new(file);
            let mut reconstructor = WasotReconstructor::new_with_writer(writer);
            for event in receiver {
                reconstructor.handle(event);
            }
        }
        None => {
            let mut reconstructor = WasotReconstructor::new();
            for event in receiver {
                reconstructor.handle(event);
            }
        }
    }

    emitter_handle
        .join()
        .map_err(|_| "Emitter thread panicked")?;

    Ok(())
}

/// Convert HTML input to specified output format
#[cfg(feature = "html")]
fn process_from_html_to_format(
    file_path: Option<&str>,
    output_format: Format,
    output_path: Option<&str>,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::fs;
    use std::io::{BufWriter, Write};
    use std::path::Path;
    use std::thread;
    use wasot_compiler::HtmlEmitter;

    let emitter = match file_path {
        Some(path) => {
            if !Path::new(path).exists() {
                return Err(format!("File not found: {path}").into());
            }
            HtmlEmitter::from_file(path)?
        }
        None => {
            let mut buffer = String::new();
            io::stdin().read_to_string(&mut buffer)?;
            HtmlEmitter::from_string(buffer)
        }
    };

    let receiver = emitter.subscribe();

    let emitter_handle = thread::spawn(move || {
        emitter.run();
    });

    match output_format {
        Format::Xml => {
            let mut xml_listener = XmlListener::new(true);
            xml_listener.run(receiver);
        }
        Format::Html => {
            use wasot_compiler::HtmlListener;
            let mut html_listener = HtmlListener::new(true);
            html_listener.run(receiver);
        }
        Format::Jsonl => {
            let temp_file = tempfile::NamedTempFile::new()?;
            let temp_path = temp_file.path().to_string_lossy().to_string();
            let mut consumer = ToJsonlConsumer::new(&temp_path)?;

            for event in receiver {
                consumer.process_event(&event);
            }
            consumer.finish()?;

            match output_path {
                Some(path) => {
                    fs::copy(&temp_path, path)?;
                }
                None => {
                    let contents = fs::read_to_string(&temp_path)?;
                    let stdout = io::stdout();
                    let mut handle = stdout.lock();
                    write!(handle, "{contents}")?;
                }
            }
        }
        #[cfg(feature = "binary")]
        Format::Binary => match output_path {
            Some(path) => {
                let file = std::fs::File::create(path)?;
                let mut consumer = BinaryProtobufConsumer::new(file);
                for event in receiver {
                    consumer.convert_and_write(&event)?;
                }
            }
            None => {
                let mut consumer = BinaryProtobufConsumer::new(std::io::stdout());
                consumer.run(receiver);
            }
        },
        Format::Wasot => match output_path {
            Some(path) => {
                let file = std::fs::File::create(path)?;
                let writer = BufWriter::new(file);
                let mut reconstructor = WasotReconstructor::new_with_writer(writer);
                for event in receiver {
                    reconstructor.handle(event);
                }
            }
            None => {
                let mut reconstructor = WasotReconstructor::new();
                for event in receiver {
                    reconstructor.handle(event);
                }
            }
        },
        _ => {
            // Consume receiver to avoid blocking
            for _ in receiver {}
            return Err(format!(
                "Conversion from Html to {output_format:?} is not yet implemented"
            )
            .into());
        }
    }

    emitter_handle
        .join()
        .map_err(|_| "Emitter thread panicked")?;

    Ok(())
}

fn process_from_markdown(
    file_path: Option<&str>,
    output_path: Option<&str>,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::io::BufWriter;
    use std::path::Path;
    use std::thread;

    let emitter = match file_path {
        Some(path) => {
            if !Path::new(path).exists() {
                return Err(format!("File not found: {path}").into());
            }
            MarkdownEmitter::from_file(path)
        }
        None => {
            let mut buffer = String::new();
            io::stdin().read_to_string(&mut buffer)?;
            MarkdownEmitter::from_string(buffer)
        }
    };

    let receiver = emitter.subscribe();

    let emitter_handle = thread::spawn(move || {
        emitter.run();
    });

    match output_path {
        Some(path) => {
            let file = std::fs::File::create(path)?;
            let writer = BufWriter::new(file);
            let mut reconstructor = WasotReconstructor::new_with_writer(writer);
            for event in receiver {
                reconstructor.handle(event);
            }
        }
        None => {
            let mut reconstructor = WasotReconstructor::new();
            for event in receiver {
                reconstructor.handle(event);
            }
        }
    }

    emitter_handle
        .join()
        .map_err(|_| "Emitter thread panicked")?;

    Ok(())
}

/// Convert Markdown input to specified output format
fn process_from_markdown_to_format(
    file_path: Option<&str>,
    output_format: Format,
    output_path: Option<&str>,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::fs;
    use std::io::{BufWriter, Write};
    use std::path::Path;
    use std::thread;

    let emitter = match file_path {
        Some(path) => {
            if !Path::new(path).exists() {
                return Err(format!("File not found: {path}").into());
            }
            MarkdownEmitter::from_file(path)
        }
        None => {
            let mut buffer = String::new();
            io::stdin().read_to_string(&mut buffer)?;
            MarkdownEmitter::from_string(buffer)
        }
    };

    let receiver = emitter.subscribe();

    let emitter_handle = thread::spawn(move || {
        emitter.run();
    });

    match output_format {
        Format::Xml => {
            let mut xml_listener = XmlListener::new(true);
            xml_listener.run(receiver);
        }
        #[cfg(feature = "html")]
        Format::Html => {
            use wasot_compiler::HtmlListener;
            let mut html_listener = HtmlListener::new(true);
            html_listener.run(receiver);
        }
        Format::Jsonl => {
            let temp_file = tempfile::NamedTempFile::new()?;
            let temp_path = temp_file.path().to_string_lossy().to_string();
            let mut consumer = ToJsonlConsumer::new(&temp_path)?;

            for event in receiver {
                consumer.process_event(&event);
            }
            consumer.finish()?;

            match output_path {
                Some(path) => {
                    fs::copy(&temp_path, path)?;
                }
                None => {
                    let contents = fs::read_to_string(&temp_path)?;
                    let stdout = io::stdout();
                    let mut handle = stdout.lock();
                    write!(handle, "{contents}")?;
                }
            }
        }
        #[cfg(feature = "binary")]
        Format::Binary => match output_path {
            Some(path) => {
                let file = std::fs::File::create(path)?;
                let mut consumer = BinaryProtobufConsumer::new(file);
                for event in receiver {
                    consumer.convert_and_write(&event)?;
                }
            }
            None => {
                let mut consumer = BinaryProtobufConsumer::new(std::io::stdout());
                consumer.run(receiver);
            }
        },
        Format::Wasot => match output_path {
            Some(path) => {
                let file = std::fs::File::create(path)?;
                let writer = BufWriter::new(file);
                let mut reconstructor = WasotReconstructor::new_with_writer(writer);
                for event in receiver {
                    reconstructor.handle(event);
                }
            }
            None => {
                let mut reconstructor = WasotReconstructor::new();
                for event in receiver {
                    reconstructor.handle(event);
                }
            }
        },
        #[cfg(not(feature = "html"))]
        Format::Html => {
            // Consume receiver to avoid blocking
            for _ in receiver {}
            return Err("HTML output requires the 'html' feature".into());
        }
        _ => {
            // Consume receiver to avoid blocking
            for _ in receiver {}
            return Err(format!(
                "Conversion from Markdown to {output_format:?} is not yet implemented"
            )
            .into());
        }
    }

    emitter_handle
        .join()
        .map_err(|_| "Emitter thread panicked")?;

    Ok(())
}

fn process_to_sql(
    file_path: Option<&str>,
    output_path: Option<&str>,
    no_auto_ids: bool,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::fs;
    use std::io::Write;
    use std::path::Path;
    use std::thread;

    let emitter = match file_path {
        Some(path) => {
            if !Path::new(path).exists() {
                return Err(format!("File not found: {path}").into());
            }
            WasotFileEmitter::from_file(path).with_auto_generate_ids(!no_auto_ids)
        }
        None => {
            let mut buffer = String::new();
            io::stdin().read_to_string(&mut buffer)?;
            WasotFileEmitter::from_string(buffer).with_auto_generate_ids(!no_auto_ids)
        }
    };

    let receiver = emitter.subscribe();

    let emitter_handle = thread::spawn(move || {
        emitter.run();
    });

    match output_path {
        Some(path) => {
            let mut consumer = SqlFileConsumer::new(path)?;
            for event in receiver {
                consumer.process_event(&event);
            }
            consumer.finish()?;
        }
        None => {
            // Write to temp file, then copy to stdout
            let temp_file = tempfile::NamedTempFile::new()?;
            let temp_path = temp_file.path().to_string_lossy().to_string();
            let mut consumer = SqlFileConsumer::new(&temp_path)?;

            for event in receiver {
                consumer.process_event(&event);
            }
            consumer.finish()?;

            // Copy temp file to stdout
            let contents = fs::read_to_string(&temp_path)?;
            let stdout = io::stdout();
            let mut handle = stdout.lock();
            write!(handle, "{contents}")?;
        }
    }

    emitter_handle
        .join()
        .map_err(|_| "Emitter thread panicked")?;

    Ok(())
}

#[cfg(feature = "binary")]
fn process_to_wasot(
    file_path: Option<&str>,
    output_path: Option<&str>,
    paragraph_blocks: bool,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::fs::File;
    use std::thread;

    match file_path {
        Some(path) => {
            // Read first few bytes to detect format
            // Note: 256 bytes to avoid truncating multi-byte UTF-8 sequences
            let mut file = std::fs::File::open(path)?;
            let mut header = [0u8; 256];
            let bytes_read = file.read(&mut header)?;

            if is_likely_binary(&header[..bytes_read]) {
                // Binary protobuf input -> Wasot output
                #[cfg(feature = "binary")]
                {
                    let emitter = ProtobufBinaryEmitter::from_file(path);
                    let receiver = emitter.subscribe();

                    let emitter_handle = thread::spawn(move || {
                        emitter.run();
                    });

                    match output_path {
                        Some(out_path) => {
                            let file = File::create(out_path)?;
                            let writer = BufWriter::new(file);
                            let mut reconstructor = WasotReconstructor::new_with_writer(writer);
                            for event in receiver {
                                reconstructor.on_element_event(&event);
                            }
                        }
                        None => {
                            let mut reconstructor = WasotReconstructor::new();
                            for event in receiver {
                                reconstructor.on_element_event(&event);
                            }
                        }
                    }

                    emitter_handle
                        .join()
                        .map_err(|_| "Emitter thread panicked")?;
                }
                #[cfg(not(feature = "binary"))]
                {
                    return Err("Binary protobuf format requires the 'binary' feature".into());
                }
            } else {
                // Markdown input -> Wasot output
                let emitter =
                    MarkdownEmitter::from_file(path).with_paragraph_blocks(paragraph_blocks);
                let receiver = emitter.subscribe();

                let emitter_handle = thread::spawn(move || {
                    emitter.run();
                });

                if let Some(out_path) = output_path {
                    let file = File::create(out_path)?;
                    let mut wasot_consumer = WasotReconstructor::new_with_writer(file);
                    wasot_consumer.run(receiver);
                } else {
                    let mut wasot_consumer = WasotReconstructor::new();
                    wasot_consumer.run(receiver);
                }

                emitter_handle
                    .join()
                    .map_err(|_| "Emitter thread panicked")?;
            }
        }
        None => {
            // Read all stdin to detect format
            let mut buffer = Vec::new();
            io::stdin().read_to_end(&mut buffer)?;

            if is_likely_binary(&buffer) {
                // Binary protobuf from stdin
                #[cfg(feature = "binary")]
                {
                    let emitter = ProtobufBinaryEmitter::from_bytes(buffer);
                    let receiver = emitter.subscribe();

                    let emitter_handle = thread::spawn(move || {
                        emitter.run();
                    });

                    match output_path {
                        Some(out_path) => {
                            let file = File::create(out_path)?;
                            let writer = BufWriter::new(file);
                            let mut reconstructor = WasotReconstructor::new_with_writer(writer);
                            for event in receiver {
                                reconstructor.on_element_event(&event);
                            }
                        }
                        None => {
                            let mut reconstructor = WasotReconstructor::new();
                            for event in receiver {
                                reconstructor.on_element_event(&event);
                            }
                        }
                    }

                    emitter_handle
                        .join()
                        .map_err(|_| "Emitter thread panicked")?;
                }
                #[cfg(not(feature = "binary"))]
                {
                    return Err("Binary protobuf format requires the 'binary' feature".into());
                }
            } else {
                // Markdown from stdin
                let text = String::from_utf8(buffer).map_err(|e| e.to_string())?;
                let emitter =
                    MarkdownEmitter::from_string(text).with_paragraph_blocks(paragraph_blocks);
                let receiver = emitter.subscribe();

                let emitter_handle = thread::spawn(move || {
                    emitter.run();
                });

                if let Some(out_path) = output_path {
                    let file = File::create(out_path)?;
                    let mut wasot_consumer = WasotReconstructor::new_with_writer(file);
                    wasot_consumer.run(receiver);
                } else {
                    let mut wasot_consumer = WasotReconstructor::new();
                    wasot_consumer.run(receiver);
                }

                emitter_handle
                    .join()
                    .map_err(|_| "Emitter thread panicked")?;
            }
        }
    }

    Ok(())
}

async fn process_convert(
    from: Format,
    to: Format,
    input_path: Option<&str>,
    output_path: Option<&str>,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::fs::File;

    match (from, to) {
        // Wasot -> JSONL
        (Format::Wasot, Format::Jsonl) => {
            let temp_file = if output_path.is_none() {
                Some(tempfile::NamedTempFile::new()?)
            } else {
                None
            };

            let output_file_path = match output_path {
                Some(path) => path.to_string(),
                None => temp_file
                    .as_ref()
                    .unwrap()
                    .path()
                    .to_string_lossy()
                    .to_string(),
            };

            let emitter = match input_path {
                Some(path) => WasotFileEmitter::from_file(path),
                None => {
                    let mut buffer = String::new();
                    io::stdin().read_to_string(&mut buffer)?;
                    WasotFileEmitter::from_string(buffer)
                }
            };

            let mut consumer = ToJsonlConsumer::new(&output_file_path)?;

            let events = emitter.subscribe();
            let emitter_handle = std::thread::spawn(move || {
                emitter.run();
            });

            for event in events {
                consumer.process_event(&event);
            }

            emitter_handle.join().unwrap();
            consumer.finish()?;

            if output_path.is_none() {
                let mut file = File::open(&output_file_path)?;
                let mut stdout = io::stdout();
                io::copy(&mut file, &mut stdout)?;
            }
        }

        // JSONL -> Wasot
        (Format::Jsonl, Format::Wasot) => {
            let emitter = match input_path {
                Some(path) => JsonlEmitter::from_file(path)?,
                None => JsonlEmitter::from_stdin(),
            };

            let receiver = emitter.subscribe();
            let emitter_handle = std::thread::spawn(move || {
                emitter.run();
            });

            match output_path {
                Some(path) => {
                    let file = std::fs::File::create(path)?;
                    let writer = BufWriter::new(file);
                    let mut reconstructor = WasotReconstructor::new_with_writer(writer);
                    for event in receiver {
                        reconstructor.on_element_event(&event);
                    }
                }
                None => {
                    let mut reconstructor = WasotReconstructor::new();
                    for event in receiver {
                        reconstructor.on_element_event(&event);
                    }
                }
            }

            emitter_handle.join().unwrap();
        }

        // XML -> Wasot
        (Format::Xml, Format::Wasot) => {
            let emitter = match input_path {
                Some(path) => XmlEmitter::from_file(path)?,
                None => XmlEmitter::from_stdin(),
            };

            let receiver = emitter.subscribe();
            let emitter_handle = std::thread::spawn(move || {
                emitter.run();
            });

            match output_path {
                Some(path) => {
                    let file = std::fs::File::create(path)?;
                    let writer = BufWriter::new(file);
                    let mut reconstructor = WasotReconstructor::new_with_writer(writer);
                    for event in receiver {
                        reconstructor.on_element_event(&event);
                    }
                }
                None => {
                    let mut reconstructor = WasotReconstructor::new();
                    for event in receiver {
                        reconstructor.on_element_event(&event);
                    }
                }
            }

            emitter_handle.join().unwrap();
        }

        // Binary -> Wasot
        #[cfg(feature = "binary")]
        (Format::Binary, Format::Wasot) => {
            let emitter = match input_path {
                Some(path) => ProtobufBinaryEmitter::from_file(path),
                None => ProtobufBinaryEmitter::from_stdin(),
            };

            let receiver = emitter.subscribe();
            let emitter_handle = std::thread::spawn(move || {
                emitter.run();
            });

            match output_path {
                Some(path) => {
                    let file = std::fs::File::create(path)?;
                    let writer = BufWriter::new(file);
                    let mut reconstructor = WasotReconstructor::new_with_writer(writer);
                    for event in receiver {
                        reconstructor.on_element_event(&event);
                    }
                }
                None => {
                    let mut reconstructor = WasotReconstructor::new();
                    for event in receiver {
                        reconstructor.on_element_event(&event);
                    }
                }
            }

            emitter_handle.join().unwrap();
        }

        // Wasot -> Binary
        #[cfg(feature = "binary")]
        (Format::Wasot, Format::Binary) => {
            let emitter = match input_path {
                Some(path) => WasotFileEmitter::from_file(path),
                None => {
                    let mut buffer = String::new();
                    io::stdin().read_to_string(&mut buffer)?;
                    WasotFileEmitter::from_string(buffer)
                }
            };

            let receiver = emitter.subscribe();
            let emitter_handle = std::thread::spawn(move || {
                emitter.run();
            });

            match output_path {
                Some(path) => {
                    let file = std::fs::File::create(path)?;
                    let mut consumer = BinaryProtobufConsumer::new(file);
                    for event in receiver {
                        consumer.convert_and_write(&event)?;
                    }
                }
                None => {
                    let mut consumer = BinaryProtobufConsumer::new(std::io::stdout());
                    consumer.run(receiver);
                }
            }

            emitter_handle.join().unwrap();
        }

        // Wasot -> XML
        (Format::Wasot, Format::Xml) => {
            let emitter = match input_path {
                Some(path) => WasotFileEmitter::from_file(path),
                None => {
                    let mut buffer = String::new();
                    io::stdin().read_to_string(&mut buffer)?;
                    WasotFileEmitter::from_string(buffer)
                }
            };

            let receiver = emitter.subscribe();
            let emitter_handle = std::thread::spawn(move || {
                emitter.run();
            });

            let mut xml_listener = XmlListener::new(true);
            xml_listener.run(receiver);

            emitter_handle.join().unwrap();
        }

        // Wasot -> SQLite
        #[cfg(feature = "sqlite")]
        (Format::Wasot, Format::Sqlite) => match output_path {
            Some(path) => {
                std::fs::File::create(path)?;
                let database_url = format!("sqlite:{path}");

                let mut sqlite_consumer = SqliteConsumer::new(&database_url).await?;

                let emitter = match input_path {
                    Some(file_path) => WasotFileEmitter::from_file(file_path),
                    None => {
                        let mut buffer = String::new();
                        io::stdin().read_to_string(&mut buffer)?;
                        WasotFileEmitter::from_string(buffer)
                    }
                };

                let receiver = emitter.subscribe();
                let emitter_handle = std::thread::spawn(move || {
                    emitter.run();
                });

                sqlite_consumer.run(receiver);
                emitter_handle.join().unwrap();

                drop(sqlite_consumer);
                std::thread::sleep(std::time::Duration::from_millis(100));
            }
            None => {
                let temp_db = tempfile::NamedTempFile::new()?;
                let database_url = format!("sqlite:{}", temp_db.path().to_string_lossy());

                let mut sqlite_consumer = SqliteConsumer::new(&database_url).await?;

                let emitter = match input_path {
                    Some(file_path) => WasotFileEmitter::from_file(file_path),
                    None => {
                        let mut buffer = String::new();
                        io::stdin().read_to_string(&mut buffer)?;
                        WasotFileEmitter::from_string(buffer)
                    }
                };

                let receiver = emitter.subscribe();
                let emitter_handle = std::thread::spawn(move || {
                    emitter.run();
                });

                sqlite_consumer.run(receiver);
                emitter_handle.join().unwrap();

                drop(sqlite_consumer);
                std::thread::sleep(std::time::Duration::from_millis(100));

                let mut file = File::open(temp_db.path())?;
                let mut stdout = io::stdout();
                io::copy(&mut file, &mut stdout)?;
            }
        },

        // SQLite -> Wasot
        #[cfg(feature = "sqlite")]
        (Format::Sqlite, Format::Wasot) => {
            let database_url = match input_path {
                Some(path) => format!("sqlite:{path}"),
                None => {
                    let temp_db = tempfile::NamedTempFile::new()?;
                    let mut file = File::create(temp_db.path())?;
                    io::copy(&mut io::stdin(), &mut file)?;
                    format!("sqlite:{}", temp_db.path().to_string_lossy())
                }
            };

            let emitter = SqliteEmitter::from_database_url(&database_url).await?;
            let receiver = emitter.subscribe();

            let emitter_handle = std::thread::spawn(move || {
                emitter.run();
            });

            match output_path {
                Some(path) => {
                    let file = std::fs::File::create(path)?;
                    let writer = BufWriter::new(file);
                    let mut reconstructor = WasotReconstructor::new_with_writer(writer);
                    for event in receiver {
                        reconstructor.on_element_event(&event);
                    }
                }
                None => {
                    let mut reconstructor = WasotReconstructor::new();
                    for event in receiver {
                        reconstructor.on_element_event(&event);
                    }
                }
            }

            emitter_handle.join().unwrap();
        }

        // Other conversions not yet implemented
        _ => {
            return Err(
                format!("Conversion from {from:?} to {to:?} is not yet implemented").into(),
            );
        }
    }

    Ok(())
}

/// Extract document ID from a single Wasot file
/// Returns Some(id) if found, None if not found
fn extract_document_id(file_path: &std::path::Path) -> Option<String> {
    use std::thread;
    use wasot_compiler::events::Event;

    let emitter = WasotFileEmitter::from_file(file_path.to_str()?);
    let receiver = emitter.subscribe();

    let emitter_handle = thread::spawn(move || {
        emitter.run();
    });

    let mut document_id: Option<String> = None;
    for event in receiver {
        if let Event::ContextLine { meta_line } = event {
            if meta_line.type_name == "id" {
                document_id = Some(meta_line.inline_text.content.clone());
                break;
            }
        }
    }

    let _ = emitter_handle.join();
    document_id
}

/// Collect document IDs from files or directories (parallel processing)
/// Returns Vec<(document_id, relative_path)> sorted by path
/// The base_path is used to compute relative paths
fn collect_document_ids(
    paths: &[String],
    base_path: &std::path::Path,
) -> Result<Vec<(String, String)>, Box<dyn std::error::Error>> {
    use ignore::WalkBuilder;
    use std::path::Path;
    use std::sync::Mutex;
    use std::thread;

    // Collect all files from all paths, respecting .gitignore
    let mut all_files = Vec::new();
    for path_str in paths {
        let path = Path::new(path_str);

        if !path.exists() {
            return Err(format!("Path not found: {path_str}").into());
        }

        if path.is_file() {
            // Single file - add directly
            all_files.push(path.to_path_buf());
        } else {
            // Directory - walk with .gitignore support
            let walker = WalkBuilder::new(path)
                .hidden(false)
                .git_ignore(true)
                .git_global(true)
                .git_exclude(true)
                .build();

            for entry in walker.flatten() {
                let entry_path = entry.path();
                if entry_path.is_file() && entry_path.extension().is_some_and(|ext| ext == "wasot")
                {
                    all_files.push(entry_path.to_path_buf());
                }
            }
        }
    }

    // Process files in parallel, collect results
    let results: Mutex<Vec<(String, String)>> = Mutex::new(Vec::new());

    // Use thread::scope for parallel processing without 'static lifetime requirements
    thread::scope(|s| {
        // Limit concurrency to avoid overwhelming the system
        let num_threads = thread::available_parallelism()
            .map(|n| n.get())
            .unwrap_or(4)
            .min(all_files.len().max(1));

        let chunk_size = (all_files.len() + num_threads - 1) / num_threads.max(1);

        for chunk in all_files.chunks(chunk_size.max(1)) {
            let results_ref = &results;
            let base_ref = base_path;

            s.spawn(move || {
                let mut local_results = Vec::new();

                for file in chunk {
                    if let Some(id) = extract_document_id(file) {
                        let rel_path = file
                            .strip_prefix(base_ref)
                            .unwrap_or(file)
                            .to_string_lossy()
                            .to_string();
                        local_results.push((id, rel_path));
                    }
                }

                // Batch insert to reduce lock contention
                if !local_results.is_empty() {
                    results_ref.lock().unwrap().extend(local_results);
                }
            });
        }
    });

    // Sort by path (second element)
    let mut results = results.into_inner().unwrap();
    results.sort_by(|a, b| a.1.cmp(&b.1));

    Ok(results)
}

/// Scan and print document IDs to stdout
/// Output format: <document-id> <relative-path>
fn scan_document_ids(paths: &[String]) -> Result<(), Box<dyn std::error::Error>> {
    let cwd = std::env::current_dir()?;
    let paths = if paths.is_empty() {
        vec![".".to_string()]
    } else {
        paths.to_vec()
    };
    let results = collect_document_ids(&paths, &cwd)?;

    for (id, path) in results {
        println!("{id} {path}");
    }

    Ok(())
}

/// Update the .wasot-ids index file
/// Format: <document-id> <relative-path-from-index-file>
/// For duplicate IDs, last write wins (sorted by path, so deterministic)
fn update_wasot_ids(paths: &[String], output: &str) -> Result<(), Box<dyn std::error::Error>> {
    use std::collections::HashMap;
    use std::fs::File;
    use std::io::Write;
    use std::path::Path;

    // Determine the base path for relative paths (directory containing the index file)
    let output_path = Path::new(output);
    let base_path = output_path
        .parent()
        .map(|p| {
            if p.as_os_str().is_empty() {
                std::env::current_dir().unwrap_or_else(|_| Path::new(".").to_path_buf())
            } else {
                p.to_path_buf()
            }
        })
        .unwrap_or_else(|| {
            std::env::current_dir().unwrap_or_else(|_| Path::new(".").to_path_buf())
        });

    // Default to output file's directory if no paths given
    let default_paths = vec![base_path.to_string_lossy().to_string()];
    let paths = if paths.is_empty() {
        &default_paths
    } else {
        paths
    };

    let results = collect_document_ids(paths, &base_path)?;

    // Build a map with last write wins (since results are sorted by path, this is deterministic)
    let mut id_to_path: HashMap<String, String> = HashMap::new();
    for (id, path) in results {
        id_to_path.insert(id, path);
    }

    // Sort by ID for consistent output
    let mut entries: Vec<_> = id_to_path.into_iter().collect();
    entries.sort_by(|a, b| a.0.cmp(&b.0));

    // Write to file
    let mut file = File::create(output)?;
    for (id, path) in entries {
        writeln!(file, "{id} {path}")?;
    }

    Ok(())
}

/// Get the path for a document ID from the index file
/// Exits with code 1 if not found
fn get_document_id_path(id: &str, index: &str) -> Result<(), Box<dyn std::error::Error>> {
    use std::fs::File;
    use std::io::{BufRead, BufReader};
    use std::path::Path;

    let index_path = Path::new(index);
    if !index_path.exists() {
        return Err(format!("Index file not found: {index}").into());
    }

    let file = File::open(index_path)?;
    let reader = BufReader::new(file);

    for line in reader.lines() {
        let line = line?;
        let line = line.trim();
        if line.is_empty() {
            continue;
        }

        // Split on first space
        if let Some((line_id, path)) = line.split_once(' ') {
            if line_id == id {
                println!("{path}");
                return Ok(());
            }
        }
    }

    Err(format!("Document ID not found: {id}").into())
}

/// Handle errors uniformly: print to stderr and exit with appropriate code
fn handle_error(err: Box<dyn std::error::Error>) -> ! {
    let msg = err.to_string();
    eprintln!("wasot: {msg}");
    process::exit(exit_code_for_error(&msg))
}

#[tokio::main]
async fn main() {
    // Initialize dhat profiler for heap analysis
    #[cfg(feature = "dhat-heap")]
    let _profiler = dhat::Profiler::new_heap();

    // Reset SIGPIPE to default behavior so broken pipes don't cause errors
    // This allows `wasot file.wasot | head -1` to work cleanly
    #[cfg(unix)]
    unsafe {
        libc::signal(libc::SIGPIPE, libc::SIG_DFL);
    }

    let cli = Cli::parse();

    // Initialize the document ID index for resolving external references
    if cli.opts.no_id_index {
        wasot_compiler::parsers::wasot::document_id_index::init_empty_index();
    } else {
        wasot_compiler::parsers::wasot::document_id_index::init_global_index();
    }

    let result: Result<(), Box<dyn std::error::Error>> = match &cli.command {
        None => process_default(&cli.opts).await,
        #[cfg(feature = "sqlite")]
        Some(Commands::ToSqlite {
            file_path,
            database,
        }) => process_to_sqlite(file_path.as_deref(), database, cli.opts.no_auto_ids).await,
        #[cfg(feature = "sqlite")]
        Some(Commands::FromSqlite { database, output }) => {
            process_from_sqlite(database, output.as_deref())
        }
        Some(Commands::Tools { command }) => match command {
            ToolCommands::Check { files } => process_check(files),
            ToolCommands::Info { file } => process_info(file.as_deref()),
            ToolCommands::Pretty {
                paths,
                stdout,
                verbose,
                yes,
            } => process_pretty_batch(paths, *stdout, *verbose, *yes),
            ToolCommands::Completions { shell } => {
                generate(*shell, &mut Cli::command(), "wasot", &mut io::stdout());
                Ok(())
            }
            ToolCommands::DocumentIds(sub) => match sub {
                DocumentIdsCommands::Scan { paths } => scan_document_ids(paths),
                DocumentIdsCommands::Update { paths, output } => update_wasot_ids(paths, output),
                DocumentIdsCommands::Get { id, index } => get_document_id_path(id, index),
            },
        },
        Some(Commands::Lsp) => {
            wasot_compiler::lsp::start_lsp_server().await;
            Ok(())
        }
    };

    if let Err(err) = result {
        handle_error(err);
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_format_enum() {
        let wasot = Format::Wasot;
        let xml = Format::Xml;
        let binary = Format::Binary;
        let jsonl = Format::Jsonl;

        assert!(matches!(wasot, Format::Wasot));
        assert!(matches!(xml, Format::Xml));
        assert!(matches!(binary, Format::Binary));
        assert!(matches!(jsonl, Format::Jsonl));

        #[cfg(feature = "sqlite")]
        {
            let sqlite = Format::Sqlite;
            assert!(matches!(sqlite, Format::Sqlite));
        }
    }

    #[test]
    fn test_is_likely_binary() {
        // Text that looks like wasot
        assert!(!is_likely_binary(b".document"));
        assert!(!is_likely_binary(b"# Heading"));
        assert!(!is_likely_binary(b"Some text"));
        assert!(!is_likely_binary(b"(hello")); // Text starting with (

        // Binary data with magic header
        #[cfg(feature = "binary")]
        {
            use wasot_compiler::binary_format;
            let header = binary_format::build_header();
            assert!(is_likely_binary(&header));
        }

        // Binary data (invalid UTF-8)
        assert!(is_likely_binary(&[0xFF, 0xFE, 0x00, 0x00])); // Invalid UTF-8
    }

    #[test]
    fn test_detect_format_from_content() {
        // Wasot format detection
        assert_eq!(detect_format_from_content(b".document"), Format::Wasot);
        assert_eq!(detect_format_from_content(b"# Heading"), Format::Wasot);
        assert_eq!(detect_format_from_content(b"Some text"), Format::Wasot);
        assert_eq!(detect_format_from_content(b"(hello"), Format::Wasot); // Text starting with (

        // Whitespace skipping
        assert_eq!(detect_format_from_content(b"  .document"), Format::Wasot);
        assert_eq!(detect_format_from_content(b"\n\n.document"), Format::Wasot);
        assert_eq!(detect_format_from_content(b"\t\t# Heading"), Format::Wasot);
        assert_eq!(
            detect_format_from_content(b"  \n  \t  .block"),
            Format::Wasot
        );

        // XML detection (non-HTML XML)
        assert_eq!(detect_format_from_content(b"<root>"), Format::Xml);
        assert_eq!(detect_format_from_content(b"  <xml>"), Format::Xml);
        assert_eq!(detect_format_from_content(b"\n<?xml"), Format::Xml);
        assert_eq!(detect_format_from_content(b"<svg>"), Format::Xml);
        assert_eq!(detect_format_from_content(b"<rss>"), Format::Xml);

        // HTML detection
        assert_eq!(detect_format_from_content(b"<!DOCTYPE html>"), Format::Html);
        assert_eq!(detect_format_from_content(b"<!doctype HTML>"), Format::Html); // case insensitive
        assert_eq!(
            detect_format_from_content(b"<!DOCTYPE html PUBLIC"),
            Format::Html
        );
        assert_eq!(detect_format_from_content(b"<html>"), Format::Html);
        assert_eq!(detect_format_from_content(b"<HTML>"), Format::Html); // case insensitive
        assert_eq!(detect_format_from_content(b"  <html>"), Format::Html); // with whitespace
        assert_eq!(detect_format_from_content(b"\n<html>"), Format::Html);
        assert_eq!(
            detect_format_from_content(b"<html lang=\"en\">"),
            Format::Html
        );

        // JSONL detection
        assert_eq!(
            detect_format_from_content(b"{\"key\": \"value\"}"),
            Format::Jsonl
        );
        assert_eq!(detect_format_from_content(b"[1, 2, 3]"), Format::Jsonl);
        assert_eq!(detect_format_from_content(b"  {\"a\": 1}"), Format::Jsonl);

        // Binary detection via magic bytes
        #[cfg(feature = "binary")]
        {
            use wasot_compiler::binary_format;
            let header = binary_format::build_header();
            assert_eq!(detect_format_from_content(&header), Format::Binary);
        }

        // Binary detection via invalid UTF-8 (without grpc, this will be detected as binary due to invalid UTF-8)
        assert_eq!(
            detect_format_from_content(&[0xFF, 0xFE, 0x00, 0x01]),
            Format::Binary
        );

        // Empty = wasot
        assert_eq!(detect_format_from_content(b""), Format::Wasot);
        assert_eq!(detect_format_from_content(b"   "), Format::Wasot);
    }

    #[test]
    fn test_detect_format_from_extension() {
        assert_eq!(
            detect_format_from_extension("file.wasot"),
            Some(Format::Wasot)
        );
        assert_eq!(detect_format_from_extension("file.xml"), Some(Format::Xml));
        assert_eq!(
            detect_format_from_extension("file.html"),
            Some(Format::Html)
        );
        assert_eq!(detect_format_from_extension("file.htm"), Some(Format::Html));
        assert_eq!(
            detect_format_from_extension("file.jsonl"),
            Some(Format::Jsonl)
        );
        assert_eq!(
            detect_format_from_extension("file.wasotb"),
            Some(Format::Binary)
        );
        assert_eq!(detect_format_from_extension("file.txt"), None);
        assert_eq!(detect_format_from_extension("file"), None);
        assert_eq!(
            detect_format_from_extension("/path/to/file.wasot"),
            Some(Format::Wasot)
        );
        assert_eq!(
            detect_format_from_extension("/path/to/page.html"),
            Some(Format::Html)
        );
        // Markdown extension detection
        assert_eq!(
            detect_format_from_extension("file.md"),
            Some(Format::Markdown)
        );
        assert_eq!(
            detect_format_from_extension("file.markdown"),
            Some(Format::Markdown)
        );
        assert_eq!(
            detect_format_from_extension("/path/to/readme.md"),
            Some(Format::Markdown)
        );
    }
}
