from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
UTC = timezone.utc
from pathlib import Path
from threading import Thread, Lock

import duckdb
import logging
import math
import os
import time
import json
import re
import numpy as np
try:
    import psutil
except Exception:  # pragma: no cover - psutil is expected but we stay defensive
    psutil = None

logger = logging.getLogger(__name__)
_SAVGOL_SKIP_LOGS = 0
_SAVGOL_APPLY_LOGS = 0
_SAVGOL_SKIP_INFO_LOGS = 0
_SAVGOL_APPLY_INFO_LOGS = 0
_SAVGOL_LOG_LIMIT = 25
_SCAN_LOOKUP_LOCK = Lock()
_DB_LOCK_COUNTS: dict[str, int] = {}
_DB_LOCK_COUNTS_LOCK = Lock()

try:
    import lttbc as _lttbc
except Exception:
    _lttbc = None

try:
    from scipy.signal import savgol_filter as _savgol_filter
except Exception:
    _savgol_filter = None
from .sample_metadata import GROUP_COLUMNS

FULL_RANGE_DOWNSAMPLE_POINTS = 1000
FULL_RANGE_DOWNSAMPLE_BATCH = 200
FULL_RANGE_SAVGOL_WINDOW = 10
FULL_RANGE_SAVGOL_ORDER = 2


def _apply_lttb_downsampling(scan_time, intensity, n_out=FULL_RANGE_DOWNSAMPLE_POINTS, min_points=None):
    if n_out is None:
        n_out = FULL_RANGE_DOWNSAMPLE_POINTS

    try:
        n_out = int(n_out)
    except (TypeError, ValueError):
        return scan_time, intensity

    if _lttbc is None:
        return scan_time, intensity

    if min_points is None:
        min_points = max(n_out * 2, 10)
    try:
        min_points = int(min_points)
    except (TypeError, ValueError):
        min_points = 0

    if not scan_time or n_out <= 0 or len(scan_time) <= n_out:
        return scan_time, intensity
    if min_points > 0 and len(scan_time) < min_points:
        return scan_time, intensity

    downsampled_x, downsampled_y = _lttbc.downsample(scan_time, intensity, n_out)
    if len(downsampled_x) == 0:
        return scan_time, intensity

    return downsampled_x, downsampled_y


def _savgol_coeffs_numpy(window_length, polyorder, deriv=0, delta=1.0):
    half_window = window_length // 2
    pos = np.arange(-half_window, half_window + 1, dtype=float)
    a = np.vander(pos, polyorder + 1, increasing=True)
    b = np.zeros(polyorder + 1, dtype=float)
    b[deriv] = math.factorial(deriv) / (delta ** deriv)
    return np.linalg.lstsq(a.T, b, rcond=None)[0]


def _savgol_filter_numpy(intensity, window_length, polyorder):
    intensity = np.asarray(intensity, dtype=float)
    n_points = intensity.size
    if n_points == 0:
        return intensity

    half_window = window_length // 2
    if n_points < window_length:
        return intensity

    coeffs = _savgol_coeffs_numpy(window_length, polyorder)
    filtered = np.empty_like(intensity, dtype=float)

    interior = np.convolve(intensity, coeffs[::-1], mode='valid')
    filtered[half_window:-half_window] = interior

    x = np.arange(window_length, dtype=float)
    left_poly = np.polyfit(x, intensity[:window_length], polyorder)
    filtered[:half_window] = np.polyval(left_poly, x[:half_window])
    right_poly = np.polyfit(x, intensity[-window_length:], polyorder)
    filtered[-half_window:] = np.polyval(right_poly, x[-half_window:])

    return filtered


def _apply_savgol_smoothing(intensity,
                            window_length=FULL_RANGE_SAVGOL_WINDOW,
                            polyorder=FULL_RANGE_SAVGOL_ORDER,
                            min_points=None):
    # Smoothing disabled per user request (2026-01-26)
    return intensity

    # The following logic is disabled:
    """
    if window_length is None:
        window_length = FULL_RANGE_SAVGOL_WINDOW
    if polyorder is None:
        polyorder = FULL_RANGE_SAVGOL_ORDER

    try:
        window_length = int(window_length)
        polyorder = int(polyorder)
    except (TypeError, ValueError):
        return intensity

    intensity = np.asarray(intensity, dtype=float)
    n_points = intensity.size

    # CHECK 1: No data to smooth
    if n_points == 0:
        return intensity

    if min_points is None:
        min_points = max(window_length * 2 + 1, 7)
    try:
        min_points = int(min_points)
    except (TypeError, ValueError):
        min_points = 0

    # CHECK 1.5: Require a minimum number of points before smoothing
    if min_points > 0 and n_points < min_points:
        global _SAVGOL_SKIP_LOGS, _SAVGOL_SKIP_INFO_LOGS
        if _SAVGOL_SKIP_INFO_LOGS < 5:
            logger.info(
                "Full-range savgol skipped: n_points=%d < min_points=%d (window=%d, order=%d)",
                n_points,
                min_points,
                window_length,
                polyorder,
            )
            _SAVGOL_SKIP_INFO_LOGS += 1
        if _SAVGOL_SKIP_LOGS < _SAVGOL_LOG_LIMIT:
            logger.debug(
                "Full-range savgol skipped: n_points=%d < min_points=%d (window=%d, order=%d)",
                n_points,
                min_points,
                window_length,
                polyorder,
            )
            _SAVGOL_SKIP_LOGS += 1
        return intensity

    # CHECK 2: Window too large for data size (MAVEN Rule: > n/3)
    if window_length > n_points // 3:
        window_length = int(n_points // 3)

    # CHECK 3: Window too small to be useful
    if window_length <= 1:
        return intensity

    # Ensure valid window for SavGol (must be odd)
    if window_length % 2 == 0:
        window_length -= 1

    # Re-check after odd adjustment
    if window_length <= 1:
        return intensity

    if polyorder < 0:
        polyorder = 0
    # Strict parameter bounds: Polynomial order cannot exceed window size
    if polyorder >= window_length:
        polyorder = window_length - 1

    if _savgol_filter is not None:
        try:
            smoothed = _savgol_filter(intensity, window_length, polyorder, mode='interp')
        except Exception:
             # Fallback if scipy fails
             smoothed = intensity
    else:
        smoothed = _savgol_filter_numpy(intensity, window_length, polyorder)

    global _SAVGOL_APPLY_LOGS, _SAVGOL_APPLY_INFO_LOGS
    if _SAVGOL_APPLY_INFO_LOGS < 5:
        logger.info(
            "Full-range savgol applied: n_points=%d (window=%d, order=%d)",
            n_points,
            window_length,
            polyorder,
        )
        _SAVGOL_APPLY_INFO_LOGS += 1
    if _SAVGOL_APPLY_LOGS < _SAVGOL_LOG_LIMIT:
        logger.debug(
            "Full-range savgol applied: n_points=%d (window=%d, order=%d)",
            n_points,
            window_length,
            polyorder,
        )
        _SAVGOL_APPLY_LOGS += 1

    # Negative value clipping (0.0)
    return np.maximum(smoothed, 0.0)
    """


def get_physical_cores() -> int:
    """
    Get the number of physical CPU cores (not hyperthreads).
    Falls back to os.cpu_count() // 2 if psutil can't detect.
    """
    import os
    import psutil
    
    physical = psutil.cpu_count(logical=False)
    if physical is None:
        # Fallback: assume hyperthreading, so divide logical by 2
        physical = max(1, (os.cpu_count() or 4) // 2)
    return physical


def calculate_optimal_params(user_cpus: int = None, user_ram: int = None) -> tuple:
    """
    Calculate optimal CPU, RAM, and batch_size based on system resources.
    
    Algorithm (data-driven from experimental benchmarks):
    1. CPUs: min(logical // 2, physical_cores) - avoids hyperthreads
    2. RAM: 50% of available, balanced with 1.5GB per CPU minimum
    3. Batch: 300 × RAM_GB, capped at 5000
    
    If user provides explicit values, those are used instead of auto-detection.
    
    Args:
        user_cpus: Optional user-specified CPU count
        user_ram: Optional user-specified RAM in GB
        
    Returns:
        (cpus, ram_gb, batch_size) tuple
    """
    import os
    import psutil
    
    # Step 1: CPU calculation
    if user_cpus is not None:
        target_cpus = user_cpus
    else:
        logical_cores = os.cpu_count() or 4
        physical_cores = get_physical_cores()
        # Use half of logical, but never exceed physical (no hyperthreading benefit)
        target_cpus = min(logical_cores // 2, physical_cores)
        target_cpus = max(1, target_cpus)  # At least 1 CPU
    
    # Step 2: RAM calculation
    if user_ram is not None:
        usable_ram = user_ram
    else:
        available_ram = psutil.virtual_memory().available / (1024 ** 3)
        usable_ram = int(available_ram * 0.5)  # 50% of available
        usable_ram = max(4, usable_ram)  # Minimum 4GB
    
    # Step 3: Balance CPUs and RAM (1.5GB per CPU minimum)
    min_ram_per_cpu = 1.5
    if usable_ram < target_cpus * min_ram_per_cpu:
        # RAM is limiting factor - reduce CPUs to match
        cpus = max(1, int(usable_ram // min_ram_per_cpu))
        ram_gb = usable_ram
    else:
        # CPU is limiting factor
        cpus = target_cpus
        
        if user_ram is not None:
            # If user explicitly set RAM, respect it fully
            ram_gb = usable_ram
        else:
            # Cap RAM at 1.5× CPUs (no benefit beyond that based on experiments)
            ram_gb = min(usable_ram, int(cpus * min_ram_per_cpu))
    
    # Step 4: Batch size optimization
    # Benchmarks (Jan 2026) showed 1000-3000 is the "sweet spot" for throughput
    # and stability. Larger batches (5000+) reduced speed by 3x and increased crash risk.
    # New formula: 300 * RAM_GB, capped at 5000.
    batch_size = min(300 * ram_gb, 5000)
    batch_size = max(1000, batch_size)  # Minimum 1000 for efficiency
    
    return cpus, ram_gb, batch_size


def calculate_optimal_batch_size(ram_gb: int = None, total_pairs: int = 0, n_cpus: int = None, ms_type: str = None) -> int:
    """
    Calculate optimal batch size for chromatogram/results extraction.
    
    This is a simplified wrapper around calculate_optimal_params() for
    backward compatibility with existing code that only needs batch_size.
    
    Formula (revised Jan 2026 based on large dataset benchmarks):
    - batch = 300 × RAM_GB, capped at 5000
    - Experiments showed batch 1000 was 3x faster than batch 5000
    - Large batches (8000+) caused high memory pressure and crashes
    
    Args:
        ram_gb: RAM in GB. If None, auto-calculates 50% of available.
        total_pairs: Total pairs (used for progress reporting constraint)
        n_cpus: Number of CPUs (used for balancing with RAM)
        
    Returns:
        Optimal batch size
    """
    if ms_type is not None:
        ms_type = str(ms_type).strip().lower()
    logger.debug(f"DEBUG_BATCH: ram_gb={ram_gb}, n_cpus={n_cpus}, ms_type={ms_type}, total_pairs={total_pairs}")
    
    _, effective_ram, batch_size = calculate_optimal_params(
        user_cpus=n_cpus,
        user_ram=ram_gb
    )
    
    # Ensure at least 10 batches for progress reporting
    if total_pairs > 0:
        batch_size = min(batch_size, max(total_pairs // 10, 500))

    # For non-MS1 data (MS2, All, or unspecified), we cap the batch size at 1000 to avoid OOM issues.
    # MS1 data can safely use larger batches (up to 5000) for better throughput.
    if ms_type != 'ms1':
        batch_size = min(batch_size, 1000)
    
    logger.debug(f"DEBUG_BATCH: Result batch_size={batch_size}")
    return batch_size


def get_effective_cpus(n_cpus: int, ram_gb: int) -> int:
    """
    Calculate effective CPUs, capped at RAM (1.5GB per CPU minimum).
    
    Args:
        n_cpus: Requested number of CPUs
        ram_gb: Available RAM in GB
        
    Returns:
        Effective number of CPUs to use
    """
    if not n_cpus or not ram_gb:
        return n_cpus or 4  # Default to 4 if not specified
    return max(1, min(n_cpus, int(ram_gb / 1.5)))


# Required tables and their core columns for validation
REQUIRED_TABLES = {
    'samples': ['ms_file_label'],
    'targets': ['peak_label', 'mz_mean', 'rt', 'rt_min', 'rt_max'],
    'ms1_data': ['ms_file_label', 'scan_id', 'mz', 'intensity', 'scan_time'],
    'chromatograms': ['peak_label', 'ms_file_label'],
    'results': ['peak_label', 'ms_file_label', 'peak_area'],
}


def validate_mint_database(db_path: str) -> tuple[bool, str, dict]:
    """
    Validate that a DuckDB file is a valid MINT database.
    
    Args:
        db_path: Path to the database file
        
    Returns:
        (is_valid, error_message, stats_dict)
        stats_dict contains row counts for each table when valid
        
    Checks:
        - File exists and is readable
        - Contains required tables
        - Tables have expected core columns
    """
    import shutil
    
    db_path = Path(db_path)
    stats = {}
    
    # Check file exists
    if not db_path.exists():
        return False, f"File not found: {db_path}", stats
    
    if not db_path.is_file():
        return False, f"Not a file: {db_path}", stats
    
    # Check file extension
    if db_path.suffix.lower() not in ['.db', '.duckdb']:
        return False, f"Invalid file extension: {db_path.suffix}. Expected .db or .duckdb", stats
    
    # Try to open the database
    con = None
    try:
        con = duckdb.connect(database=str(db_path), read_only=True)
        
        # Get list of tables
        tables_result = con.execute("SHOW TABLES").fetchall()
        existing_tables = {row[0] for row in tables_result}
        
        # Check required tables
        missing_tables = []
        for table in REQUIRED_TABLES:
            if table not in existing_tables:
                missing_tables.append(table)
        
        if missing_tables:
            return False, f"Missing required tables: {', '.join(missing_tables)}", stats
        
        # Check columns for each required table
        for table, required_cols in REQUIRED_TABLES.items():
            cols_result = con.execute(f"DESCRIBE {table}").fetchall()
            existing_cols = {row[0] for row in cols_result}
            
            missing_cols = [col for col in required_cols if col not in existing_cols]
            if missing_cols:
                return False, f"Table '{table}' missing columns: {', '.join(missing_cols)}", stats
            
            # Get row count
            count = con.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
            stats[table] = count
        
        # Check optional ms2_data table
        if 'ms2_data' in existing_tables:
            count = con.execute("SELECT COUNT(*) FROM ms2_data").fetchone()[0]
            stats['ms2_data'] = count
        
        return True, "", stats
        
    except duckdb.IOException as e:
        return False, f"Cannot read database file: {e}", stats
    except duckdb.InvalidInputException as e:
        return False, f"Invalid database file: {e}", stats
    except Exception as e:
        return False, f"Database validation error: {e}", stats
    finally:
        if con:
            con.close()


def import_database_as_workspace(
    db_path: str, 
    workspace_name: str, 
    mint_root: Path
) -> tuple[bool, str, str]:
    """
    Import a DuckDB file as a new workspace.
    
    Args:
        db_path: Path to the source database file
        workspace_name: Name for the new workspace
        mint_root: Root directory for MINT data (e.g., /path/to/MINT/Local)
        
    Returns:
        (success, error_message, workspace_key)
        
    Steps:
        1. Validate database
        2. Create workspace record in mint.db
        3. Create workspace folder
        4. Copy database to workspace folder
    """
    import shutil
    import uuid
    
    db_path = Path(db_path)
    mint_root = Path(mint_root)
    
    # Step 1: Validate the source database
    is_valid, error_msg, stats = validate_mint_database(str(db_path))
    if not is_valid:
        return False, error_msg, ""
    
    # Step 2: Create workspace record
    workspace_key = None
    try:
        with duckdb_connection_mint(mint_root) as mint_conn:
            if mint_conn is None:
                return False, "Cannot connect to MINT database", ""
            
            # Check if name already exists
            existing = mint_conn.execute(
                "SELECT COUNT(*) FROM workspaces WHERE name = ?", 
                (workspace_name,)
            ).fetchone()[0]
            
            if existing > 0:
                return False, f"Workspace name '{workspace_name}' already exists", ""
            
            # Deactivate current active workspace
            mint_conn.execute("UPDATE workspaces SET active = false WHERE active = true")
            
            # Insert new workspace
            result = mint_conn.execute(
                """INSERT INTO workspaces (name, description, active, created_at, last_activity) 
                   VALUES (?, ?, true, NOW(), NOW()) RETURNING key""",
                (workspace_name, f"Imported from {db_path.name}")
            ).fetchone()
            
            if result:
                workspace_key = str(result[0])
            else:
                return False, "Failed to create workspace record", ""
                
    except Exception as e:
        logger.error(f"Error creating workspace record: {e}", exc_info=True)
        return False, f"Failed to create workspace: {e}", ""
    
    # Step 3: Create workspace folder
    workspace_path = mint_root / 'workspaces' / workspace_key
    try:
        workspace_path.mkdir(parents=True, exist_ok=True)
        ensure_workspace_name_marker(workspace_path, workspace_name, workspace_key=workspace_key)
    except Exception as e:
        # Rollback: delete the workspace record
        try:
            with duckdb_connection_mint(mint_root) as mint_conn:
                if mint_conn:
                    mint_conn.execute("DELETE FROM workspaces WHERE key = ?", (workspace_key,))
        except Exception:
            pass
        return False, f"Failed to create workspace folder: {e}", ""
    
    # Step 4: Copy database file
    dest_db_path = workspace_path / 'workspace_mint.db'
    try:
        shutil.copy2(str(db_path), str(dest_db_path))
        logger.info(f"Imported database from {db_path} to workspace {workspace_name} (key: {workspace_key})")
    except Exception as e:
        # Rollback: delete folder and workspace record
        try:
            shutil.rmtree(workspace_path, ignore_errors=True)
            with duckdb_connection_mint(mint_root) as mint_conn:
                if mint_conn:
                    mint_conn.execute("DELETE FROM workspaces WHERE key = ?", (workspace_key,))
        except Exception:
            pass
        return False, f"Failed to copy database file: {e}", ""
    
    return True, "", workspace_key


def ensure_exploration_workspace(mint_root: Path, *, seed: int = 7) -> bool:
    """
    Create an exploration workspace on first run.
    Prefer seeding from a bundled sample snapshot; otherwise seed synthetic data.
    Returns True if a workspace was created, False otherwise.
    """
    if os.environ.get("MINT_DISABLE_EXPLORATION_WORKSPACE"):
        return False

    mint_root = Path(mint_root)
    try:
        with duckdb_connection_mint(mint_root) as mint_conn:
            if mint_conn is None:
                return False
            existing = mint_conn.execute("SELECT COUNT(*) FROM workspaces").fetchone()
            if existing and existing[0] > 0:
                return False
    except Exception as e:
        logger.error(f"Failed to check existing workspaces: {e}", exc_info=True)
        return False

    bundle_path = _resolve_exploration_bundle_path()
    manifest = _load_exploration_manifest(bundle_path) if bundle_path else None
    ws_name = (manifest or {}).get("workspace_name") or "Explore"
    description = _exploration_workspace_description(manifest)

    workspace_key = _create_exploration_workspace_record(
        mint_root,
        name=ws_name,
        description=description,
    )
    if not workspace_key:
        return False

    workspace_path = mint_root / "workspaces" / workspace_key
    try:
        workspace_path.mkdir(parents=True, exist_ok=True)
        ensure_workspace_name_marker(workspace_path, ws_name, workspace_key=workspace_key)
        if bundle_path and _seed_exploration_workspace_from_bundle(workspace_path, bundle_path):
            logger.info(f"Seeded exploration workspace from bundle: {bundle_path}")
        else:
            logger.warning("Falling back to synthetic exploration data.")
            _seed_exploration_workspace_db(workspace_path, seed=seed)
    except Exception as e:
        logger.error(f"Failed to seed exploration workspace: {e}", exc_info=True)
        return False

    logger.info(f"Created exploration workspace at {workspace_path}")
    return True


def _create_exploration_workspace_record(
    mint_root: Path,
    *,
    name: str,
    description: str,
) -> str | None:
    try:
        with duckdb_connection_mint(mint_root) as mint_conn:
            if mint_conn is None:
                return None
            result = mint_conn.execute(
                """INSERT INTO workspaces (name, description, active, created_at, last_activity)
                   VALUES (?, ?, true, NOW(), NOW()) RETURNING key""",
                (name, description)
            ).fetchone()
            return str(result[0]) if result else None
    except Exception as e:
        logger.error(f"Failed to create exploration workspace record: {e}", exc_info=True)
        return None


def _workspace_marker_filename(workspace_name: str) -> str:
    safe_name = re.sub(r"[\\/]", "_", workspace_name).strip()
    if not safe_name:
        safe_name = "workspace"
    return f"{safe_name}.txt"


def ensure_workspace_name_marker(
    workspace_path: Path | str,
    workspace_name: str | None,
    workspace_key: str | None = None,
    created_at: datetime | None = None,
) -> None:
    if not workspace_path or not workspace_name:
        return

    try:
        marker_path = Path(workspace_path) / _workspace_marker_filename(workspace_name)
        if marker_path.exists():
            return

        created_at = created_at or datetime.now(UTC)
        lines = [
            "MINT Workspace",
            f"Name: {workspace_name}",
        ]
        if workspace_key:
            lines.append(f"Key: {workspace_key}")
        lines.append(f"Created (UTC): {created_at.isoformat(timespec='seconds')}")
        marker_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
    except Exception as exc:
        logger.debug("Failed to write workspace marker for %s: %s", workspace_name, exc)


def _resolve_exploration_bundle_path() -> Path | None:
    env_path = os.environ.get("MINT_EXPLORATION_BUNDLE_PATH")
    if env_path:
        candidate = Path(env_path).expanduser()
        if candidate.exists():
            return candidate
        logger.warning(f"MINT_EXPLORATION_BUNDLE_PATH does not exist: {candidate}")

    bundle_path = Path(__file__).resolve().parent / "assets" / "explore_bundle"
    if bundle_path.exists():
        return bundle_path

    return None


def _load_exploration_manifest(bundle_path: Path | None) -> dict | None:
    if not bundle_path:
        return None
    manifest_path = bundle_path / "manifest.json"
    if not manifest_path.exists():
        return None
    try:
        import json
        with open(manifest_path, "r", encoding="utf-8") as fh:
            return json.load(fh)
    except Exception as e:
        logger.warning(f"Failed to read exploration manifest: {e}")
        return None


def _exploration_workspace_description(manifest: dict | None) -> str:
    if not manifest:
        return "Auto-generated exploration workspace (synthetic demo data)."
    source = manifest.get("source_label") or "sampled real data"
    return f"Auto-generated exploration workspace ({source})."


def _seed_exploration_workspace_from_bundle(
    workspace_path: Path,
    bundle_path: Path,
) -> bool:
    manifest = _load_exploration_manifest(bundle_path) or {}
    tables = manifest.get("tables") or {}

    with duckdb_connection(workspace_path, register_activity=False) as conn:
        if conn is None:
            return False

        for table in ("samples", "targets", "chromatograms", "results"):
            parquet_file = bundle_path / f"{table}.parquet"
            if not parquet_file.exists():
                continue

            dest_cols = [row[0] for row in conn.execute(f"DESCRIBE {table}").fetchall()]
            src_cols = tables.get(table, {}).get("columns") or []
            columns = [col for col in dest_cols if col in src_cols]
            if not columns:
                continue

            quoted_cols = ", ".join(f'"{col}"' for col in columns)
            parquet_path = str(parquet_file).replace("'", "''")
            conn.execute(
                f"INSERT INTO {table} ({quoted_cols}) "
                f"SELECT {quoted_cols} FROM read_parquet('{parquet_path}')"
            )

    return True


def _seed_exploration_workspace_db(workspace_path: Path, *, seed: int = 7) -> None:
    """Populate a new workspace with lightweight synthetic data for exploration."""
    with duckdb_connection(workspace_path, register_activity=False) as conn:
        if conn is None:
            raise RuntimeError("Could not open workspace database for seeding.")

        existing = conn.execute("SELECT COUNT(*) FROM samples").fetchone()
        if existing and existing[0] > 0:
            return

        rng = np.random.default_rng(seed)
        base_time = datetime(2024, 1, 1, 9, 0, 0)

        sample_specs = [
            ("EC_01", "E. coli 1", "EC", "#4C78A8", "Batch 1", "Plate A", True),
            ("EC_02", "E. coli 2", "EC", "#4C78A8", "Batch 2", "Plate A", True),
            ("CA_01", "C. albicans 1", "CA", "#F58518", "Batch 1", "Plate B", True),
            ("CA_02", "C. albicans 2", "CA", "#F58518", "Batch 2", "Plate B", True),
            ("SA_01", "S. aureus 1", "SA", "#54A24B", "Batch 1", "Plate C", True),
            ("SA_02", "S. aureus 2", "SA", "#54A24B", "Batch 2", "Plate C", True),
            ("QC_01", "QC Mix", "QC", "#B279A2", "Batch 1", "Plate A", True),
            ("BLK_01", "Blank", "Blank", "#9E9E9E", "Batch 1", "Plate A", False),
        ]

        sample_rows = []
        for idx, (ms_file_label, label, sample_type, color, group_1, group_2, use_opt) in enumerate(sample_specs):
            sample_rows.append(
                (
                    ms_file_label,
                    "ms1",
                    "mzML",
                    use_opt,
                    True,
                    True,
                    "Positive",
                    color,
                    label,
                    sample_type,
                    group_1,
                    group_2,
                    base_time + timedelta(minutes=12 * idx),
                )
            )

        conn.executemany(
            """
            INSERT INTO samples (
                ms_file_label, ms_type, file_type, use_for_optimization, use_for_processing,
                use_for_analysis, polarity, color, label, sample_type, group_1, group_2,
                acquisition_datetime
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """,
            sample_rows,
        )

        targets = [
            ("GLUCOSE", 179.056, 0.012, 1.25, "Carbohydrate"),
            ("LACTATE", 89.024, 0.010, 2.10, "Organic Acid"),
            ("SUCCINATE", 117.019, 0.012, 3.05, "Organic Acid"),
            ("CITRATE", 191.019, 0.015, 4.35, "Organic Acid"),
            ("ALANINE", 90.055, 0.010, 5.20, "Amino Acid"),
            ("GLUTAMATE", 148.060, 0.012, 6.10, "Amino Acid"),
        ]

        target_rows = []
        for idx, (peak_label, mz, mz_width, rt, category) in enumerate(targets):
            target_rows.append(
                (
                    peak_label,
                    mz,
                    mz_width,
                    mz,
                    rt,
                    rt - 0.20,
                    rt + 0.20,
                    "min",
                    500.0,
                    "Positive",
                    None,
                    "ms1",
                    category,
                    True if idx < 4 else False,
                    round(0.85 + 0.03 * idx, 3),
                    False,
                    "demo",
                    "Synthetic demo target",
                )
            )

        conn.executemany(
            """
            INSERT INTO targets (
                peak_label, mz_mean, mz_width, mz, rt, rt_min, rt_max, rt_unit,
                intensity_threshold, polarity, filterLine, ms_type, category,
                peak_selection, score, bookmark, source, notes
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """,
            target_rows,
        )

        type_factors = {
            "EC": 1.2,
            "CA": 0.9,
            "SA": 1.1,
            "QC": 1.0,
            "Blank": 0.12,
        }

        time_axis = np.linspace(0.5, 8.0, 120)
        scan_time = np.round(time_axis, 4).tolist()

        chrom_rows = []
        result_rows = []
        for ms_file_label, _label, sample_type, _color, _g1, _g2, _use_opt in sample_specs:
            type_factor = type_factors.get(sample_type, 1.0)
            for idx, (peak_label, mz, _mz_width, rt, _category) in enumerate(targets):
                sigma = 0.08 + (idx * 0.015)
                base_amp = 6000 + (idx * 900)
                amp = base_amp * type_factor * float(rng.uniform(0.85, 1.15))
                signal = amp * np.exp(-0.5 * ((time_axis - rt) / sigma) ** 2)
                noise = rng.normal(0.0, amp * 0.02, size=time_axis.size)
                intensity = np.maximum(signal + noise + 40.0, 0.0)

                intensity_list = np.round(intensity, 3).tolist()
                mz_arr = [float(mz)] * len(scan_time)

                chrom_rows.append(
                    (
                        peak_label,
                        ms_file_label,
                        scan_time,
                        intensity_list,
                        None,
                        None,
                        mz_arr,
                        "ms1",
                    )
                )

                peak_max = float(np.max(intensity))
                peak_min = float(np.min(intensity))
                peak_mean = float(np.mean(intensity))
                peak_median = float(np.median(intensity))
                peak_rt_of_max = float(scan_time[int(np.argmax(intensity))])
                peak_area = float(np.trapz(intensity, time_axis))
                total_intensity = float(np.sum(intensity))
                peak_area_top3 = float(np.sort(intensity)[-3:].sum())

                result_rows.append(
                    (
                        peak_label,
                        ms_file_label,
                        total_intensity,
                        peak_area,
                        peak_area_top3,
                        peak_max,
                        peak_min,
                        peak_mean,
                        peak_rt_of_max,
                        peak_median,
                        len(scan_time),
                        False,
                        0.0,
                        scan_time,
                        intensity_list,
                    )
                )

        conn.executemany(
            """
            INSERT INTO chromatograms (
                peak_label, ms_file_label, scan_time, intensity,
                scan_time_full_ds, intensity_full_ds, mz_arr, ms_type
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
            """,
            chrom_rows,
        )

        conn.executemany(
            """
            INSERT INTO results (
                peak_label, ms_file_label, total_intensity, peak_area,
                peak_area_top3, peak_max, peak_min, peak_mean,
                peak_rt_of_max, peak_median, peak_n_datapoints,
                rt_aligned, rt_shift, scan_time, intensity
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """,
            result_rows,
        )

def _send_progress(set_progress, percent, stage: str = "", detail: str = ""):
    """
    Safely call the provided set_progress callback.

    Supports custom stage/detail strings when the callback accepts them,
    and falls back to simple percent-only updates otherwise.
    
    IMPORTANT: Re-raises Cancelled/PreventUpdate exceptions for proper cancellation.
    """
    if not set_progress:
        return
    try:
        set_progress(percent, stage, detail)
    except TypeError:
        try:
            set_progress(percent)
        except (SystemExit, KeyboardInterrupt):
            raise  # Always re-raise these
        except Exception as e:
            # Re-raise cancel-related exceptions
            if 'Cancelled' in type(e).__name__ or 'PreventUpdate' in type(e).__name__:
                raise
            pass  # Suppress other exceptions
    except (SystemExit, KeyboardInterrupt):
        raise  # Always re-raise these
    except Exception as e:
        # Re-raise cancel-related exceptions (dash.exceptions.Cancelled)
        if 'Cancelled' in type(e).__name__ or 'PreventUpdate' in type(e).__name__:
            raise
        pass  # Suppress other exceptions




def _update_workspace_activity(mint_root: Path, workspace_id: str, retries: int = 3, delay_s: float = 0.05):
    # Throttle updates to avoid hot-loop write contention on mint.db.
    now = time.time()
    with _LAST_ACTIVITY_UPDATE_LOCK:
        last_ts = _LAST_ACTIVITY_UPDATE_TS.get(workspace_id, 0.0)
        if now - last_ts < 15.0:
            return
        _LAST_ACTIVITY_UPDATE_TS[workspace_id] = now

    for attempt in range(retries):
        try:
            with duckdb_connection_mint(mint_root) as mint_conn:
                if mint_conn:
                    mint_conn.execute(
                        "UPDATE workspaces SET last_activity = NOW() WHERE key = ?",
                        [workspace_id],
                    )
            return
        except Exception as e:
            message = str(e)
            if (
                "TransactionContext Error: Conflict on update!" in message
                or "used by another process" in message
                or "different configuration than existing connections" in message
            ):
                time.sleep(delay_s * (attempt + 1))
                continue
            logger.error(f"Error updating workspace activity: {e}")
            return


class DatabaseCorruptionError(Exception):
    """Raised when the DuckDB file is corrupted."""
    pass


# Global tracker for corrupted workspaces - allows plugins to show notifications
_corrupted_workspaces: set[str] = set()
_corrupted_workspaces_logged: set[str] = set()
_busy_workspaces: dict[str, dict] = {}
_workspace_page_loads: dict[str, dict] = {}
_workspace_page_loads_lock = Lock()
_PAGE_LOAD_TTL_SECONDS = 300.0  # 5 minutes - long enough for sleep/throttling, short enough for abandoned jobs
_PAGE_LOAD_STATE_FILENAME = ".mint_page_load_state.json"

# Global registry for errors encountered during background processing
_PROCESSING_ERRORS: dict[str, list[str]] = {}
_PROCESSING_ERRORS_LOCK = Lock()
_LAST_ACTIVITY_UPDATE_TS: dict[str, float] = {}
_LAST_ACTIVITY_UPDATE_LOCK = Lock()


def is_workspace_corrupted(workspace_path: Path | str) -> bool:
    """Check if a workspace has been marked as corrupted."""
    if not workspace_path:
        return False
    return str(workspace_path) in _corrupted_workspaces


def is_workspace_busy(workspace_path: Path | str) -> bool:
    """Check if a workspace is currently marked as busy (write-locked elsewhere)."""
    if not workspace_path:
        return False
    return str(workspace_path) in _busy_workspaces


def clear_corruption_flag(workspace_path: Path | str):
    """Clear the corruption flag for a workspace (e.g., after user acknowledges)."""
    _corrupted_workspaces.discard(str(workspace_path))
    _corrupted_workspaces_logged.discard(str(workspace_path))


def clear_busy_flag(workspace_path: Path | str):
    """Clear the busy flag for a workspace."""
    _busy_workspaces.pop(str(workspace_path), None)


def clear_processing_errors(workspace_path: Path | str):
    """Clear any recorded processing errors for a workspace."""
    with _PROCESSING_ERRORS_LOCK:
        _PROCESSING_ERRORS.pop(str(workspace_path), None)


def _add_processing_error(workspace_path: Path | str, message: str):
    """Record a processing error for a workspace."""
    ws_key = str(workspace_path)
    with _PROCESSING_ERRORS_LOCK:
        if ws_key not in _PROCESSING_ERRORS:
            _PROCESSING_ERRORS[ws_key] = []
        _PROCESSING_ERRORS[ws_key].append(message)


def get_processing_errors(workspace_path: Path | str) -> list[str]:
    """Retrieve and clear any recorded processing errors for a workspace."""
    ws_key = str(workspace_path)
    with _PROCESSING_ERRORS_LOCK:
        return _PROCESSING_ERRORS.pop(ws_key, [])


def _normalize_workspace_key(workspace_path: Path | str | None) -> str | None:
    if not workspace_path:
        return None
    try:
        return str(Path(workspace_path).resolve())
    except Exception:
        return str(workspace_path)


class CancelledByRefresh(Exception):
    """Raised when a page refresh/abandon is detected for a long-running job."""


def _page_state_path(workspace_path: Path | str | None) -> Path | None:
    key = _normalize_workspace_key(workspace_path)
    if not key:
        return None
    try:
        ws_path = Path(key)
        return ws_path / _PAGE_LOAD_STATE_FILENAME
    except Exception:
        return None


def _read_page_state(state_path: Path | None) -> dict | None:
    if not state_path or not state_path.exists():
        return None
    try:
        return json.loads(state_path.read_text())
    except Exception:
        return None


def _write_page_state(state_path: Path | None, payload: dict) -> None:
    if not state_path:
        return
    try:
        # Never recreate deleted workspace folders just to persist page state.
        if not state_path.parent.exists():
            return
        tmp_path = state_path.with_suffix(state_path.suffix + ".tmp")
        tmp_path.write_text(json.dumps(payload))
        os.replace(tmp_path, state_path)
    except Exception as exc:
        logger.debug(f"Failed to write page state file {state_path}: {exc}")


def clear_page_load_state(workspace_path: Path | str | None) -> None:
    """Best-effort cleanup of in-memory and on-disk page-load state for a workspace."""
    key = _normalize_workspace_key(workspace_path)
    if not key:
        return
    with _workspace_page_loads_lock:
        _workspace_page_loads.pop(key, None)
    state_path = _page_state_path(key)
    if not state_path:
        return
    try:
        state_path.unlink(missing_ok=True)
    except Exception:
        pass


def mark_page_load_active(workspace_path: Path | str | None, page_load_id: str | None) -> None:
    """Record the latest active page-load token for a workspace."""
    key = _normalize_workspace_key(workspace_path)
    if not key or not page_load_id:
        return
    state_path = _page_state_path(key)
    previous_page_load_id = None
    previous_state = _read_page_state(state_path)
    if previous_state:
        previous_page_load_id = previous_state.get('page_load_id')
    else:
        with _workspace_page_loads_lock:
            previous = _workspace_page_loads.get(key)
            if previous:
                previous_page_load_id = previous.get('page_load_id')
    
    # CRITICAL: Write new ID FIRST so running jobs can detect the change
    payload = {
        'page_load_id': page_load_id,
        'timestamp': time.time(),
    }
    _write_page_state(state_path, payload)
    with _workspace_page_loads_lock:
        _workspace_page_loads[key] = payload
    
    # If the token changed (e.g., user refreshed/reopened tab), give running
    # jobs time to see the change and self-cancel before we proceed.
    if previous_page_load_id and previous_page_load_id != page_load_id:
        logger.debug(
            "Workspace session token changed (old=%s, new=%s); waiting for background jobs to cancel...",
            previous_page_load_id[:8] + "..." if previous_page_load_id else None,
            page_load_id[:8] + "..." if page_load_id else None,
        )
        # Brief pause to let running jobs detect the ID mismatch and self-cancel
        time.sleep(1.0)
        # Then aggressively clean up any external processes still holding the DB
        cleaned = _cleanup_workspace_db_users(key, aggressive=True)
        if cleaned:
            logger.warning(
                "Terminated %s prior DB process(es) for workspace=%s",
                cleaned,
                key,
            )


def is_page_load_active(
    workspace_path: Path | str | None,
    page_load_id: str | None,
    ttl_seconds: float = _PAGE_LOAD_TTL_SECONDS,
) -> bool:
    """Check whether the given page-load token is still the active owner."""
    key = _normalize_workspace_key(workspace_path)
    if not key or not page_load_id:
        return True
    state_path = _page_state_path(key)
    entry = _read_page_state(state_path)
    if not entry:
        with _workspace_page_loads_lock:
            entry = _workspace_page_loads.get(key)
    if not entry:
        # If no heartbeat has been recorded yet, assume active.
        return True
    if entry.get('page_load_id') != page_load_id:
        return False
    # ID matches - page is active (no TTL check to avoid false positives from sleep/throttling)
    return True


def ensure_page_load_active(
    workspace_path: Path | str | None,
    page_load_id: str | None,
    *,
    where: str = "",
    ttl_seconds: float = _PAGE_LOAD_TTL_SECONDS,
) -> None:
    """
    Cancel long-running work when the originating page is no longer active.

    This is a server-side guard for the "user hit F5/left the modal" case.
    """
    if not page_load_id:
        return
    if is_page_load_active(workspace_path, page_load_id, ttl_seconds=ttl_seconds):
        return
    location = f" at {where}" if where else ""
    msg = (
        "Cancelling background job due to page refresh/abandon"
        f"{location} (workspace={workspace_path})."
    )
    logger.warning(msg)
    raise CancelledByRefresh(msg)


def _mark_corrupted(workspace_path: Path | str):
    """Mark a workspace as corrupted."""
    _corrupted_workspaces.add(str(workspace_path))


def _extract_pid_from_busy_error(message: str) -> int | None:
    match = re.search(r"\bPID\s+(\d+)\b", message or "")
    if not match:
        return None
    try:
        return int(match.group(1))
    except Exception:
        return None


def _mark_busy(workspace_path: Path | str, message: str):
    """Mark a workspace as busy and store diagnostic info for notifications."""
    _busy_workspaces[str(workspace_path)] = {
        'message': message,
        'pid': _extract_pid_from_busy_error(message),
        'timestamp': time.time(),
    }


def get_corruption_notification():
    """
    Returns a notification component for a corrupted database.
    
    Plugins can use this when they detect conn is None and is_workspace_corrupted() is True.
    Returns a dict suitable for fac.AntdNotification or None if not applicable.
    """
    return {
        'message': "[!] Database Corrupted",
        'description': "This workspace has a corrupted database. Please delete it and restore from backup or recreate the workspace.",
        'type': "error",
        'duration': 10,
        'placement': 'bottom',
        'showProgress': True,
    }


def get_busy_notification(workspace_path: Path | str):
    """
    Returns a notification component description when a DB is busy.

    The busy flag is set when we refuse to open a write connection because
    another process appears to hold the lock.
    """
    if not workspace_path:
        return None
    info = _busy_workspaces.get(str(workspace_path))
    if not info:
        return None
    pid = info.get('pid')
    if pid:
        description = (
            f"This workspace is currently write-locked by another MINT process (PID {pid}). "
            "Please wait; you'll be notified when it is available again."
        )
    else:
        description = (
            "This workspace is currently write-locked by another MINT process. "
            "Please wait; you'll be notified when it is available again."
        )
    return {
        'message': "[!] Database Busy",
        'description': description,
        'type': "warning",
        'duration': 8,
        'placement': 'bottom',
        'showProgress': True,
    }


class DatabaseBusyError(RuntimeError):
    """Raised when another live process appears to hold the workspace DB."""


def _db_lockfile_path(db_file: Path) -> Path:
    return db_file.with_suffix(db_file.suffix + ".lock")


def is_workspace_busy_probe(workspace_path: Path | str | None) -> bool:
    """
    Best-effort busy check that does NOT open the database.
    Uses the app-level lockfile and validates the owning PID when possible.
    """
    if not workspace_path:
        return False
    try:
        workspace_path = Path(workspace_path).resolve()
    except Exception:
        workspace_path = Path(str(workspace_path))
    db_file = workspace_path / 'workspace_mint.db'
    lock_file = _db_lockfile_path(db_file)

    if not lock_file.exists():
        clear_busy_flag(workspace_path)
        return False

    if psutil is None:
        # Without psutil we can't verify ownership; treat as busy.
        return True

    info = _safe_read_lockfile(lock_file) or {}
    other_pid = int(info.get('pid', -1))
    other_ctime = info.get('create_time')

    if other_pid > 0 and _pid_matches_create_time(other_pid, other_ctime):
        return True

    # Stale lockfile or dead process: clear and mark free.
    try:
        lock_file.unlink(missing_ok=True)
    except Exception:
        pass
    clear_busy_flag(workspace_path)
    return False


def _safe_read_lockfile(lock_file: Path) -> dict | None:
    try:
        return json.loads(lock_file.read_text())
    except Exception:
        return None


def _pid_matches_create_time(pid: int, expected_create_time: float | None) -> bool:
    """Best-effort protection against PID reuse."""
    if psutil is None:
        # Without psutil we cannot safely inspect other processes.
        return pid == os.getpid()
    try:
        proc = psutil.Process(pid)
        if not proc.is_running():
            return False
        if expected_create_time is None:
            return True
        actual_create_time = proc.create_time()
        return abs(actual_create_time - float(expected_create_time)) < 2.0
    except (psutil.NoSuchProcess, psutil.ZombieProcess, psutil.AccessDenied, ValueError):
        return False


def _is_likely_mint_process(proc, workspace_path: Path) -> bool:
    """Avoid touching unrelated processes when cleaning up orphans."""
    if psutil is None:
        return False
    try:
        cmdline = " ".join(proc.cmdline()).lower()
    except (psutil.AccessDenied, psutil.ZombieProcess):
        cmdline = ""
    ws_str = workspace_path.as_posix().lower()
    return ("ms_mint_app" in cmdline) or (ws_str and ws_str in cmdline)


def _is_orphan_process(proc) -> bool:
    """Heuristic: parent missing, dead, or effectively re-parented to init."""
    if psutil is None:
        return False
    try:
        parent = proc.parent()
        if parent is None:
            return True
        if not parent.is_running():
            return True
        parent_pid = parent.pid
        # On Unix-like systems, orphaned processes are often re-parented to PID 1.
        if os.name != 'nt' and parent_pid == 1:
            return True
        # On Windows, orphaned processes can end up with the System process
        # (PID 4) or PID 0 as an ancestor/parent. We treat those as orphaned.
        if os.name == 'nt' and parent_pid in (0, 4):
            return True
        # Additional Windows heuristic: if the parent is the System process
        # by name, treat it as orphaned.
        if os.name == 'nt':
            try:
                parent_name = (parent.name() or "").lower()
                if parent_name in {"system", "system idle process"}:
                    return True
            except (psutil.AccessDenied, psutil.ZombieProcess):
                pass
        return False
    except (psutil.NoSuchProcess, psutil.ZombieProcess):
        return True
    except psutil.AccessDenied:
        return False


def _terminate_process(proc, reason: str) -> bool:
    """Terminate a process gracefully, then force-kill if needed."""
    if psutil is None:
        return False
    try:
        logger.warning(f"Terminating process {proc.pid} holding DB ({reason}).")
        proc.terminate()
        try:
            proc.wait(timeout=3)
            return True
        except psutil.TimeoutExpired:
            logger.warning(f"Process {proc.pid} did not exit in time; killing.")
            proc.kill()
            proc.wait(timeout=3)
            return True
    except (psutil.NoSuchProcess, psutil.ZombieProcess):
        return True
    except psutil.AccessDenied as exc:
        logger.error(f"Access denied when terminating process {proc.pid}: {exc}")
        return False
    except Exception as exc:
        logger.error(f"Unexpected error terminating process {proc.pid}: {exc}")
        return False


def _cleanup_orphan_db_users(db_file: Path, workspace_path: Path) -> int:
    """
    Scan for orphaned MINT processes that still have the DB open and stop them.

    This is intentionally conservative: we only act on processes that appear
    to be MINT-related and orphaned.
    """
    if psutil is None:
        return 0
    if not db_file.exists():
        return 0
    db_abs = db_file.resolve().as_posix()
    current_pid = os.getpid()
    cleaned = 0

    for proc in psutil.process_iter(['pid', 'open_files']):
        try:
            if proc.pid == current_pid:
                continue
            open_files = proc.info.get('open_files') or []
            if not any(getattr(f, "path", None) == db_abs for f in open_files):
                continue

            if not _is_likely_mint_process(proc, workspace_path):
                continue
            if not _is_orphan_process(proc):
                continue

            if _terminate_process(proc, reason="orphaned DB user"):
                cleaned += 1
        except (psutil.NoSuchProcess, psutil.ZombieProcess):
            continue
        except psutil.AccessDenied:
            continue
        except Exception as exc:
            logger.debug(f"Error while scanning DB users: {exc}")

    if cleaned:
        logger.warning(f"Cleaned up {cleaned} orphaned process(es) using {db_file.name}.")
    return cleaned


def _describe_db_users(db_file: Path, workspace_path: Path, only_mint: bool = True) -> list[dict]:
    """
    Best-effort diagnostic: list other processes that currently have the DB open.
    """
    if psutil is None:
        return []
    try:
        db_abs = db_file.resolve().as_posix()
    except Exception:
        db_abs = db_file.as_posix()
    current_pid = os.getpid()
    users: list[dict] = []

    for proc in psutil.process_iter(['pid', 'name', 'open_files']):
        try:
            if proc.pid == current_pid:
                continue
            open_files = proc.info.get('open_files') or []
            if not any(getattr(f, "path", None) == db_abs for f in open_files):
                continue
            if only_mint and not _is_likely_mint_process(proc, workspace_path):
                continue

            try:
                cmdline = " ".join(proc.cmdline())
            except Exception:
                cmdline = ""
            try:
                parent_pid = proc.ppid()
            except Exception:
                parent_pid = None

            users.append({
                'pid': proc.pid,
                'name': proc.info.get('name'),
                'ppid': parent_pid,
                'orphan': _is_orphan_process(proc),
                'cmdline': cmdline[:240],
            })
        except (psutil.NoSuchProcess, psutil.ZombieProcess):
            continue
        except psutil.AccessDenied:
            continue
        except Exception:
            continue

    return users


def _cleanup_workspace_db_users(workspace_path: Path | str, aggressive: bool = False) -> int:
    """
    Clean up MINT processes holding the workspace DB.

    When aggressive=True, we terminate any other MINT process that appears to
    hold the DB, even if it is not orphaned. This is intended for explicit
    "user refreshed/abandoned the modal" cases.
    """
    if psutil is None:
        return 0
    try:
        workspace_path = Path(workspace_path).resolve()
    except Exception:
        workspace_path = Path(str(workspace_path))
    db_file = workspace_path / 'workspace_mint.db'
    if not db_file.exists():
        return 0

    db_abs = db_file.resolve().as_posix()
    current_pid = os.getpid()
    cleaned = 0

    for proc in psutil.process_iter(['pid', 'open_files']):
        try:
            if proc.pid == current_pid:
                continue
            open_files = proc.info.get('open_files') or []
            if not any(getattr(f, "path", None) == db_abs for f in open_files):
                continue
            if not _is_likely_mint_process(proc, workspace_path):
                continue
            if not aggressive and not _is_orphan_process(proc):
                continue
            if _terminate_process(proc, reason="workspace DB cleanup"):
                cleaned += 1
        except (psutil.NoSuchProcess, psutil.ZombieProcess):
            continue
        except psutil.AccessDenied:
            continue
        except Exception as exc:
            logger.debug(f"Error while cleaning workspace DB users: {exc}")

    if cleaned:
        mode = "aggressive" if aggressive else "orphan"
        logger.warning(
            "Cleaned up %s %s DB process(es) using %s.",
            cleaned,
            mode,
            db_file.name,
        )
    return cleaned


def _acquire_db_write_lock(db_file: Path, workspace_path: Path, max_attempts: int = 3) -> Path:
    """
    Acquire an app-level write lock, clearing stale/orphaned holders when safe.
    """
    lock_file = _db_lockfile_path(db_file)
    lock_key = lock_file.as_posix()
    lock_file.parent.mkdir(parents=True, exist_ok=True)
    if psutil is not None:
        proc_self = psutil.Process()
        create_time = proc_self.create_time()
        cmdline = proc_self.cmdline()
    else:
        create_time = None
        cmdline = []
    payload = {
        'pid': os.getpid(),
        'create_time': create_time,
        'cmdline': cmdline,
        'workspace_path': workspace_path.as_posix(),
        'db_path': db_file.as_posix(),
        'timestamp': time.time(),
    }

    # Allow nested/repeated connections within the same process.
    with _DB_LOCK_COUNTS_LOCK:
        existing = _DB_LOCK_COUNTS.get(lock_key, 0)
        if existing > 0 and lock_file.exists():
            _DB_LOCK_COUNTS[lock_key] = existing + 1
            return lock_file
        if existing > 0 and not lock_file.exists():
            _DB_LOCK_COUNTS.pop(lock_key, None)

    # Only run orphan cleanup when there is evidence of contention.
    # Full process scans are expensive on Windows.
    if lock_file.exists():
        _cleanup_orphan_db_users(db_file, workspace_path)

    attempt = 0
    while attempt < max_attempts:
        attempt += 1
        try:
            fd = os.open(lock_file, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
            with os.fdopen(fd, 'w') as f:
                json.dump(payload, f)
            with _DB_LOCK_COUNTS_LOCK:
                _DB_LOCK_COUNTS[lock_key] = 1
            return lock_file
        except FileExistsError:
            info = _safe_read_lockfile(lock_file) or {}
            other_pid = int(info.get('pid', -1))
            other_ctime = info.get('create_time')

            # If we already own the lockfile, just bump the refcount.
            if other_pid == os.getpid() and _pid_matches_create_time(other_pid, other_ctime):
                with _DB_LOCK_COUNTS_LOCK:
                    _DB_LOCK_COUNTS[lock_key] = _DB_LOCK_COUNTS.get(lock_key, 0) + 1
                return lock_file

            if other_pid > 0 and _pid_matches_create_time(other_pid, other_ctime):
                try:
                    if psutil is not None:
                        other_proc = psutil.Process(other_pid)
                        if _is_likely_mint_process(other_proc, workspace_path) and _is_orphan_process(other_proc):
                            if _terminate_process(other_proc, reason="orphaned lock holder"):
                                lock_file.unlink(missing_ok=True)
                                continue
                except (psutil.NoSuchProcess, psutil.ZombieProcess):
                    lock_file.unlink(missing_ok=True)
                    continue
                except psutil.AccessDenied:
                    pass

                raise DatabaseBusyError(
                    f"Workspace DB appears to be in use by PID {other_pid}: {lock_file}"
                )

            # Stale/invalid lockfile: remove and retry.
            try:
                lock_file.unlink(missing_ok=True)
            except Exception:
                pass
            time.sleep(0.1 * attempt)

    raise DatabaseBusyError(f"Unable to acquire DB write lock: {lock_file}")


def _release_db_write_lock(lock_file: Path | None) -> None:
    if not lock_file:
        return
    lock_key = lock_file.as_posix()
    with _DB_LOCK_COUNTS_LOCK:
        count = _DB_LOCK_COUNTS.get(lock_key, 0)
        if count > 1:
            _DB_LOCK_COUNTS[lock_key] = count - 1
            return
        _DB_LOCK_COUNTS.pop(lock_key, None)
    try:
        info = _safe_read_lockfile(lock_file) or {}
        if int(info.get('pid', -1)) in (-1, os.getpid()):
            lock_file.unlink(missing_ok=True)
            return
    except Exception:
        pass
    # Best-effort cleanup if the lockfile is malformed.
    try:
        lock_file.unlink(missing_ok=True)
    except Exception:
        pass


@contextmanager
def duckdb_connection(
    workspace_path: Path | str,
    register_activity=True,
    n_cpus=None,
    ram=None,
    page_load_id: str | None = None,
    read_only: bool = False,
):
    """
    Provides a DuckDB connection as a context manager.

    The database file will be named 'workspace_mint.db' and will be located inside the workspace directory.

    :param workspace_path: The path to the MINT workspace directory.
    :param read_only: If True, opens connection in read-only mode and skips schema creation/locking.
    """
    if not workspace_path:
        yield None
        return
    ensure_page_load_active(workspace_path, page_load_id, where="duckdb_connection:pre-connect")
    workspace_path = Path(workspace_path).resolve()
    if not workspace_path.exists():
        logger.debug("Workspace path does not exist for DuckDB connection: %s", workspace_path)
        yield None
        return
    db_file = Path(workspace_path, 'workspace_mint.db')
    if read_only and not db_file.exists():
        logger.debug("Skipping read-only DuckDB connection because DB file does not exist yet: %s", db_file)
        yield None
        return
    # print(f"Connecting to DuckDB at: {db_file}")
    con = None
    lock_file = None
    max_retries = 3
    retry_delay = 0.5
    requested_read_only = read_only
    for attempt in range(max_retries):
        try:
            if requested_read_only:
                # Read-only mode: Skip write lock and schema updates
                # This is much faster on Windows and allows concurrency
                con = duckdb.connect(database=str(db_file), read_only=True)
                logger.debug(
                    "Opened DuckDB READ-ONLY connection (pid=%s, db=%s)",
                    os.getpid(),
                    db_file,
                )
            else:
                # Write mode: Acquire exclusive lock and ensure schema
                lock_file = _acquire_db_write_lock(db_file, workspace_path)
                clear_busy_flag(workspace_path)
                logger.debug(
                    "Acquired DuckDB write lock (pid=%s, db=%s)",
                    os.getpid(),
                    db_file,
                )
                con = duckdb.connect(database=str(db_file), read_only=False)
                logger.debug(
                    "Opened DuckDB write connection (pid=%s, db=%s)",
                    os.getpid(),
                    db_file,
                )
            
            # Common PRAGMAs
            if not requested_read_only:
                con.execute("PRAGMA enable_checkpoint_on_shutdown")
            
            con.execute("SET enable_progress_bar = true")
            con.execute("SET enable_progress_bar_print = false")
            con.execute("SET progress_bar_time = 0")
            # Try to set temp_directory, but don't fail if it can't be changed
            try:
                con.execute(f"SET temp_directory = '{workspace_path.as_posix()}';")
            except Exception:
                pass  # temp_directory already set or can't be changed - not critical
            if n_cpus:
                # Cap CPUs at RAM (1GB per CPU minimum to prevent resource imbalance)
                effective_cpus = get_effective_cpus(n_cpus, ram) if ram else n_cpus
                con.execute(f"SET threads = {effective_cpus}")
                if effective_cpus != n_cpus:
                    logger.info(f"DuckDB threads capped at {effective_cpus} (requested {n_cpus}, RAM limit {ram}GB)")
                else:
                    logger.debug(f"DuckDB threads set to {effective_cpus}")
            if ram:
                con.execute(f"SET memory_limit = '{ram}GB'")
                # Verify the setting was applied
                actual_limit = con.execute("SELECT current_setting('memory_limit')").fetchone()[0]
                logger.debug(f"DuckDB memory_limit set to {ram}GB (verified: {actual_limit})")
            
            # Only create tables if we are in write mode
            if not requested_read_only:
                _create_tables(con)
            
            break # Success
        except DatabaseBusyError as e:
            logger.error(f"Database busy; refusing to open write connection: {e}")
            try:
                busy_users = _describe_db_users(db_file, workspace_path, only_mint=True)
                if busy_users:
                    logger.error("Processes currently holding the DB open:")
                    for u in busy_users:
                        logger.error(
                            "  pid=%s name=%s ppid=%s orphan=%s cmd=%s",
                            u.get('pid'),
                            u.get('name'),
                            u.get('ppid'),
                            u.get('orphan'),
                            u.get('cmdline'),
                        )
                else:
                    logger.error("No DB holders detected via psutil.open_files (may be permissions/race).")
            except Exception as diag_exc:
                logger.debug(f"Failed to describe DB users: {diag_exc}")
            _mark_busy(workspace_path, str(e))
            _release_db_write_lock(lock_file)
            yield None
            return
        except (duckdb.IOException, duckdb.BinderException) as e:
            _release_db_write_lock(lock_file)
            err_msg = str(e)
            # Recovery path for previously created empty placeholder DB files.
            if "not a valid DuckDB database file" in err_msg and db_file.exists():
                try:
                    if db_file.stat().st_size == 0:
                        db_file.unlink(missing_ok=True)
                        logger.warning("Removed invalid empty DuckDB file and retrying: %s", db_file)
                        if attempt < max_retries - 1:
                            time.sleep(0.05)
                            continue
                except Exception:
                    pass
            if "Corrupt database file" in err_msg:
                _mark_corrupted(workspace_path)  # Mark for UI notification
                ws_key = str(workspace_path)
                if ws_key not in _corrupted_workspaces_logged:
                    _corrupted_workspaces_logged.add(ws_key)
                    ws_name = get_workspace_name_from_wdir(workspace_path) or "Unknown"
                    logger.critical(
                        f"[!] DATABASE CORRUPTION DETECTED in {db_file} (workspace: {ws_name}): {e}\n"
                        "This usually happens due to a system crash or forced termination during a write operation.\n"
                        "Please delete this workspace and restore from backup or recreate it."
                    )
                yield None
                return
            
            if attempt < max_retries - 1:
                logger.warning(f"Error connecting to DuckDB (attempt {attempt + 1}/{max_retries}): {e}. Retrying in {retry_delay}s...")
                time.sleep(retry_delay)
                retry_delay *= 2 # Exponential backoff
            else:
                logger.error(f"Failed to connect to DuckDB after {max_retries} attempts: {e}")
                yield None
                return
        except Exception as e:
            msg = str(e)
            if requested_read_only and "different configuration than existing connections" in msg:
                logger.debug(
                    "DuckDB read-only mode conflict for %s; retrying with standard connection mode.",
                    db_file,
                )
                requested_read_only = False
                continue
            logger.error(f"Unexpected error connecting to DuckDB: {e}")
            _release_db_write_lock(lock_file)
            yield None
            return

    try:
        yield con
    finally:
        if con:
            if ram:
                con.execute("RESET memory_limit")
            con.close()
        try:
            _release_db_write_lock(lock_file)
        except Exception:
            pass # Ignore errors during lock release (e.g. folder deleted)
        if lock_file:
            logger.debug(
                "Released DuckDB write lock (pid=%s, lock=%s)",
                os.getpid(),
                lock_file,
            )


@contextmanager
def duckdb_connection_mint(mint_path: Path, workspace=None, read_only=False):
    if not mint_path:
        yield None
        return

    db_file = Path(mint_path, 'mint.db')
    
    # Auto-optimize: Check size before connecting (SKIP in read-only mode)
    if not read_only and db_file.exists():
        try:
             size_mb = db_file.stat().st_size / (1024 * 1024)
             if size_mb > 50:
                 logger.info(f"mint.db is large ({size_mb:.1f} MB). Compacting...")
                 success, msg = compact_mint_database(mint_path)
                 if success:
                     logger.info(f"mint.db compaction successful: {msg}")
                 else:
                     logger.warning(f"mint.db compaction failed: {msg}")
        except Exception as e:
            logger.warning(f"Error during mint.db auto-compaction check: {e}")

    con = None
    max_retries = 5
    retry_delay = 0.3
    last_error = None
    
    for attempt in range(max_retries):
        try:
            con = duckdb.connect(database=str(db_file), read_only=read_only)
            if not read_only:
                _create_workspace_tables(con)
            break  # Success
        except (duckdb.IOException, duckdb.BinderException) as e:
            last_error = e
            error_str = str(e).lower()
            # Check for recoverable connection conflicts
            if "already attached" in error_str or "lock" in error_str or "file handle conflict" in error_str:
                if attempt < max_retries - 1:
                    logger.debug(
                        f"mint.db connection conflict (attempt {attempt + 1}/{max_retries}): {e}. "
                        f"Retrying in {retry_delay:.1f}s..."
                    )
                    time.sleep(retry_delay)
                    retry_delay *= 2  # Exponential backoff
                    continue
            # Non-recoverable or exhausted retries
            # If read-only fails due to lock, it might be temporarily exclusive locked by a writer
            if "used by another process" in error_str or "lock" in error_str or "file handle conflict" in error_str:
                logger.debug(f"mint.db temporarily busy (read_only={read_only}): {e}")
            else:
                logger.error(f"Error connecting to mint.db (read_only={read_only}): {e}")
            yield None
            return
        except Exception as e:
            logger.error(f"Unexpected error connecting to mint.db: {e}")
            yield None
            return
    
    if con is None:
        logger.error(f"Failed to connect to mint.db after {max_retries} attempts: {last_error}")
        yield None
        return
        
    try:
        yield con
    finally:
        if con:
            # Only update activity if we have a write connection
            if workspace and not read_only:
                try:
                    con.execute("UPDATE workspaces SET last_activity = NOW() WHERE key = ?", [workspace])
                except Exception:
                    pass
            con.close()


def compact_mint_database(mint_path: Path | str) -> tuple[bool, str]:
    """
    Compact the mint.db by rebuilding it.
    """
    import shutil
    import uuid
    
    mint_path = Path(mint_path)
    db_file = mint_path / 'mint.db'
    
    if not db_file.exists():
        return False, "Database file not found"
        
    original_size = db_file.stat().st_size
    temp_db_file = mint_path / f'mint_{uuid.uuid4().hex[:8]}.db.tmp'
    backup_file = mint_path / 'mint.db.bak'
    
    try:
        new_con = duckdb.connect(database=str(temp_db_file), read_only=False)
        new_con.execute("PRAGMA enable_checkpoint_on_shutdown")
        _create_workspace_tables(new_con)
        
        # Attach old DB
        new_con.execute(f"ATTACH '{db_file}' AS old_db (READ_ONLY)")
        
        # Copy workspaces
        count = new_con.execute("SELECT COUNT(*) FROM old_db.workspaces").fetchone()[0]
        if count > 0:
            new_con.execute("INSERT INTO workspaces SELECT * FROM old_db.workspaces")
            
        new_con.execute("DETACH old_db")
        new_con.close()
        
        # Swap files
        if backup_file.exists():
            backup_file.unlink()
        db_file.rename(backup_file)
        temp_db_file.rename(db_file)
        if backup_file.exists():
            backup_file.unlink()
            
        new_size = db_file.stat().st_size
        return True, f"{original_size/1024/1024:.1f}MB -> {new_size/1024/1024:.1f}MB"
        
    except Exception as e:
        if temp_db_file.exists():
            try:
                temp_db_file.unlink()
            except:
                pass
        return False, str(e)


def get_workspace_name_from_wdir(wdir: Path | str | None) -> str | None:
    if not wdir:
        return None

    try:
        wdir_path = Path(wdir)
    except Exception:
        return None

    ws_key = wdir_path.stem
    mint_root = wdir_path.parent.parent
    try:
        with duckdb_connection_mint(mint_root) as mint_conn:
            if mint_conn is None:
                return None
            ws_row = mint_conn.execute(
                "SELECT name FROM workspaces WHERE key = ?",
                [ws_key],
            ).fetchone()
            if ws_row is not None:
                return ws_row[0]
    except Exception:
        return None

    return None


def compact_database(workspace_path: Path | str, max_retries: int = 5, initial_delay: float = 0.5) -> tuple[bool, str]:
    """
    Compact the workspace database by rebuilding it.
    
    DuckDB doesn't automatically reclaim space after DELETEs, so this function
    creates a new database file with only the current data, then replaces
    the old file.
    
    Args:
        workspace_path: Path to the workspace directory
        max_retries: Maximum number of retry attempts for lock conflicts
        initial_delay: Initial delay between retries (doubles each attempt)
        
    Returns:
        (success, error_message)
    """
    import shutil
    import uuid
    
    workspace_path = Path(workspace_path)
    db_file = workspace_path / 'workspace_mint.db'
    
    if not db_file.exists():
        return False, f"Database file not found: {db_file}"
    
    # Get original file size for logging
    original_size = db_file.stat().st_size
    original_size_mb = original_size / (1024 * 1024)
    
    # Create temporary file name for the new database
    temp_db_file = workspace_path / f'workspace_mint_{uuid.uuid4().hex[:8]}.db.tmp'
    backup_file = workspace_path / 'workspace_mint.db.bak'
    
    try:
        # Create new database and copy all tables
        new_con = None
        try:
            new_con = duckdb.connect(database=str(temp_db_file), read_only=False)
            new_con.execute("PRAGMA enable_checkpoint_on_shutdown")
            
            # Limit memory to 50% of available RAM to prevent system freeze
            import psutil
            available_ram_gb = psutil.virtual_memory().available / (1024 ** 3)
            memory_limit_gb = max(2, int(available_ram_gb * 0.5))  # At least 2GB, max 50%
            new_con.execute(f"SET memory_limit = '{memory_limit_gb}GB'")
            logger.debug(f"Compaction memory limit set to {memory_limit_gb}GB (50% of {available_ram_gb:.1f}GB available)")
            
            # Create all types and tables in the new database
            _create_tables(new_con)
            
            # Attach the old database with retry logic for lock conflicts
            delay = initial_delay
            attached = False
            last_error = None
            
            for attempt in range(max_retries):
                try:
                    new_con.execute(f"ATTACH '{db_file}' AS old_db (READ_ONLY)")
                    attached = True
                    break
                except duckdb.IOException as e:
                    last_error = e
                    if "lock" in str(e).lower() and attempt < max_retries - 1:
                        logger.debug(f"Compact: lock conflict on attach, retry {attempt + 1}/{max_retries} after {delay:.1f}s")
                        time.sleep(delay)
                        delay *= 2  # Exponential backoff
                    else:
                        raise
            
            if not attached:
                raise last_error or Exception("Failed to attach old database")
            
            # Copy data from all tables
            tables_to_copy = ['samples', 'targets', 'ms1_data', 'ms2_data', 'chromatograms', 'results']
            for table in tables_to_copy:
                # Check if table has data in old database
                try:
                    count = new_con.execute(f"SELECT COUNT(*) FROM old_db.{table}").fetchone()[0]
                    if count > 0:
                        # Copy data - INSERT INTO ... SELECT * FROM ...
                        new_con.execute(f"INSERT INTO {table} SELECT * FROM old_db.{table}")
                        logger.debug(f"Compacted table '{table}': {count} rows")
                except Exception as e:
                    # Table might not exist in old DB, which is fine
                    logger.debug(f"Skipping table '{table}' during compaction: {e}")
            
            # Detach old database
            new_con.execute("DETACH old_db")
            
            # Checkpoint to ensure all data is written
            new_con.execute("CHECKPOINT")
            
        finally:
            if new_con:
                new_con.close()
        
        # Now swap the files
        # Rename old to backup
        if backup_file.exists():
            backup_file.unlink()
        db_file.rename(backup_file)
        
        # Rename new to original
        temp_db_file.rename(db_file)
        
        # Remove backup
        if backup_file.exists():
            backup_file.unlink()
        
        # Log the size reduction
        new_size = db_file.stat().st_size
        new_size_mb = new_size / (1024 * 1024)
        reduction_pct = ((original_size - new_size) / original_size * 100) if original_size > 0 else 0
        
        logger.info(f"Database compacted: {original_size_mb:.1f}MB -> {new_size_mb:.1f}MB ({reduction_pct:.0f}% reduction)")
        
        return True, f"Compacted {original_size_mb:.1f}MB -> {new_size_mb:.1f}MB"
        
    except Exception as e:
        logger.error(f"Error compacting database: {e}", exc_info=True)
        
        # Cleanup: restore backup if it exists
        if backup_file.exists() and not db_file.exists():
            try:
                backup_file.rename(db_file)
                logger.info("Restored database from backup after failed compaction")
            except Exception as restore_error:
                logger.error(f"Failed to restore backup: {restore_error}")
        
        # Remove temp file if it exists
        if temp_db_file.exists():
            try:
                temp_db_file.unlink()
            except Exception:
                pass
        
        return False, str(e)


def _create_tables(conn: duckdb.DuckDBPyConnection):
    # Create tables if they don't exist
    conn.execute("CREATE TYPE IF NOT EXISTS ms_type_enum AS ENUM ('ms1', 'ms2');")
    conn.execute("CREATE TYPE IF NOT EXISTS polarity_enum AS ENUM ('Positive', 'Negative');")
    conn.execute("CREATE TYPE IF NOT EXISTS unit_type_enum AS ENUM ('s', 'min');")

    conn.execute("""
                 CREATE TABLE IF NOT EXISTS samples
                 (
                     ms_file_label        VARCHAR PRIMARY KEY,
                     ms_type              ms_type_enum,
                     file_type            VARCHAR,
                     use_for_optimization BOOLEAN DEFAULT true,
                     use_for_processing   BOOLEAN DEFAULT true,
                     use_for_analysis     BOOLEAN DEFAULT true,
                     polarity             polarity_enum,
                     color                VARCHAR DEFAULT '#BBBBBB',
                     label                VARCHAR,
                     sample_type          VARCHAR DEFAULT 'Sample',
                     group_1              VARCHAR,
                     group_2              VARCHAR,
                     group_3              VARCHAR,
                     group_4              VARCHAR,
                     group_5              VARCHAR
                 );
                 """)

    # Backfill new processing flag for existing DBs
    conn.execute("ALTER TABLE samples ADD COLUMN IF NOT EXISTS use_for_processing BOOLEAN DEFAULT true;")
    conn.execute("ALTER TABLE samples ADD COLUMN IF NOT EXISTS file_type VARCHAR;")
    conn.execute("ALTER TABLE samples ADD COLUMN IF NOT EXISTS acquisition_datetime TIMESTAMP;")
    for col in GROUP_COLUMNS:
        conn.execute(f"ALTER TABLE samples ADD COLUMN IF NOT EXISTS {col} VARCHAR;")
    try:
        conn.execute("""
                     UPDATE samples
                     SET use_for_processing = COALESCE(use_for_processing, use_for_analysis, TRUE)
                     """)
    except Exception:
        # Avoid failing during initialization if another write is in flight.
        pass

    conn.execute("""
                 CREATE TABLE IF NOT EXISTS ms1_data
                 (
                     ms_file_label      VARCHAR,  -- Label of the MS file, linking to samples
                     scan_id            INTEGER,  -- Scan ID
                     mz                 DOUBLE,   -- Mass-to-charge ratio
                     intensity          DOUBLE,   -- Intensity
                     scan_time          DOUBLE    -- Scan time
                 );
                 """)
    conn.execute("""
                 CREATE TABLE IF NOT EXISTS ms2_data
                 (
                     ms_file_label      VARCHAR,  -- Label of the MS file, linking to samples
                     scan_id            INTEGER,  -- Scan ID
                     mz                 DOUBLE,   -- Mass-to-charge ratio
                     intensity          DOUBLE,   -- Intensity
                     scan_time          DOUBLE,   -- Scan time
                     mz_precursor       DOUBLE,   -- Precursor m/z
                     filterLine         VARCHAR,  -- Filter line from the raw file
                     filterLine_ELMAVEN VARCHAR   -- Filter line formatted for El-Maven
                 );
                 """)

    conn.execute("""
                 CREATE TABLE IF NOT EXISTS targets
                 (
                     peak_label          VARCHAR PRIMARY KEY, -- Label for the peak
                     mz_mean             DOUBLE,              -- Mean mass-to-charge ratio
                     mz_width            DOUBLE,              -- Width of the m/z window
                     mz                  DOUBLE,              -- Mass-to-charge ratio
                     rt                  DOUBLE,              -- Retention time
                     rt_min              DOUBLE,              -- Minimum retention time
                     rt_max              DOUBLE,              -- Maximum retention time
                     rt_unit             unit_type_enum,      -- Unit of retention time
                     intensity_threshold DOUBLE,              -- Intensity threshold
                     polarity            polarity_enum,       -- Polarity of the target
                     filterLine          VARCHAR,             -- Filter line from the raw file
                     ms_type             ms_type_enum,        -- MS type (ms1 or ms2)
                     category            VARCHAR,             -- Category of the target
                     peak_selection      BOOLEAN,             -- Preselected target
                     score               DOUBLE,              -- Score of the target
                     bookmark            BOOLEAN,             -- Bookmark the target
                     source              VARCHAR,             -- Filename of the target list
                     notes               VARCHAR,             -- Additional notes for the target
                     formula             VARCHAR,             -- Chemical formula
                     maven_id            VARCHAR,             -- Original ID from El-Maven
                     adduct_name         VARCHAR,             -- Adduct name (e.g., [M+H]+)
                     rt_auto_adjusted    BOOLEAN DEFAULT FALSE -- RT was auto-adjusted (outside span)
                 );
                 """)
    # Backfill rt_auto_adjusted for existing DBs
    # Handle concurrency: If multiple threads try to ALTER simultaneously, ignore conflicts
    try:
        conn.execute("ALTER TABLE targets ADD COLUMN IF NOT EXISTS rt_auto_adjusted BOOLEAN DEFAULT FALSE;")
        conn.execute("ALTER TABLE targets ADD COLUMN IF NOT EXISTS formula VARCHAR;")
        conn.execute("ALTER TABLE targets ADD COLUMN IF NOT EXISTS maven_id VARCHAR;")
        conn.execute("ALTER TABLE targets ADD COLUMN IF NOT EXISTS adduct_name VARCHAR;")
        
        # RT Alignment columns for storing alignment parameters
        conn.execute("ALTER TABLE targets ADD COLUMN IF NOT EXISTS rt_align_enabled BOOLEAN DEFAULT FALSE;")
        conn.execute("ALTER TABLE targets ADD COLUMN IF NOT EXISTS rt_align_reference_rt DOUBLE;")
        conn.execute("ALTER TABLE targets ADD COLUMN IF NOT EXISTS rt_align_shifts JSON;")
        conn.execute("ALTER TABLE targets ADD COLUMN IF NOT EXISTS rt_align_rt_min DOUBLE;")
        conn.execute("ALTER TABLE targets ADD COLUMN IF NOT EXISTS rt_align_rt_max DOUBLE;")
    except Exception as e:
        # "Catalog write-write conflict" means another thread is doing this right now.
        # "Column ... already exists" means it's done. 
        # In both cases, we are safe to proceed.
        pass

    conn.execute("""
                 CREATE TABLE IF NOT EXISTS chromatograms
                 (
                     peak_label    VARCHAR,
                     ms_file_label VARCHAR,
                     scan_time     DOUBLE[],
                     intensity     DOUBLE[],
                     scan_time_full_ds DOUBLE[],
                     intensity_full_ds DOUBLE[],
                     mz_arr        DOUBLE[],
                     ms_type       ms_type_enum,
                     PRIMARY KEY (ms_file_label, peak_label)
                 );
                 """)

    conn.execute("""
                 CREATE TABLE IF NOT EXISTS results
                 (
                     peak_label        VARCHAR,
                     ms_file_label     VARCHAR,
                     total_intensity   DOUBLE,
                     peak_area         DOUBLE,
                     peak_area_top3    DOUBLE,
                     peak_max          DOUBLE,
                     peak_min          DOUBLE,
                     peak_mean         DOUBLE,
                     peak_rt_of_max    DOUBLE,
                     peak_median       DOUBLE,
                     peak_n_datapoints INT,
                     rt_aligned        BOOLEAN,  -- TRUE if RT alignment was applied
                     rt_shift          DOUBLE,   -- Shift value applied (0 if not aligned)
                     scan_time         DOUBLE[],
                     intensity         DOUBLE[],
                     PRIMARY KEY (ms_file_label, peak_label)
                 );
                 """)

    # Migration: Add mz_arr column to chromatograms table
    try:
        chrom_cols = {row[0] for row in conn.execute("DESCRIBE chromatograms").fetchall()}
        if 'mz_arr' not in chrom_cols:
            conn.execute("ALTER TABLE chromatograms ADD COLUMN mz_arr DOUBLE[]")
            logger.debug("Migration: Added 'mz_arr' column to chromatograms table")
        if 'scan_time_full_ds' not in chrom_cols:
            conn.execute("ALTER TABLE chromatograms ADD COLUMN scan_time_full_ds DOUBLE[]")
            logger.debug("Migration: Added 'scan_time_full_ds' column to chromatograms table")
        if 'intensity_full_ds' not in chrom_cols:
            conn.execute("ALTER TABLE chromatograms ADD COLUMN intensity_full_ds DOUBLE[]")
            logger.debug("Migration: Added 'intensity_full_ds' column to chromatograms table")
    except Exception:
        pass
    
    # Migration: Add result columns idempotently; tolerate concurrent migration races.
    conn.execute("ALTER TABLE results ADD COLUMN IF NOT EXISTS rt_aligned BOOLEAN")
    conn.execute("ALTER TABLE results ADD COLUMN IF NOT EXISTS rt_shift DOUBLE")
    conn.execute("ALTER TABLE results ADD COLUMN IF NOT EXISTS peak_mz_of_max DOUBLE")
    
    # Migration: Add EMG peak fitting columns to existing results tables
    fitting_columns = {
        'peak_area_fitted': 'DOUBLE',      # Area under fitted EMG curve
        'peak_sigma': 'DOUBLE',            # Gaussian width (σ)
        'peak_tau': 'DOUBLE',              # Exponential tail decay (τ/gamma)
        'peak_asymmetry': 'DOUBLE',        # τ/σ ratio
        'peak_rt_fitted': 'DOUBLE',        # Peak center from EMG fit
        'fit_r_squared': 'DOUBLE',         # Goodness of fit (R²)
        'fit_success': 'BOOLEAN',          # Whether fitting converged
    }
    for col_name, col_type in fitting_columns.items():
        conn.execute(f"ALTER TABLE results ADD COLUMN IF NOT EXISTS {col_name} {col_type}")



def _create_workspace_tables(conn: duckdb.DuckDBPyConnection):
    conn.execute("""
                 CREATE TABLE IF NOT EXISTS workspaces
                 (
                     key           UUID DEFAULT uuidv4() PRIMARY KEY,
                     name          VARCHAR,
                     description   VARCHAR,
                     active        BOOLEAN,
                     created_at    TIMESTAMP,
                     last_activity TIMESTAMP
                 )
                 """
                 )


def build_where_and_params(filter_, filterOptions):
    where_sql, params = [], []

    if not isinstance(filter_, dict) or not filter_:
        return "", []

    for key, value in filter_.items():
        if not value:
            continue
        # keyword (ILIKE)
        if filterOptions[key].get('filterMode') == 'keyword':
            where_sql.append(f'"{key}" ILIKE ?')
            params.append(f"%{value[0]}%")
        # multiple selection (IN)
        else:
            ph = ",".join("?" for _ in value)
            where_sql.append(f'"{key}" IN ({ph})')
            params.extend(value)
    where_clause = f"WHERE {' AND '.join(where_sql)}" if where_sql else ""
    return where_clause, params


def build_order_by(
        sorter: dict | None,
        column_types: dict[str, str],
        *,
        tie: tuple[str, str] | None = None,  # e.g. ("id", "ASC"); used ONLY when a sorter is present
        nocase_text: bool = True
) -> str:
    """
    Returns 'ORDER BY ...' or '' if there is no valid sorter.
    - sorter: {'columns': [...], 'orders': ['ascend'|'descend', ...]}
    - column_types: map {col -> DUCKDB type} (from DESCRIBE)
    - tie: optional (col, dir); added ONLY when there is at least one sortable column in sorter
    """
    # 0) Normalize input
    cols_in = (sorter or {}).get("columns") or []
    ords_in = (sorter or {}).get("orders") or []
    if not cols_in:
        return ""  # no sorter => no ORDER BY

    # Fill missing order entries
    if len(ords_in) < len(cols_in):
        ords_in = ords_in + ["ascend"] * (len(cols_in) - len(ords_in))

    order_map = {"ascend": "ASC", "descend": "DESC"}
    parts: list[str] = []
    used_cols: set[str] = set()

    for col, ord_ in zip(cols_in, ords_in):
        if col not in column_types:
            continue  # ignore invalid columns
        direction = order_map.get(ord_, "ASC")
        nulls = "NULLS LAST" if direction == "ASC" else "NULLS FIRST"
        ctype = (column_types.get(col) or "").upper()
        is_text = any(t in ctype for t in ("CHAR", "VARCHAR", "TEXT", "STRING"))
        expr = f'"{col}" COLLATE NOCASE' if (nocase_text and is_text) else f'"{col}"'
        parts.append(f"{expr} {direction} {nulls}")
        used_cols.add(col)

    # If nothing valid remains, do not sort (and do not add tie)
    if not parts:
        return ""

    # Add tie ONLY if there is a valid sorter, tie was requested, and the column is not duplicated
    if tie:
        tie_col, tie_dir = tie[0], tie[1].upper()
        if tie_col not in used_cols and tie_col in column_types:
            parts.append(f'"{tie_col}" {tie_dir}')

    return f"ORDER BY {', '.join(parts)}"


def build_paginated_query_by_peak(
        conn,
        filter_: dict | None = None,
        filterOptions: dict | None = None,
        sorter: dict | None = None,
        limit: int = 10,
        offset: int = 0
) -> tuple[str, list]:
    """
    Build a paginated query grouped by peak_label.
    Uses build_where_and_params() and build_order_by() without modifying them.
    """

    # Get column types
    column_types = {
        row[0]: row[1]
        for row in conn.execute("DESCRIBE results").fetchall()
    }

    # 1. Build WHERE clause and params to filter rows
    where_sql, where_params = build_where_and_params(filter_, filterOptions or {})

    # 2. Build ORDER BY for individual rows
    order_by_sql = build_order_by(
        sorter,
        column_types,
        tie=("peak_label", "ASC"),
        nocase_text=True
    )

    # 3. Extract ordering columns to aggregate by peak_label
    agg_exprs = []
    order_exprs = []

    if order_by_sql:
        # Parse ORDER BY to extract columns
        order_part = order_by_sql.replace("ORDER BY", "").strip()
        for clause in order_part.split(","):
            clause = clause.strip()
            # Remove modifiers
            clean = clause.replace("COLLATE NOCASE", "").replace("NULLS LAST", "").replace("NULLS FIRST", "")
            parts = clean.split()
            if not parts:
                continue

            col = parts[0].strip('"')
            direction = parts[1] if len(parts) > 1 else "ASC"

            if col == "peak_label":
                agg_exprs.append(f"peak_label AS _ord_{col}")
                order_exprs.append(f"_ord_{col} {direction}")
            else:
                # For other columns, use MAX/MIN depending on direction
                ctype = (column_types.get(col) or "").upper()
                is_numeric = any(t in ctype for t in ("INT", "DOUBLE", "FLOAT", "DECIMAL", "NUMERIC", "REAL"))

                if is_numeric:
                    # For numeric: MAX if DESC, MIN if ASC (representative value)
                    agg_func = "MAX" if "DESC" in direction else "MIN"
                    agg_exprs.append(f'{agg_func}("{col}") AS _ord_{col}')
                    order_exprs.append(f"_ord_{col} {direction}")
                else:
                    # For text: MAX/MIN depending on direction
                    agg_func = "MAX" if "DESC" in direction else "MIN"
                    agg_exprs.append(f'{agg_func}("{col}") AS _ord_{col}')
                    order_exprs.append(f"_ord_{col} {direction}")

    # If there is no ordering, use peak_label by default
    if not agg_exprs:
        agg_exprs.append("peak_label AS _ord_peak_label")
        order_exprs.append("_ord_peak_label ASC")

    # 4. Build the query with CTEs
    sql = f"""
    WITH filtered AS (
      SELECT *
      FROM results
      {where_sql}
    ),
    peak_ordering AS (
      SELECT 
        peak_label,
        {', '.join(agg_exprs)},
        ROW_NUMBER() OVER(ORDER BY {', '.join(order_exprs)}) AS _rn
      FROM filtered
      GROUP BY peak_label
    ),
    total_peaks AS (
      SELECT COUNT(*) AS __total__
      FROM peak_ordering
    ),
    paged_peaks AS (
      SELECT peak_label, _rn
      FROM peak_ordering
      WHERE _rn > ? AND _rn <= ? + ?
    ),
    paged AS (
      SELECT 
        f.*,
        pp._rn AS __peak_order__,
        (SELECT __total__ FROM total_peaks) AS __total__
      FROM filtered f
      JOIN paged_peaks pp ON f.peak_label = pp.peak_label
      ORDER BY pp._rn, f.ms_file_label
    )
    SELECT * FROM paged;
    """

    # 5. Combine parameters: first WHERE params, then pagination
    all_params = where_params + [offset, offset, limit]

    return sql, all_params


def compute_and_insert_chromatograms_from_ms_data(con: duckdb.DuckDBPyConnection,
                                                  set_progress=None,
                                                  for_optimization=True,
                                                  recompute_ms1=False,
                                                  recompute_ms2=False):
    """
    Computes chromatograms from raw MS data and inserts them into the 'chromatograms' table.

    :param con: An active DuckDB connection.
    :param set_progress: Optional callback function to report progress (0-100).
    :param recompute_ms1: If True, deletes existing MS1 chromatograms before recomputing.
    :param recompute_ms2: If True, deletes existing MS2 chromatograms before recomputing.
    """

    info = con.execute("""
                       WITH samples_to_use AS (SELECT DISTINCT ms_file_label
                                               FROM samples
                                               WHERE use_for_optimization = TRUE
                                                  OR use_for_processing = TRUE),
                            ms1_targets AS (SELECT DISTINCT t.peak_label, s.ms_file_label
                                            FROM targets t
                                                     CROSS JOIN samples_to_use s
                                            WHERE t.mz_mean IS NOT NULL
                                                AND t.mz_width IS NOT NULL
                                                AND (t.peak_selection IS TRUE
                                                   OR NOT EXISTS (SELECT 1
                                                                  FROM targets t1
                                                                  WHERE t1.peak_selection IS TRUE))
                                                AND
                                                  EXISTS(SELECT 1 FROM ms1_data md WHERE md.ms_file_label = s.ms_file_label)),
                            ms2_targets AS (SELECT DISTINCT t.peak_label, s.ms_file_label
                                            FROM targets t
                                                     CROSS JOIN samples_to_use s
                                            WHERE t.filterLine IS NOT NULL -- ensures this is MS2
                                                AND (t.peak_selection IS TRUE
                                                   OR NOT EXISTS (SELECT 1
                                                                  FROM targets t1
                                                                  WHERE t1.peak_selection IS TRUE))
                                                AND
                                                  EXISTS(SELECT 1 FROM ms2_data md WHERE md.ms_file_label = s.ms_file_label)),
                            existing_chromatograms AS (SELECT DISTINCT peak_label, ms_file_label
                                                       FROM chromatograms)
                       SELECT
                           -- MS1 info
                           (SELECT COUNT(*) FROM ms1_targets)                          AS ms1_total_pairs,
                           (SELECT COUNT(*)
                            FROM ms1_targets mt
                                     JOIN existing_chromatograms ec
                                          ON ec.peak_label = mt.peak_label
                                              AND ec.ms_file_label = mt.ms_file_label) AS ms1_existing_pairs,
                           (SELECT COUNT(*)
                            FROM ms1_targets mt
                                     LEFT JOIN existing_chromatograms ec
                                               ON ec.peak_label = mt.peak_label
                                                   AND ec.ms_file_label = mt.ms_file_label
                            WHERE ec.peak_label IS NULL)                               AS ms1_missing_pairs,

                           -- MS2 info
                           (SELECT COUNT(*) FROM ms2_targets)                          AS ms2_total_pairs,
                           (SELECT COUNT(*)
                            FROM ms2_targets mt
                                     JOIN existing_chromatograms ec
                                          ON ec.peak_label = mt.peak_label
                                              AND ec.ms_file_label = mt.ms_file_label) AS ms2_existing_pairs,
                           (SELECT COUNT(*)
                            FROM ms2_targets mt
                                     LEFT JOIN existing_chromatograms ec
                                               ON ec.peak_label = mt.peak_label
                                                   AND ec.ms_file_label = mt.ms_file_label
                            WHERE ec.peak_label IS NULL)                               AS ms2_missing_pairs
                       """).fetchone()

    (ms1_total, ms1_existing, ms1_missing,
     ms2_total, ms2_existing, ms2_missing) = info

    # Decide what to process
    process_ms1 = ms1_total > 0 and (recompute_ms1 or ms1_missing > 0)
    process_ms2 = ms2_total > 0 and (recompute_ms2 or ms2_missing > 0)

    # Informational logging
    if ms1_total > 0:
        logger.info(f"MS1: {ms1_existing} existing, {ms1_missing} missing (total: {ms1_total})")
    if ms2_total > 0:
        logger.info(f"MS2: {ms2_existing} existing, {ms2_missing} missing (total: {ms2_total})")

    if not process_ms1 and not process_ms2:
        logger.info("No chromatograms to process.")
        return

    # Remove existing chromatograms if recomputation is requested
    if recompute_ms1 and ms1_existing > 0:
        logger.info(f"Deleting {ms1_existing} existing MS1 chromatograms for recalculation...")
        con.execute("""
                    DELETE
                    FROM chromatograms
                    WHERE EXISTS(SELECT 1
                                 FROM targets t
                                          CROSS JOIN samples s
                                 WHERE s.use_for_optimization = TRUE
                                    OR s.use_for_processing = TRUE
                                     AND t.mz_mean IS NOT NULL
                                     AND t.mz_width IS NOT NULL
                                     AND (t.peak_selection IS TRUE
                                        OR NOT EXISTS (SELECT 1
                                                       FROM targets t1
                                                       WHERE t1.peak_selection IS TRUE))
                                     AND chromatograms.peak_label = t.peak_label
                                     AND chromatograms.ms_file_label = s.ms_file_label)
                    """)
        ms1_to_compute = ms1_total
    else:
        ms1_to_compute = ms1_missing

    if recompute_ms2 and ms2_existing > 0:
        logger.info(f"Deleting {ms2_existing} existing MS2 chromatograms for recalculation...")
        con.execute("""
                    DELETE
                    FROM chromatograms
                    WHERE EXISTS(SELECT 1
                                 FROM targets t
                                          CROSS JOIN samples s
                                 WHERE s.use_for_optimization = TRUE
                                    OR s.use_for_processing = TRUE
                                     AND t.filterLine IS NOT NULL
                                     AND (t.peak_selection IS TRUE
                                        OR NOT EXISTS (SELECT 1
                                                       FROM targets t1
                                                       WHERE t1.peak_selection IS TRUE))
                                     AND chromatograms.peak_label = t.peak_label
                                     AND chromatograms.ms_file_label = s.ms_file_label)
                    """)
        ms2_to_compute = ms2_total
    else:
        ms2_to_compute = ms2_missing

    # Compute weights for progress reporting
    total_to_compute = ms1_to_compute + ms2_to_compute
    ms1_weight = ms1_to_compute / total_to_compute if process_ms1 else 0
    ms2_weight = ms2_to_compute / total_to_compute if process_ms2 else 0

    logger.info(f"Computing {ms1_to_compute} MS1 and {ms2_to_compute} MS2 chromatograms...")

    query_ms1 = """
                INSERT INTO chromatograms (peak_label, ms_file_label, scan_time, intensity, ms_type)
                WITH pairs_to_process AS (SELECT t.peak_label,
                                                 t.mz_mean,
                                                 t.mz_width,
                                                 CASE WHEN t.rt_unit = 'min' THEN t.rt_min * 60 ELSE t.rt_min END AS rt_min,
                                                 CASE WHEN t.rt_unit = 'min' THEN t.rt_max * 60 ELSE t.rt_max END AS rt_max,
                                                 s.ms_file_label
                                          FROM targets t
                                                   JOIN samples s
                                                        ON (CASE WHEN ? THEN s.use_for_optimization ELSE s.use_for_processing END) =
                                                           TRUE
                                          WHERE t.mz_mean IS NOT NULL
                                              AND t.mz_width IS NOT NULL
                                              AND (t.peak_selection IS TRUE
                                                 OR NOT EXISTS (SELECT 1
                                                                FROM targets t1
                                                                WHERE t1.peak_selection IS TRUE))
                                              AND (
                                                    ? -- recompute_ms1
                                                        OR NOT EXISTS (SELECT 1
                                                                       FROM chromatograms c
                                                                       WHERE c.peak_label = t.peak_label
                                                                         AND c.ms_file_label = s.ms_file_label)
                                                    )),
                     filtered AS (SELECT p.peak_label,
                                         p.ms_file_label,
                                         ROUND(ms1.scan_time, 3) AS scan_time,
                                         ROUND(ms1.intensity, 0) AS intensity
                                  FROM pairs_to_process p
                                           JOIN ms1_data ms1
                                                ON ms1.ms_file_label = p.ms_file_label
                                                    AND ms1.mz BETWEEN p.mz_mean - (p.mz_mean * p.mz_width / 1e6)
                                                       AND p.mz_mean + (p.mz_mean * p.mz_width / 1e6)
                                                    -- RT filter: only extract ±30s around target window
                                                    AND ms1.scan_time BETWEEN COALESCE(p.rt_min, 0) - 30 
                                                                          AND COALESCE(p.rt_max, 999999) + 30
                                  QUALIFY
                                        ROW_NUMBER() OVER (
                                          PARTITION BY p.peak_label, p.ms_file_label, ROUND(ms1.scan_time, 2)
                                            ORDER BY ms1.intensity DESC
                                        ) = 1),
                     agg AS (SELECT peak_label,
                                    ms_file_label,
                                    LIST(scan_time ORDER BY scan_time) AS scan_time,
                                    LIST(intensity ORDER BY scan_time) AS intensity
                             FROM filtered
                             GROUP BY peak_label, ms_file_label)
                SELECT peak_label, ms_file_label, scan_time, intensity, 'ms1' AS ms_type
                FROM agg;
                """

    query_ms2 = """
                INSERT INTO chromatograms (peak_label, ms_file_label, scan_time, intensity, ms_type)
                WITH pairs_to_process AS (SELECT t.peak_label,
                                                 t.filterLine,
                                                 CASE WHEN t.rt_unit = 'min' THEN t.rt_min * 60 ELSE t.rt_min END AS rt_min,
                                                 CASE WHEN t.rt_unit = 'min' THEN t.rt_max * 60 ELSE t.rt_max END AS rt_max,
                                                 s.ms_file_label
                                          FROM targets AS t
                                                   JOIN samples s
                                                        ON (CASE WHEN ? THEN s.use_for_optimization ELSE s.use_for_processing END) =
                                                           TRUE
                                          WHERE t.filterLine IS NOT NULL
                                              AND (t.peak_selection IS TRUE
                                                 OR NOT EXISTS (SELECT 1
                                                                FROM targets t1
                                                                WHERE t1.peak_selection IS TRUE))
                                              AND (
                                                    ? -- recompute_ms2
                                                        OR NOT EXISTS (SELECT 1
                                                                       FROM chromatograms c
                                                                       WHERE c.peak_label = t.peak_label
                                                                         AND c.ms_file_label = s.ms_file_label)
                                                    )),
                     pre AS (SELECT p.peak_label,
                                    p.ms_file_label,
                                    ROUND(ms2.scan_time, 3) AS scan_time,
                                    ROUND(ms2.intensity, 0) AS intensity
                             -- ms2.mz
                             FROM pairs_to_process p
                                      JOIN ms2_data ms2
                                           ON ms2.ms_file_label = p.ms_file_label
                                               AND ms2.filterLine = p.filterLine
                                               -- RT filter: only extract ±30s around target window
                                               AND ms2.scan_time BETWEEN COALESCE(p.rt_min, 0) - 30 
                                                                     AND COALESCE(p.rt_max, 999999) + 30),
                     grouped AS (SELECT peak_label,
                                        ms_file_label,
                                        scan_time,
                                        MAX(intensity) AS intensity -- max per time bin
                                 -- AVG(mz_mean)   AS mz_mean    -- stable within the bin
                                 FROM pre
                                 GROUP BY peak_label, ms_file_label, scan_time),
                     aggregated_chromatograms AS (SELECT peak_label,
                                                         ms_file_label,
                                                         LIST(scan_time ORDER BY scan_time) AS scan_time,
                                                         LIST(intensity ORDER BY scan_time) AS intensity
                                                  -- list(mz ORDER BY scan_time) AS mz
                                                  FROM grouped
                                                  GROUP BY peak_label, ms_file_label)
                SELECT peak_label, ms_file_label, scan_time, intensity, 'ms2' AS ms_type
                FROM aggregated_chromatograms;
                """

    # Shared variable to accumulate progress
    accumulated_progress = [0.0]
    stop_monitoring = [False]
    current_query_type = ['ms1']  # Track which query is running

    def monitor_progress():
        """Monitor progress of the current query"""
        while not stop_monitoring[0]:
            try:
                qp = con.query_progress()
                if qp != -1 and qp > 0:
                    # Total progress depends on which query is executing
                    if current_query_type[0] == 'ms1':
                        total_progress = qp * ms1_weight
                    else:  # ms2
                        total_progress = (ms1_weight * 100) + (qp * ms2_weight)

                    accumulated_progress[0] = total_progress
                    if set_progress:
                        set_progress(round(total_progress, 1))
                    logger.info(f"Progress: {total_progress:.1f}%")

                time.sleep(0.05)

            except (duckdb.InvalidInputException, duckdb.ConnectionException):
                break
            except Exception as e:
                logger.error(f"Progress monitoring error: {e}")
                break

    # Start monitoring
    if set_progress:
        progress_thread = Thread(target=monitor_progress, daemon=True)
        progress_thread.start()

    try:
        # Run MS1
        if process_ms1:
            logger.info("Processing MS1 chromatograms...")
            current_query_type[0] = 'ms1'
            con.execute(query_ms1, [for_optimization, recompute_ms1])
            accumulated_progress[0] = ms1_weight * 100
            if set_progress:
                set_progress(round(accumulated_progress[0], 1))

        # Run MS2
        if process_ms2:
            logger.info("Processing MS2 chromatograms...")
            current_query_type[0] = 'ms2'
            con.execute(query_ms2, [for_optimization, recompute_ms2])
            accumulated_progress[0] = 100.0
            if set_progress:
                set_progress(100.0)

        logger.info("Chromatograms computed and inserted into DuckDB.")

    finally:
        stop_monitoring[0] = True
        if set_progress:
            progress_thread.join(timeout=0.5)

    logger.info("Chromatograms computed and inserted into DuckDB.")




def compute_chromatograms_in_batches(wdir: str,
                                     # conn: duckdb.DuckDBPyConnection,
                                     use_for_optimization: bool,
                                     batch_size: int = None,
                                     checkpoint_every: int = 10,
                                     set_progress=None,
                                     recompute_ms1=False,
                                     recompute_ms2=False,
                                     n_cpus=None,
                                     ram=None,
                                     use_bookmarked: bool = False,
                                     page_load_id: str | None = None,
                                     ):

    logger.info(f"Computing chromatograms in batches. wDir: {wdir}")
    ensure_page_load_active(wdir, page_load_id, where="compute_chromatograms_in_batches:start")
    QUERY_CREATE_SCAN_LOOKUP = """
                               CREATE TABLE IF NOT EXISTS ms_file_scans AS
                               SELECT DISTINCT ms_file_label,
                                      scan_id,
                                      scan_time,
                                      'ms1' AS ms_type
                               FROM ms1_data
                                   UNION ALL
                               SELECT DISTINCT ms_file_label,
                                      scan_id,
                                      scan_time,
                                      'ms2' AS ms_type
                               FROM ms2_data
                               ORDER BY ms_file_label, scan_id, ms_type;

                               CREATE INDEX IF NOT EXISTS idx_ms_file_scans_file
                                   ON ms_file_scans (ms_file_label);

                               CREATE INDEX IF NOT EXISTS idx_ms_file_scans_file_scan
                                   ON ms_file_scans (ms_file_label, scan_id, ms_type);
                               """

    QUERY_CREATE_PENDING_PAIRS = """
                                 CREATE TABLE IF NOT EXISTS pending_pairs AS
                                 WITH target_filter AS (SELECT peak_label,
                                                               ms_type,
                                                               mz_mean,
                                                               mz_width,
                                                               filterLine,
                                                               rt_min,
                                                               rt_max,
                                                               rt_unit,
                                                               bookmark
                                                        FROM targets t
                                                        WHERE (
                                                            t.peak_selection IS TRUE
                                                                OR NOT EXISTS (SELECT 1
                                                                               FROM targets t1
                                                                               WHERE t1.peak_selection IS TRUE
                                                                                 AND t1.ms_type = t.ms_type)
                                                            )
                                                          AND (
                                                            CASE
                                                                WHEN ?
                                                                    THEN t.bookmark IS TRUE -- use_bookmarked = True → solo marcados
                                                                ELSE TRUE -- use_bookmarked = False → no filtra
                                                                END
                                                            )),
                                      sample_filter AS (SELECT ms_file_label
                                                        FROM samples
                                                        WHERE (CASE WHEN ? THEN use_for_optimization ELSE use_for_processing END) = TRUE),
                                      existing_pairs AS (SELECT DISTINCT peak_label,
                                                                         ms_file_label,
                                                                         ms_type
                                                         FROM chromatograms),
                                      all_possible_pairs AS (SELECT t.peak_label,
                                                                    s.ms_file_label,
                                                                    t.ms_type,
                                                                    t.mz_mean,
                                                                    t.mz_width,
                                                                    t.filterLine,
                                                                    CASE WHEN t.rt_unit = 'min' THEN t.rt_min * 60 ELSE t.rt_min END AS rt_min,
                                                                    CASE WHEN t.rt_unit = 'min' THEN t.rt_max * 60 ELSE t.rt_max END AS rt_max
                                                             FROM target_filter t
                                                                      CROSS JOIN sample_filter s),
                                      pending AS (SELECT a.peak_label,
                                                         a.ms_file_label,
                                                         a.ms_type,
                                                         a.mz_mean,
                                                         a.mz_width,
                                                         a.filterLine,
                                                         a.rt_min,
                                                         a.rt_max,
                                                         ROW_NUMBER() OVER () AS pair_id
                                                  FROM all_possible_pairs a
                                                           LEFT JOIN existing_pairs e
                                                                     ON a.peak_label = e.peak_label
                                                                         AND a.ms_file_label = e.ms_file_label
                                                                         AND a.ms_type = e.ms_type
                                                  WHERE e.peak_label IS NULL)
                                 SELECT pair_id,
                                        peak_label,
                                        ms_file_label,
                                        ms_type,
                                        mz_mean,
                                        mz_width,
                                        filterLine,
                                        rt_min,
                                        rt_max
                                 FROM pending
                                 ORDER BY pair_id;
                                 """

    QUERY_PROCESS_BATCH_MS1 = """
                              INSERT INTO chromatograms (peak_label, ms_file_label, scan_time, intensity, mz_arr, ms_type)
                              WITH batch_pairs AS (SELECT peak_label, ms_file_label, ms_type, mz_mean, mz_width, rt_min, rt_max
                                                   FROM pending_pairs
                                                   WHERE ms_type = 'ms1'
                                                     AND pair_id BETWEEN ? AND ?),
                                   -- Step 1: Find max intensity and corresponding mz per scan
                                   matched_with_mz AS (
                                       SELECT bp.peak_label,
                                              bp.ms_file_label,
                                              ms1.scan_id,
                                              ms1.intensity,
                                              ms1.mz,
                                              ROW_NUMBER() OVER (
                                                  PARTITION BY bp.peak_label, bp.ms_file_label, ms1.scan_id
                                                  ORDER BY ms1.intensity DESC
                                              ) AS rn
                                       FROM batch_pairs bp
                                       JOIN ms1_data ms1
                                            ON ms1.ms_file_label = bp.ms_file_label
                                                AND ms1.mz BETWEEN
                                                   bp.mz_mean - (bp.mz_mean * bp.mz_width / 1e6)
                                                   AND
                                                   bp.mz_mean + (bp.mz_mean * bp.mz_width / 1e6)
                                   ),
                                   matched_intensities AS (
                                       SELECT peak_label, ms_file_label, scan_id, intensity, mz
                                       FROM matched_with_mz
                                       WHERE rn = 1
                                   ),
                                   -- Step 2: Expand to scans within RT window (±30s margin)
                                   all_scans_needed AS (SELECT DISTINCT bp.peak_label,
                                                                        bp.ms_file_label,
                                                                        s.scan_id,
                                                                        s.scan_time
                                                        FROM batch_pairs bp
                                                                 JOIN ms_file_scans s ON s.ms_file_label = bp.ms_file_label
                                                                     -- RT filter: only include scans ±30s around target window
                                                                     AND s.scan_time BETWEEN COALESCE(bp.rt_min, 0) - 30 
                                                                                         AND COALESCE(bp.rt_max, 999999) + 30),
                                   -- Step 3: LEFT JOIN (both tables are small)
                                   complete_data AS (SELECT a.peak_label,
                                                            a.ms_file_label,
                                                            a.scan_time,
                                                            a.scan_id,
                                                            COALESCE(ROUND(m.intensity, 0), 1) AS intensity,
                                                            m.mz AS mz_val
                                                     FROM all_scans_needed a
                                                              LEFT JOIN matched_intensities m
                                                                        ON a.peak_label = m.peak_label
                                                                            AND a.ms_file_label = m.ms_file_label
                                                                            AND a.scan_id = m.scan_id),
                                   agg AS (SELECT peak_label,
                                                  ms_file_label,
                                                  LIST(scan_time ORDER BY scan_time) AS scan_time,
                                                  LIST(intensity ORDER BY scan_time) AS intensity,
                                                  LIST(mz_val ORDER BY scan_time) AS mz_arr
                                           FROM complete_data
                                           GROUP BY peak_label, ms_file_label)
                              SELECT peak_label, ms_file_label, scan_time, intensity, mz_arr, 'ms1' AS ms_type
                              FROM agg;
                              """

    QUERY_PROCESS_BATCH_MS2 = """
                              INSERT INTO chromatograms (peak_label, ms_file_label, scan_time, intensity, ms_type)
                              WITH batch_pairs AS (SELECT peak_label, ms_file_label, ms_type, filterLine, rt_min, rt_max
                                                   FROM pending_pairs
                                                   WHERE ms_type = 'ms2'
                                                     AND pair_id BETWEEN ? AND ?),
                                   -- Step 1: Find intensities (only rows with signal)
                                   matched_filterline AS (SELECT bp.peak_label,
                                                      bp.ms_file_label,
                                                      ms2.scan_id,
                                                      ms2.intensity
                                               FROM batch_pairs bp
                                                        JOIN ms2_data ms2
                                                             ON ms2.ms_file_label = bp.ms_file_label
                                                                 AND ms2.filterLine = bp.filterLine),
                                  -- Step 2: Expand to scans within RT window (±30s margin)
                                   all_scans_needed AS (SELECT DISTINCT bp.peak_label,
                                                                        bp.ms_file_label,
                                                                        s.scan_id,
                                                                        s.scan_time
                                                        FROM batch_pairs bp
                                                                 JOIN ms_file_scans s ON s.ms_file_label = bp.ms_file_label
                                                                     -- RT filter: only include scans ±30s around target window
                                                                     AND s.scan_time BETWEEN COALESCE(bp.rt_min, 0) - 30 
                                                                                         AND COALESCE(bp.rt_max, 999999) + 30),
                                  -- Step 3: LEFT JOIN (both tables are small)
                                   complete_data AS (SELECT a.peak_label,
                                                            a.ms_file_label,
                                                            a.scan_time,
                                                            a.scan_id,
                                                            a.scan_time,
                                                            COALESCE(ROUND(m.intensity, 0), 1) AS intensity
                                                     FROM all_scans_needed a
                                                              LEFT JOIN matched_filterline m
                                                                        ON a.peak_label = m.peak_label
                                                                            AND a.ms_file_label = m.ms_file_label
                                                                            AND a.scan_id = m.scan_id),
                                   -- Step 2: Aggregate after expanding scans
                                   agg AS (SELECT peak_label,
                                                  ms_file_label,
                                                  LIST(scan_time ORDER BY scan_time) AS scan_time,
                                                  LIST(intensity ORDER BY scan_time) AS intensity
                                           FROM complete_data
                                           GROUP BY peak_label, ms_file_label)
                              SELECT peak_label,
                                     ms_file_label,
                                     scan_time,
                                     intensity,
                                     'ms2' AS ms_type
                              FROM agg;
                              """

    if recompute_ms1:
        ensure_page_load_active(wdir, page_load_id, where="compute_chromatograms_in_batches:recompute_ms1")
        logger.info("Deleting existing MS1 chromatograms for recalculation...")
        with duckdb_connection(wdir, n_cpus=n_cpus, ram=ram, page_load_id=page_load_id) as con:
            con.execute("DELETE FROM chromatograms WHERE ms_type = 'ms1'")
    if recompute_ms2:
        ensure_page_load_active(wdir, page_load_id, where="compute_chromatograms_in_batches:recompute_ms2")
        logger.info("Deleting existing MS2 chromatograms for recalculation...")
        with duckdb_connection(wdir, n_cpus=n_cpus, ram=ram, page_load_id=page_load_id) as con:
            con.execute("DELETE FROM chromatograms WHERE ms_type = 'ms2'")

    ensure_page_load_active(wdir, page_load_id, where="compute_chromatograms_in_batches:scan_lookup")
    with duckdb_connection(wdir, n_cpus=n_cpus, ram=ram, page_load_id=page_load_id) as conn:
        # Ensure clean database state before processing (clears any accumulated WAL)
        logger.info("Running CHECKPOINT to ensure clean database state...")
        conn.execute("CHECKPOINT")
        
        conn.execute("DROP TABLE IF EXISTS pending_pairs")
        try:
            count = conn.execute("SELECT COUNT(*) FROM ms_file_scans").fetchone()[0]
            logger.info(f"Lookup table exists ({count:,} entries)")
        except:
            logger.warning("Lookup table does not exist. Creating...")

            start = time.perf_counter()
            conn.execute(QUERY_CREATE_SCAN_LOOKUP)
            elapsed = time.perf_counter() - start

            result = conn.execute("""
                                  SELECT COUNT(*)                      as total_entries,
                                         COUNT(DISTINCT ms_file_label) as total_files,
                                         AVG(scans_per_file)           as avg_scans
                                  FROM (SELECT ms_file_label, COUNT(*) as scans_per_file
                                        FROM ms_file_scans
                                        GROUP BY ms_file_label)
                                  """).fetchone()
            logger.info(f"  Total entries: {result[0]:,}")
            logger.info(f"  Total MS files: {result[1]:,}")
            logger.info(f"  Average scans per file: {result[2]:.0f}")
            logger.info(f"  Time elapsed: {elapsed:.2f}s")

    ensure_page_load_active(wdir, page_load_id, where="compute_chromatograms_in_batches:pending_pairs")
    with duckdb_connection(wdir, n_cpus=n_cpus, ram=ram, page_load_id=page_load_id) as conn:
        # Ensure clean database state before processing (clears any accumulated WAL)
        logger.info("Running CHECKPOINT to ensure clean database state...")
        conn.execute("CHECKPOINT")
        
        logger.info("Getting pending pairs...")
        start_time = time.time()
        conn.execute(QUERY_CREATE_PENDING_PAIRS, [use_bookmarked, use_for_optimization])
        rows = conn.execute("""
                            SELECT ms_type,
                                   COUNT(*)     AS total,
                                   MIN(pair_id) AS min_id,
                                   MAX(pair_id) AS max_id
                            FROM pending_pairs
                            GROUP BY ms_type
                            ORDER BY ms_type
                            """).fetchall()
        elapsed = time.time() - start_time

        if not rows:
            logger.info(f"No pending pairs ({elapsed:.2f}s)")
            conn.execute("DROP TABLE IF EXISTS pending_pairs")
            return {
                'total_pairs': 0,
                'processed': 0,
                'failed': 0,
                'batches': 0
            }

        global_total_pairs = sum(r[1] for r in rows)
        files_per_type = {}
        if set_progress:
            files_per_type = dict(conn.execute("""
                                               SELECT ms_type,
                                                      COUNT(DISTINCT ms_file_label) AS total_files
                                               FROM pending_pairs
                                               GROUP BY ms_type
                                               """).fetchall())

        _send_progress(
            set_progress,
            0,
            stage="Chromatograms",
            detail=f"Pending pairs: {global_total_pairs:,}",
        )



        logger.info(f"{global_total_pairs:,} pending pairs ({elapsed:.2f}s)")
        
        # Auto-calculate optimal batch size if not explicitly provided
        if batch_size is None:
            ram_gb = ram if ram else 16  # Default to 16GB if not specified
            cpus = n_cpus if n_cpus else 4  # Default to 4 if not specified
            batch_size = calculate_optimal_batch_size(ram_gb, global_total_pairs, cpus)
            logger.info(f"Auto-calculated batch size: {batch_size} (based on {ram_gb}GB RAM, {cpus} CPUs, {global_total_pairs:,} pairs)")
        else:
            logger.info(f"Using specified batch size: {batch_size}")

        global_processed = 0  # accumulated counter
        global_stats: dict[str, dict] = {}

        for ms_type, total_pairs_type, min_id, max_id in rows:
            ensure_page_load_active(wdir, page_load_id, where=f"compute_chromatograms_in_batches:ms_type:{ms_type}")
            logger.info(f"--- Processing {ms_type} ---")
            logger.info(f"Pending pairs: {total_pairs_type:,} (pair_id {min_id}-{max_id})")

            if total_pairs_type == 0 or min_id is None or max_id is None:
                global_stats[ms_type] = {
                    'total_pairs': 0,
                    'processed': 0,
                    'failed': 0,
                    'batches': 0,
                }
                continue

            processed = 0
            failed = 0
            batches = 0
            processed_files: set[str] = set()

            current_id = min_id
            batch_num = 1
            total_batches = (total_pairs_type + batch_size - 1) // batch_size
            batches_since_checkpoint = 0
            total_files_type = files_per_type.get(ms_type, 0)

            with duckdb_connection(wdir, n_cpus=n_cpus, ram=ram, page_load_id=page_load_id) as conn:
                # Process batches in a single connection; checkpoint periodically to avoid WAL stalls
                conn.execute("BEGIN TRANSACTION")

                while current_id <= max_id:
                    ensure_page_load_active(
                        wdir,
                        page_load_id,
                        where=f"compute_chromatograms_in_batches:batch:{ms_type}:{batch_num}",
                    )
                    batch_count = 0
                    start_id = current_id
                    end_id = current_id + batch_size - 1
                    log_line = None

                    try:
                        batch_count = conn.execute("""
                                                   SELECT COUNT(*)
                                                   FROM pending_pairs
                                                   WHERE ms_type = ?
                                                     AND pair_id BETWEEN ? AND ?
                                                   """, [ms_type, start_id, end_id]).fetchone()[0]

                        if batch_count == 0:
                            current_id += batch_size
                            continue



                        batch_start = time.time()

                        if ms_type == 'ms1':
                            conn.execute(QUERY_PROCESS_BATCH_MS1, [start_id, end_id])
                        elif ms_type == 'ms2':
                            conn.execute(QUERY_PROCESS_BATCH_MS2, [start_id, end_id])

                        if set_progress:
                            batch_files = conn.execute("""
                                                       SELECT DISTINCT ms_file_label
                                                       FROM pending_pairs
                                                       WHERE ms_type = ?
                                                         AND pair_id BETWEEN ? AND ?
                                                       """, [ms_type, start_id, end_id]).fetchall()
                            processed_files.update(
                                row[0] for row in batch_files if row and row[0] is not None
                            )

                        batch_elapsed = time.time() - batch_start
                        processed += batch_count
                        batches += 1
                        batches_since_checkpoint += 1

                        logger.info(f"Batch {batch_num:>4}/{total_batches} | "
                                    f"IDs {start_id:>6}-{end_id:>6} | "
                                    f"{batch_count:>3} pairs | "
                                    f"Batch time: {batch_elapsed:>5.2f}s | "
                                    f"Progress {processed:>6,}/{total_pairs_type:,}")
                        log_line = (f"Batch {batch_num}/{total_batches} | "
                                    f"Progress {processed:,}/{total_pairs_type:,} | "
                                    f"Time/batch {batch_elapsed:0.2f}s")


                        if checkpoint_every and batches_since_checkpoint >= checkpoint_every:
                            conn.execute("COMMIT")
                            conn.execute("CHECKPOINT")
                            conn.execute("BEGIN TRANSACTION")
                            batches_since_checkpoint = 0

                        batch_num += 1

                    except Exception as e:
                        if 'Cancelled' in type(e).__name__ or 'PreventUpdate' in type(e).__name__:
                            raise
                        batch_elapsed = time.time() - batch_start if 'batch_start' in locals() else 0
                        failed += batch_count

                        logger.error(f"Error processing batch: {batch_elapsed:>5.2f}s | Error: {str(e)[:80]}")
                        _add_processing_error(wdir, str(e))


                        with open(f'failed_batches_{ms_type}.log', 'a') as f:
                            f.write(f"\n{'=' * 60}\n")
                            f.write(f"Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S')}\n")
                            f.write(f"ms_type: {ms_type}\n")
                            f.write(f"Batch {batch_num}/{total_batches}\n")
                            f.write(f"IDs: {start_id}-{end_id}\n")
                            f.write(f"Error: {str(e)}\n")
                            f.write(f"{'=' * 60}\n")

                        try:
                            conn.execute("ROLLBACK")
                            conn.execute("BEGIN TRANSACTION")
                            batches_since_checkpoint = 0
                        except Exception:
                            pass

                    finally:
                        # Update global progress even if the batch failed
                        if batch_count > 0:
                            global_processed += batch_count
                            progress_pct_global = (global_processed / global_total_pairs) * 100
                            files_done = len(processed_files) if set_progress else 0
                            detail_text = (
                                f"{log_line}"
                                if log_line else
                                f"{ms_type.upper()} batch {batch_num}/{total_batches} | "
                                f"Pairs {processed:,}/{total_pairs_type:,}"
                            )
                            _send_progress(
                                set_progress,
                                round(progress_pct_global, 1),
                                stage="Chromatograms",
                                detail=detail_text,
                            )

                    current_id += batch_size

                # Final checkpoint/commit for this ms_type
                conn.execute("COMMIT")
                conn.execute("CHECKPOINT")

    ensure_page_load_active(wdir, page_load_id, where="compute_chromatograms_in_batches:cleanup")
    with duckdb_connection(wdir, n_cpus=n_cpus, ram=ram, page_load_id=page_load_id) as conn:
        conn.execute("DROP TABLE IF EXISTS ms_file_scans")
        conn.execute("DROP TABLE IF EXISTS pending_pairs")


def populate_full_range_downsampled_chromatograms(wdir: str,
                                                  n_out: int = FULL_RANGE_DOWNSAMPLE_POINTS,
                                                  batch_size: int = FULL_RANGE_DOWNSAMPLE_BATCH,
                                                  set_progress=None,
                                                  n_cpus=None,
                                                  ram=None):
    if _lttbc is None:
        logger.warning("Full-range downsampling skipped: 'lttbc' is not available.")
        _send_progress(set_progress, 100, stage="Downsampling", detail="lttbc not available")
        return

    logger.info("Preparing full-range downsampled chromatograms (MS1 only)...")
    _send_progress(
        set_progress,
        0,
        stage="Downsampling",
        detail="Preparing full-range downsampled chromatograms",
    )

    query_create_scan_lookup = """
        CREATE TABLE IF NOT EXISTS ms_file_scans AS
        SELECT DISTINCT ms_file_label,
               scan_id,
               scan_time,
               'ms1' AS ms_type
        FROM ms1_data
            UNION ALL
        SELECT DISTINCT ms_file_label,
               scan_id,
               scan_time,
               'ms2' AS ms_type
        FROM ms2_data
        ORDER BY ms_file_label, scan_id, ms_type;

        CREATE INDEX IF NOT EXISTS idx_ms_file_scans_file
            ON ms_file_scans (ms_file_label);

        CREATE INDEX IF NOT EXISTS idx_ms_file_scans_file_scan
            ON ms_file_scans (ms_file_label, scan_id, ms_type);
        """

    query_create_pending = """
        CREATE TABLE IF NOT EXISTS pending_full_ds_pairs AS
        SELECT
            ROW_NUMBER() OVER () AS pair_id,
            c.peak_label,
            c.ms_file_label,
            t.mz_mean,
            t.mz_width
        FROM chromatograms c
        JOIN targets t ON c.peak_label = t.peak_label
        WHERE c.ms_type = 'ms1'
          AND c.scan_time_full_ds IS NULL
          AND t.mz_mean IS NOT NULL
          AND t.mz_width IS NOT NULL
        ORDER BY c.ms_file_label, c.peak_label;
        """

    query_batch_full_range = """
        WITH batch_pairs AS (
            SELECT peak_label, ms_file_label, mz_mean, mz_width
            FROM pending_full_ds_pairs
            WHERE pair_id BETWEEN ? AND ?
        ),
        matched_with_mz AS (
            SELECT
                bp.peak_label,
                bp.ms_file_label,
                ms1.scan_id,
                ms1.intensity,
                ROW_NUMBER() OVER (
                    PARTITION BY bp.peak_label, bp.ms_file_label, ms1.scan_id
                    ORDER BY ms1.intensity DESC
                ) AS rn
            FROM batch_pairs bp
            JOIN ms1_data ms1
                ON ms1.ms_file_label = bp.ms_file_label
                AND ms1.mz BETWEEN
                    bp.mz_mean - (bp.mz_mean * bp.mz_width / 1e6)
                    AND bp.mz_mean + (bp.mz_mean * bp.mz_width / 1e6)
        ),
        matched_intensities AS (
            SELECT peak_label, ms_file_label, scan_id, intensity
            FROM matched_with_mz
            WHERE rn = 1
        ),
        all_scans AS (
            SELECT
                bp.peak_label,
                bp.ms_file_label,
                s.scan_id,
                s.scan_time
            FROM batch_pairs bp
            JOIN ms_file_scans s
                ON s.ms_file_label = bp.ms_file_label
                AND s.ms_type = 'ms1'
        ),
        complete_data AS (
            SELECT
                a.peak_label,
                a.ms_file_label,
                a.scan_time,
                COALESCE(ROUND(m.intensity, 0), 1) AS intensity
            FROM all_scans a
            LEFT JOIN matched_intensities m
                ON a.peak_label = m.peak_label
                AND a.ms_file_label = m.ms_file_label
                AND a.scan_id = m.scan_id
        ),
        agg AS (
            SELECT
                peak_label,
                ms_file_label,
                LIST(scan_time ORDER BY scan_time) AS scan_time,
                LIST(intensity ORDER BY scan_time) AS intensity
            FROM complete_data
            GROUP BY peak_label, ms_file_label
        )
        SELECT peak_label, ms_file_label, scan_time, intensity
        FROM agg;
        """

    created_lookup = False
    with duckdb_connection(wdir, n_cpus=n_cpus, ram=ram) as conn:
        try:
            conn.execute("SELECT COUNT(*) FROM ms_file_scans").fetchone()
        except Exception:
            logger.info("Creating scan lookup table for full-range downsampling...")
            conn.execute(query_create_scan_lookup)
            created_lookup = True

        conn.execute("DROP TABLE IF EXISTS pending_full_ds_pairs")
        conn.execute(query_create_pending)

        total_pairs = conn.execute(
            "SELECT COUNT(*) FROM pending_full_ds_pairs"
        ).fetchone()[0]

        if not total_pairs:
            logger.info("No full-range downsampled chromatograms to compute.")
            conn.execute("DROP TABLE IF EXISTS pending_full_ds_pairs")
            if created_lookup:
                conn.execute("DROP TABLE IF EXISTS ms_file_scans")
            _send_progress(set_progress, 100, stage="Downsampling", detail="No pending pairs")
            return

        total_batches = (total_pairs + batch_size - 1) // batch_size
        logger.info(f"Downsampling {total_pairs:,} MS1 chromatograms in {total_batches} batches...")

        for batch_idx in range(total_batches):
            start_id = batch_idx * batch_size + 1
            end_id = min((batch_idx + 1) * batch_size, total_pairs)

            rows = conn.execute(query_batch_full_range, [start_id, end_id]).fetchall()
            updates = []
            for peak_label, ms_file_label, scan_time, intensity in rows:
                if scan_time is None or intensity is None:
                    continue
                smoothed = _apply_savgol_smoothing(intensity)
                down_x, down_y = _apply_lttb_downsampling(scan_time, smoothed, n_out=n_out)
                if len(down_x) > n_out and len(scan_time) > n_out:
                    logger.warning(
                        "Downsampling failed for %s/%s (points=%d, n_out=%d)",
                        peak_label,
                        ms_file_label,
                        len(scan_time),
                        n_out,
                    )
                    continue
                updates.append(
                    (list(down_x), list(down_y), peak_label, ms_file_label)
                )

            if updates:
                conn.executemany(
                    """
                    UPDATE chromatograms
                    SET scan_time_full_ds = ?, intensity_full_ds = ?
                    WHERE peak_label = ? AND ms_file_label = ?
                    """,
                    updates,
                )

            progress_pct = round((end_id / total_pairs) * 100, 1)
            _send_progress(
                set_progress,
                progress_pct,
                stage="Downsampling",
                detail=f"Batch {batch_idx + 1}/{total_batches}",
            )

        conn.execute("DROP TABLE IF EXISTS pending_full_ds_pairs")
        if created_lookup:
            conn.execute("DROP TABLE IF EXISTS ms_file_scans")

    _send_progress(set_progress, 100, stage="Downsampling", detail="Done")
    logger.info("Full-range downsampled chromatograms computed.")


def populate_full_range_downsampled_chromatograms_for_target(wdir: str | None,
                                                             peak_label: str,
                                                             n_out: int = FULL_RANGE_DOWNSAMPLE_POINTS,
                                                             n_cpus=None,
                                                             ram=None,
                                                             conn: duckdb.DuckDBPyConnection | None = None) -> bool:
    if _lttbc is None:
        logger.warning("On-demand downsampling skipped: 'lttbc' is not available.")
        return False

    query_create_scan_lookup = """
        CREATE TABLE IF NOT EXISTS ms_file_scans AS
        SELECT DISTINCT ms_file_label,
               scan_id,
               scan_time,
               'ms1' AS ms_type
        FROM ms1_data
            UNION ALL
        SELECT DISTINCT ms_file_label,
               scan_id,
               scan_time,
               'ms2' AS ms_type
        FROM ms2_data
        ORDER BY ms_file_label, scan_id, ms_type;

        CREATE INDEX IF NOT EXISTS idx_ms_file_scans_file
            ON ms_file_scans (ms_file_label);

        CREATE INDEX IF NOT EXISTS idx_ms_file_scans_file_scan
            ON ms_file_scans (ms_file_label, scan_id, ms_type);
        """

    query_full_range = """
        WITH target AS (
            SELECT ?::DOUBLE AS mz_mean, ?::DOUBLE AS mz_width
        ),
        samples AS (
            SELECT DISTINCT ms_file_label
            FROM chromatograms
            WHERE peak_label = ?
              AND ms_type = 'ms1'
              AND scan_time_full_ds IS NULL
        ),
        matched_with_mz AS (
            SELECT
                s.ms_file_label,
                ms1.scan_id,
                ms1.intensity,
                ROW_NUMBER() OVER (
                    PARTITION BY s.ms_file_label, ms1.scan_id
                    ORDER BY ms1.intensity DESC
                ) AS rn
            FROM samples s
            JOIN ms1_data ms1
                ON ms1.ms_file_label = s.ms_file_label
                AND ms1.mz BETWEEN
                    (SELECT mz_mean FROM target) - ((SELECT mz_mean FROM target) * (SELECT mz_width FROM target) / 1e6)
                    AND (SELECT mz_mean FROM target) + ((SELECT mz_mean FROM target) * (SELECT mz_width FROM target) / 1e6)
        ),
        matched_intensities AS (
            SELECT ms_file_label, scan_id, intensity
            FROM matched_with_mz
            WHERE rn = 1
        ),
        all_scans AS (
            SELECT
                s.ms_file_label,
                sc.scan_id,
                sc.scan_time
            FROM samples s
            JOIN ms_file_scans sc
                ON sc.ms_file_label = s.ms_file_label
                AND sc.ms_type = 'ms1'
        ),
        complete_data AS (
            SELECT
                a.ms_file_label,
                a.scan_time,
                COALESCE(ROUND(m.intensity, 0), 1) AS intensity
            FROM all_scans a
            LEFT JOIN matched_intensities m
                ON a.ms_file_label = m.ms_file_label
                AND a.scan_id = m.scan_id
        ),
        agg AS (
            SELECT
                ms_file_label,
                LIST(scan_time ORDER BY scan_time) AS scan_time,
                LIST(intensity ORDER BY scan_time) AS intensity
            FROM complete_data
            GROUP BY ms_file_label
        )
        SELECT ms_file_label, scan_time, intensity
        FROM agg;
        """

    started_at = time.perf_counter()
    connection_ctx = None
    if conn is None:
        if wdir is None:
            return False
        connection_ctx = duckdb_connection(wdir, n_cpus=n_cpus, ram=ram)
        conn = connection_ctx.__enter__()
        if conn is None:
            return False
    try:
        target = conn.execute(
            """
            SELECT mz_mean, mz_width
            FROM targets
            WHERE peak_label = ?
              AND mz_mean IS NOT NULL
              AND mz_width IS NOT NULL
            """,
            [peak_label],
        ).fetchone()
        if not target:
            return False

        missing = conn.execute(
            """
            SELECT COUNT(*)
            FROM chromatograms
            WHERE peak_label = ?
              AND ms_type = 'ms1'
              AND scan_time_full_ds IS NULL
            """,
            [peak_label],
        ).fetchone()[0]
        if not missing:
            return True

        created_lookup = False
        with _SCAN_LOOKUP_LOCK:
            try:
                exists = conn.execute(
                    """
                    SELECT COUNT(*)
                    FROM information_schema.tables
                    WHERE table_name = 'ms_file_scans'
                      AND table_schema = 'main'
                    """
                ).fetchone()[0]
            except Exception:
                exists = 0

            if not exists:
                lookup_start = time.perf_counter()
                for attempt in range(3):
                    try:
                        conn.execute(query_create_scan_lookup)
                        created_lookup = True
                        logger.info(
                            "Created ms_file_scans lookup table in %.2fs",
                            time.perf_counter() - lookup_start,
                        )
                        break
                    except duckdb.TransactionException as exc:
                        # Another connection is creating the table; wait and retry.
                        if "Catalog write-write conflict" in str(exc) and attempt < 2:
                            time.sleep(0.1 * (attempt + 1))
                            continue
                        raise

            # Ensure table is visible before continuing.
            conn.execute("SELECT COUNT(*) FROM ms_file_scans").fetchone()

        logger.info("On-demand downsampling for target %s (%d chromatograms)", peak_label, missing)
        query_start = time.perf_counter()
        rows = conn.execute(query_full_range, [target[0], target[1], peak_label]).fetchall()
        logger.info(
            "Fetched %d full-range chromatogram rows for target %s in %.2fs",
            len(rows),
            peak_label,
            time.perf_counter() - query_start,
        )
        updates = []
        downsample_start = time.perf_counter()
        for ms_file_label, scan_time, intensity in rows:
            if scan_time is None or intensity is None:
                continue
            smoothed = _apply_savgol_smoothing(intensity)
            down_x, down_y = _apply_lttb_downsampling(scan_time, smoothed, n_out=n_out)
            updates.append((list(down_x), list(down_y), peak_label, ms_file_label))
        logger.info(
            "Downsampled %d chromatograms for target %s in %.2fs",
            len(updates),
            peak_label,
            time.perf_counter() - downsample_start,
        )

        if updates:
            update_start = time.perf_counter()
            conn.executemany(
                """
                UPDATE chromatograms
                SET scan_time_full_ds = ?, intensity_full_ds = ?
                WHERE peak_label = ? AND ms_file_label = ?
                """,
                updates,
            )
            logger.info(
                "Persisted %d full-range chromatograms for target %s in %.2fs",
                len(updates),
                peak_label,
                time.perf_counter() - update_start,
            )

        # Keep ms_file_scans persisted for future targets/toggles. Rebuilding this
        # lookup repeatedly is costly and is a major source of first-toggle latency.
        if created_lookup:
            logger.info("Keeping ms_file_scans lookup table cached for reuse.")
    finally:
        if connection_ctx is not None:
            connection_ctx.__exit__(None, None, None)

    logger.info(
        "On-demand full-range downsampling finished for target %s in %.2fs",
        peak_label,
        time.perf_counter() - started_at,
    )
    return True


def compute_results_in_batches(wdir: str,
                               use_bookmarked: bool = False,
                               recompute: bool = False,
                               batch_size: int = None,
                               checkpoint_every: int = 20,
                               set_progress=None,
                               n_cpus=None,
                               ram=None,
                               page_load_id: str | None = None):
    """
    Compute results with efficient macros.
    include_arrays=False: numeric metrics only (FAST)
    include_arrays=True: include scan_time and intensity arrays (SLOWER)
    """


    ensure_page_load_active(wdir, page_load_id, where="compute_results_in_batches:start")

    # OPTIMIZED macro using list functions - avoids UNNEST memory explosion
    # This approach filters arrays directly without creating intermediate rows,
    # reducing memory usage from 15GB+ to under 4GB for large chromatograms (37K+ points)
    QUERY_CREATE_HELPERS = """
        CREATE OR REPLACE MACRO compute_chromatogram_metrics(scan_times, intensities, rt_min, rt_max) AS TABLE (
            WITH 
            -- Filter arrays using list operations (no UNNEST = no memory explosion)
            filtered AS (
                SELECT list_filter(
                    list_transform(
                        range(1, len(scan_times) + 1),
                        i -> struct_pack(t := list_extract(scan_times, i), i := list_extract(intensities, i))
                    ),
                    p -> p.t >= rt_min AND p.t <= rt_max
                ) AS pairs
            ),
            -- Extract arrays from filtered pairs
            arrays AS (
                SELECT
                    list_transform(pairs, p -> p.t) AS scan_time_arr,
                    list_transform(pairs, p -> p.i) AS intensity_arr
                FROM filtered
            ),
            -- Compute trapezoid integration for peak_area
            -- Trapezoid rule: Area = Σ (y[i] + y[i+1])/2 × (x[i+1] - x[i])
            trapezoid AS (
                SELECT
                    scan_time_arr,
                    intensity_arr,
                    -- Build list of trapezoid areas for each segment
                    list_transform(
                        range(1, len(scan_time_arr)),
                        i -> (
                            (list_extract(intensity_arr, i) + list_extract(intensity_arr, i + 1)) / 2.0
                            * (list_extract(scan_time_arr, i + 1) - list_extract(scan_time_arr, i))
                        )
                    ) AS trapezoid_areas
                FROM arrays
                -- Removed: WHERE len(intensity_arr) > 0
            ),
            -- Compute all metrics from arrays
            metrics AS (
                SELECT
                    len(intensity_arr) AS peak_n_datapoints,
                    -- Trapezoid integration
                    ROUND(COALESCE(list_sum(trapezoid_areas), 0), 0) AS peak_area,
                    ROUND(COALESCE(list_max(intensity_arr), 0), 0) AS peak_max,
                    ROUND(COALESCE(list_min(intensity_arr), 0), 0) AS peak_min,
                    ROUND(COALESCE(list_avg(intensity_arr), 0), 0) AS peak_mean,
                    -- Median: handle empty list case
                    ROUND(COALESCE(list_sort(intensity_arr)[CAST(len(intensity_arr) / 2 + 1 AS BIGINT)], 0), 0) AS peak_median,
                    -- RT of max intensity
                    scan_time_arr[CAST(list_position(intensity_arr, list_max(intensity_arr)) AS BIGINT)] AS peak_rt_of_max,
                    -- Index of max for top3 calculation
                    CAST(list_position(intensity_arr, list_max(intensity_arr)) AS BIGINT) AS max_idx,
                    scan_time_arr,
                    intensity_arr
                FROM trapezoid
            )
            -- Final output with peak_area_top3
            SELECT
                peak_area,
                ROUND(
                    COALESCE(intensity_arr[max_idx - 1], 0) +
                    list_max(intensity_arr) +
                    COALESCE(intensity_arr[max_idx + 1], 0),
                    0
                ) AS peak_area_top3,
                peak_max,
                peak_min,
                peak_mean,
                peak_rt_of_max,
                peak_median,
                peak_n_datapoints,
                scan_time_arr AS scan_time_list,
                intensity_arr AS intensity_list
            FROM metrics
        );
    """

    QUERY_CREATE_PENDING_PAIRS = """
                                 CREATE TABLE IF NOT EXISTS pending_result_pairs AS
                                 WITH pairs_to_process AS (
                                 SELECT c.peak_label,
                                                                  c.ms_file_label
                                                           FROM chromatograms c
                                                                    JOIN targets t ON c.peak_label = t.peak_label
                                                           WHERE CASE
                WHEN ? THEN c.peak_label IN (
                    SELECT peak_label FROM targets WHERE bookmark = TRUE
                )
                                                                     ELSE TRUE
                                                               END
                                                             AND (
                                                               ? OR NOT EXISTS (
                                                               SELECT 1 FROM results r
                                                                                WHERE r.peak_label = c.peak_label
                    AND r.ms_file_label = c.ms_file_label
                )
            )
        )
                                 SELECT peak_label,
                                        ms_file_label,
                                        ROW_NUMBER() OVER () AS pair_id
                                 FROM pairs_to_process
                                 ORDER BY peak_label, ms_file_label;
                                 """

    # Direct query - no intermediate CTE, streaming insert
    # When rt_align_enabled=TRUE, applies per-sample RT shifts to the integration window
    QUERY_PROCESS_BATCH = """
        INSERT INTO results (
            peak_label,
            ms_file_label,
            peak_area,
            peak_area_top3,
            peak_max,
            peak_min,
            peak_mean,
            peak_rt_of_max,
            peak_mz_of_max,
            peak_median,
            peak_n_datapoints,
            rt_aligned,
            rt_shift,
            scan_time,
            intensity
        )
        WITH batch_pairs AS (
            SELECT peak_label, ms_file_label
            FROM pending_result_pairs
            WHERE pair_id BETWEEN ? AND ?
        ),
        -- Join chromatograms with targets and calculate per-sample shift
        paired_data AS (
            SELECT 
                c.peak_label,
                c.ms_file_label,
                c.scan_time,
                c.intensity,
                c.mz_arr,
                t.rt_min,
                t.rt_max,
                COALESCE(t.rt_align_enabled, FALSE) AS rt_align_enabled,
                -- Extract per-sample shift from JSON, default to 0.0 if not found
                COALESCE(
                    CASE 
                        WHEN t.rt_align_enabled AND t.rt_align_shifts IS NOT NULL 
                        THEN TRY_CAST(json_extract(t.rt_align_shifts, '$."' || c.ms_file_label || '"') AS DOUBLE)
                        ELSE NULL
                    END,
                    0.0
                ) AS sample_shift
            FROM chromatograms c
            JOIN batch_pairs bp ON c.peak_label = bp.peak_label AND c.ms_file_label = bp.ms_file_label
            JOIN targets t ON c.peak_label = t.peak_label
        )
        SELECT 
            pd.peak_label,
            pd.ms_file_label,
            m.peak_area,
            m.peak_area_top3,
            m.peak_max,
            m.peak_min,
            m.peak_mean,
            m.peak_rt_of_max,
            -- peak_mz_of_max: Get mz at same index as peak max intensity
            CASE 
                WHEN pd.mz_arr IS NOT NULL AND list_position(pd.intensity, m.peak_max) IS NOT NULL
                THEN pd.mz_arr[CAST(list_position(pd.intensity, m.peak_max) AS BIGINT)]
                ELSE NULL
            END AS peak_mz_of_max,
            m.peak_median,
            m.peak_n_datapoints,
            pd.rt_align_enabled AS rt_aligned,
            pd.sample_shift AS rt_shift,
            m.scan_time_list,
            m.intensity_list
        FROM paired_data pd
        CROSS JOIN LATERAL compute_chromatogram_metrics(
            pd.scan_time, 
            pd.intensity, 
            -- Apply inverse shift: if peak was shifted by +X for visualization,
            -- we need to look at [rt_min - X, rt_max - X] in original data
            pd.rt_min - pd.sample_shift,
            pd.rt_max - pd.sample_shift
        ) AS m;
                          """



    if recompute:
        ensure_page_load_active(wdir, page_load_id, where="compute_results_in_batches:recompute")
        logger.info("Deleting existing results for recalculation...")
        with duckdb_connection(wdir, n_cpus=n_cpus, ram=ram, page_load_id=page_load_id) as con:
            con.execute("DELETE FROM results")

    # Create helper macro
    ensure_page_load_active(wdir, page_load_id, where="compute_results_in_batches:helpers")
    with duckdb_connection(wdir, n_cpus=n_cpus, ram=ram, page_load_id=page_load_id) as conn:
        logger.info("Creating helper macro...")
        conn.execute(QUERY_CREATE_HELPERS)

    # Create pending pairs table
    ensure_page_load_active(wdir, page_load_id, where="compute_results_in_batches:pending_pairs")
    with duckdb_connection(wdir, n_cpus=n_cpus, ram=ram, page_load_id=page_load_id) as conn:
        # Ensure clean database state before processing (clears any accumulated WAL)
        logger.info("Running CHECKPOINT to ensure clean database state...")
        conn.execute("CHECKPOINT")
        
        conn.execute("DROP TABLE IF EXISTS pending_result_pairs")

        logger.info("Getting pending pairs...")
        start_time = time.time()
        conn.execute(QUERY_CREATE_PENDING_PAIRS, [use_bookmarked, recompute])

        total_pairs = conn.execute("""
            SELECT COUNT(*) AS total,
                                          MIN(pair_id) AS min_id,
                                          MAX(pair_id) AS max_id
                                   FROM pending_result_pairs
                                   """).fetchone()

        elapsed = time.time() - start_time

        if total_pairs[0] == 0 or total_pairs[1] is None:
            logger.info(f"No pending pairs ({elapsed:.2f}s)")
            conn.execute("DROP TABLE IF EXISTS pending_result_pairs")
            return {'total_pairs': 0, 'processed': 0, 'failed': 0, 'batches': 0}

        total_count, min_id, max_id = total_pairs
        logger.info(f"{total_count:,} pending pairs ({elapsed:.2f}s)")
        
        # Auto-calculate optimal batch size if not explicitly provided
        if batch_size is None:
            ram_gb = ram if ram else 16  # Default to 16GB if not specified
            cpus = n_cpus if n_cpus else 4  # Default to 4 if not specified
            batch_size = calculate_optimal_batch_size(ram_gb, total_count, cpus)
            logger.info(f"Auto-calculated batch size: {batch_size} (based on {ram_gb}GB RAM, {cpus} CPUs, {total_count:,} pairs)")
        else:
            logger.info(f"Using specified batch size: {batch_size}")
        total_files = 0
        if set_progress:
            total_files = conn.execute("""
                                       SELECT COUNT(DISTINCT ms_file_label)
                                       FROM pending_result_pairs
                                       """).fetchone()[0]
        _send_progress(
            set_progress,
            0,
            stage="Results",
            detail=f"Pending pairs: {total_count:,}",
        )

    # Process in batches
    processed = 0
    failed = 0
    batches = 0
    processed_files: set[str] = set()

    current_id = min_id
    batch_num = 1
    total_batches = (total_count + batch_size - 1) // batch_size

    ensure_page_load_active(wdir, page_load_id, where="compute_results_in_batches:process")
    with duckdb_connection(wdir, n_cpus=n_cpus, ram=ram, page_load_id=page_load_id) as conn:
        # Settings for bulk writes
        conn.execute("SET wal_autocheckpoint='1GB'")
        conn.execute("BEGIN TRANSACTION")

        batches_in_txn = 0

        while current_id <= max_id:
            ensure_page_load_active(
                wdir,
                page_load_id,
                where=f"compute_results_in_batches:batch:{batch_num}",
            )
            start_id = current_id
            end_id = current_id + batch_size - 1
            log_line = None  # Initialize before try block to avoid UnboundLocalError in finally

            try:
                batch_count = conn.execute("""
                                           SELECT COUNT(*)
                                           FROM pending_result_pairs
                                           WHERE pair_id BETWEEN ? AND ?
                                           """, [start_id, end_id]).fetchone()[0]

                if batch_count == 0:
                    current_id += batch_size
                    continue

                batch_start = time.time()

                conn.execute(QUERY_PROCESS_BATCH, [start_id, end_id])

                if set_progress:
                    batch_files = conn.execute("""
                                               SELECT DISTINCT ms_file_label
                                               FROM pending_result_pairs
                                               WHERE pair_id BETWEEN ? AND ?
                                               """, [start_id, end_id]).fetchall()
                    processed_files.update(
                        row[0] for row in batch_files if row and row[0] is not None
                    )

                batch_elapsed = time.time() - batch_start
                processed += batch_count
                batches += 1
                batches_in_txn += 1

                pairs_per_sec = batch_count / batch_elapsed
                logger.info(f"Batch {batch_num:>4}/{total_batches} | "
                      f"IDs {start_id:>6}-{end_id:>6} | "
                      f"{batch_count:>4} pairs | "
                      f"Batch time: {batch_elapsed:>5.2f}s | "
                      f"Progress {processed:>6,}/{total_count:,}")
                log_line = (f"Batch {batch_num}/{total_batches} | "
                            f"Progress {processed:,}/{total_count:,} | "
                            f"Time/batch {batch_elapsed:0.2f}s"
                            # f"Processing ({pairs_per_sec:0.1f} pairs/s)"
                            )


                # Periodic checkpoint
                if batches_in_txn >= checkpoint_every:
                    flush_start = time.time()
                    conn.execute("COMMIT")
                    conn.execute("CHECKPOINT")
                    conn.execute("BEGIN TRANSACTION")
                    logger.debug(f"  [Commit + Checkpoint]... {time.time() - flush_start:.2f}s")
                    batches_in_txn = 0

                batch_num += 1

            except Exception as e:
                if 'Cancelled' in type(e).__name__ or 'PreventUpdate' in type(e).__name__:
                    raise
                batch_elapsed = time.time() - batch_start if 'batch_start' in locals() else 0
                failed += batch_count if 'batch_count' in locals() else 0

                logger.error(f"Error processing batch: {batch_elapsed:>5.2f}s | Error: {str(e)[:80]}")
                _add_processing_error(wdir, str(e))

                # Recover from aborted transaction to allow subsequent batches to proceed
                try:
                    conn.execute("ROLLBACK")
                    conn.execute("BEGIN TRANSACTION")
                    batches_in_txn = 0
                except Exception as rollback_error:
                    logger.error(f"Failed to rollback transaction: {rollback_error}")

                with open('failed_batches_results.log', 'a') as f:
                    f.write(f"\n{'=' * 60}\n")
                    f.write(f"Timestamp: {time.strftime('%Y-%m-%d %H:%M:%S')}\n")
                    f.write(f"Batch {batch_num}/{total_batches}\n")
                    f.write(f"IDs: {start_id}-{end_id}\n")
                    f.write(f"Error: {str(e)}\n")
                    f.write(f"{'=' * 60}\n")
                
                batch_num += 1  # Move to next batch even on failure

            finally:
                if 'batch_count' in locals() and batch_count > 0:
                    progress_pct = (processed / total_count) * 100
                    files_done = len(processed_files) if set_progress else 0
                    detail_text = (
                        f"{log_line}"
                        if log_line else
                        f"Results batch {batch_num}/{total_batches} | "
                        f"Pairs {processed:,}/{total_count:,}"
                    )
                    _send_progress(
                        set_progress,
                        round(progress_pct, 1),
                        stage="Results",
                        detail=detail_text,
                    )

            current_id += batch_size

        # Commit final
        logger.info("Final commit + checkpoint...")
        flush_start = time.time()
        conn.execute("COMMIT")
        conn.execute("CHECKPOINT")
        logger.info(f"Checkpoint completed in {time.time() - flush_start:.2f}s")

    # Clean up
    ensure_page_load_active(wdir, page_load_id, where="compute_results_in_batches:cleanup")
    with duckdb_connection(wdir, n_cpus=n_cpus, ram=ram, page_load_id=page_load_id) as conn:
        conn.execute("DROP TABLE IF EXISTS pending_result_pairs")

    logger.info(
        f"Results computation complete. "
        f"Total pairs: {total_count:,}, Processed: {processed:,}, "
        f"Failed: {failed:,}, Batches: {batches:,}"
    )

    return {
        'total_pairs': total_count,
        'processed': processed,
        'failed': failed,
        'batches': batches
    }


def compute_fitted_results(
    wdir: str,
    use_bookmarked: bool = False,
    recompute: bool = False,
    n_workers: int = 8,
    set_progress=None,
    n_cpus=None,
    ram=None,
    page_load_id: str | None = None,
) -> dict:
    """
    Compute EMG peak fitting for all results.
    
    This function:
    1. Queries existing results with chromatogram data
    2. Fits EMG model to each peak using parallel processing
    3. Updates results table with fitted metrics
    
    Parameters:
        wdir: Working directory path
        use_bookmarked: Only process bookmarked targets
        recompute: Recompute even if fitting was already done
        n_workers: Number of parallel workers (default 8)
        set_progress: Progress callback function
        n_cpus: CPU cores for DuckDB
        ram: RAM allocation for DuckDB
    
    Returns:
        Dictionary with processing statistics
    """
    from .peak_fitting import fit_peaks_batch
    
    ensure_page_load_active(wdir, page_load_id, where="compute_fitted_results:start")
    _send_progress(set_progress, 0, stage="Peak Fitting", detail="Preparing data...")
    
    # Query results that need fitting
    ensure_page_load_active(wdir, page_load_id, where="compute_fitted_results:query")
    with duckdb_connection(wdir, n_cpus=n_cpus, ram=ram, page_load_id=page_load_id) as conn:
        # Build query to get peaks needing fitting
        where_conditions = []
        if use_bookmarked:
            where_conditions.append(
                "r.peak_label IN (SELECT peak_label FROM targets WHERE bookmark = TRUE)"
            )
        if not recompute:
            where_conditions.append("(r.fit_success IS NULL OR r.fit_success = FALSE)")
        
        where_clause = f"WHERE {' AND '.join(where_conditions)}" if where_conditions else ""
        
        # Get chromatogram data for fitting
        query = f"""
            SELECT 
                r.peak_label,
                r.ms_file_label,
                r.scan_time,
                r.intensity,
                t.rt
            FROM results r
            JOIN targets t ON r.peak_label = t.peak_label
            {where_clause}
        """
        
        data = conn.execute(query).fetchall()
        
    if not data:
        logger.info("No peaks require fitting")
        _send_progress(set_progress, 100, stage="Peak Fitting", detail="No peaks to fit")
        return {'total': 0, 'fitted': 0, 'failed': 0}
    
    ensure_page_load_active(wdir, page_load_id, where="compute_fitted_results:fit_start")
    total_peaks = len(data)
    logger.info(f"Fitting {total_peaks:,} peaks with {n_workers} workers...")
    
    _send_progress(
        set_progress, 
        5, 
        stage="Peak Fitting", 
        detail=f"Fitting {total_peaks:,} peaks..."
    )
    
    # Prepare data for parallel processing
    # Format: (peak_label, ms_file_label, scan_time, intensity, expected_rt)
    peaks_data = [
        (row[0], row[1], row[2], row[3], row[4])
        for row in data
    ]
    
    # Progress callback for fitting - maps to 5-80% of progress bar
    batch_start_time = [time.time()]  # Start of current batch
    last_batch_duration = [0.0]       # Duration of last completed batch
    last_logged_batch = [0]
    batch_size = max(100, total_peaks // 20)  # ~20 batches or 100 peaks minimum
    
    def fitting_progress(completed, total, rate):
        # Map fitting progress (0-100%) to UI progress (5-80%)
        fit_pct = (completed / total) * 100 if total > 0 else 0
        ui_pct = 5 + (fit_pct * 0.75)  # 5% + up to 75% = 80% max
        
        # Calculate batch info
        current_batch_idx = (completed - 1) // batch_size
        batch_num = current_batch_idx + 1
        total_batches = (total + batch_size - 1) // batch_size
        
        current_time = time.time()
        
        # Check if we moved to a new batch (or finished)
        if batch_num > last_logged_batch[0] or completed == total:
            duration = current_time - batch_start_time[0]
            last_batch_duration[0] = duration
            
            logger.info(
                f"Batch {batch_num:4}/{total_batches} | "
                f"Peaks {completed:6,}/{total:,} | "
                f"Batch time: {duration:5.2f}s | "
                f"Rate: {rate:.0f}/sec"
            )
            
            # Prepare for next batch
            if completed < total:
                batch_start_time[0] = current_time
                last_logged_batch[0] = batch_num
        
        # Display time: Use last batch duration for stability, or current elapsed for batch 1
        display_time = last_batch_duration[0] if batch_num > 1 else (current_time - batch_start_time[0])
        
        # Consistent format for UI: "Batch X/Y | Progress X/Y | Time/batch Zs"
        detail = f"Batch {batch_num}/{total_batches} | Progress {completed:,}/{total:,} | Time/batch {display_time:.2f}s"
        
        _send_progress(set_progress, round(ui_pct, 1), stage="Peak Fitting", detail=detail)
    
    # Run parallel fitting with progress updates
    start_time = time.time()
    fit_results = fit_peaks_batch(peaks_data, n_workers=n_workers, progress_callback=fitting_progress)
    elapsed = time.time() - start_time
    
    logger.info(f"Fitting completed in {elapsed:.2f}s ({total_peaks/elapsed:.0f} peaks/sec)")
    
    _send_progress(
        set_progress, 
        80, 
        stage="Peak Fitting", 
        detail="Updating database..."
    )
    
    # Update database with fit results in batches for progress tracking
    fitted_count = 0
    failed_count = 0
    total_results = len(fit_results)
    update_batch_size = max(100, total_results // 10)  # ~10 updates
    db_start_time = time.time()
    last_db_batch_time = time.time()
    last_db_duration = 0.0
    
    with duckdb_connection(wdir, n_cpus=n_cpus, ram=ram) as conn:
        conn.execute("BEGIN TRANSACTION")
        
        for i, result in enumerate(fit_results):
            try:
                conn.execute("""
                    UPDATE results
                    SET peak_area_fitted = ?,
                        peak_sigma = ?,
                        peak_tau = ?,
                        peak_asymmetry = ?,
                        peak_rt_fitted = ?,
                        fit_r_squared = ?,
                        fit_success = ?
                    WHERE peak_label = ? AND ms_file_label = ?
                """, [
                    result['peak_area_fitted'],
                    result['peak_sigma'],
                    result['peak_tau'],
                    result['peak_asymmetry'],
                    result['peak_rt_fitted'],
                    result['fit_r_squared'],
                    result['fit_success'],
                    result['peak_label'],
                    result['ms_file_label'],
                ])
                
                if result['fit_success']:
                    fitted_count += 1
                else:
                    failed_count += 1
                    
            except Exception as e:
                logger.error(f"Failed to update {result['peak_label']}: {e}")
                _add_processing_error(wdir, str(e))
                failed_count += 1
            
            # Progress update every batch
            if (i + 1) % update_batch_size == 0 or (i + 1) == total_results:
                # Map progress from 80-98%
                db_pct = ((i + 1) / total_results) * 18  # 18% range for DB updates
                ui_pct = 80 + db_pct
                
                current_time = time.time()
                batch_duration = current_time - last_db_batch_time
                last_db_duration = batch_duration
                last_db_batch_time = current_time
                
                db_elapsed = current_time - db_start_time
                rate = (i + 1) / db_elapsed if db_elapsed > 0 else 0
                batch_num = (i + 1) // update_batch_size
                total_batches = (total_results // update_batch_size) + (1 if total_results % update_batch_size else 0)
                
                # Consistent format for UI: "Batch X/Y | Progress X/Y | Time/batch Zs"
                detail = f"Batch {batch_num}/{total_batches} | Progress {i+1:,}/{total_results:,} | Time/batch {batch_duration:.2f}s"
                
                logger.info(
                    f"Batch {batch_num:4}/{total_batches} | "
                    f"Rows {i+1:6,}/{total_results:,} | "
                    f"Batch time: {batch_duration:5.2f}s | "
                    f"Progress {i+1:,}/{total_results:,}"
                )
                _send_progress(set_progress, round(ui_pct, 1), stage="Peak Fitting", detail=detail)
        
        conn.execute("COMMIT")
        conn.execute("CHECKPOINT")
    
    _send_progress(
        set_progress, 
        100, 
        stage="Peak Fitting", 
        detail=f"Fitted {fitted_count:,} peaks"
    )
    
    total_elapsed = time.time() - start_time
    logger.info(
        f"Peak fitting complete. "
        f"Total: {total_peaks:,}, Fitted: {fitted_count:,}, Failed: {failed_count:,}"
    )
    
    return {
        'total': total_peaks,
        'fitted': fitted_count,
        'failed': failed_count,
        'elapsed_seconds': total_elapsed
    }


def compute_peak_properties(con: duckdb.DuckDBPyConnection,
                            set_progress=None,
                            recompute=False,
                            bookmarked=False
                            ):
    if recompute:
        logger.info("Deleting existing results for recalculation...")
        con.execute("DELETE FROM results")

    query = """
            INSERT INTO results (peak_label,
                                 ms_file_label,
                                 total_intensity,
                                 peak_area,
                                 peak_area_top3,
                                 peak_max,
                                 peak_min,
                                 peak_mean,
                                 peak_rt_of_max,
                                 peak_median,
                                 peak_n_datapoints,
                                 scan_time,
                                 intensity)
            WITH pairs_to_process AS (SELECT c.peak_label,
                                             c.ms_file_label
                                      FROM chromatograms c
                                               JOIN targets t ON c.peak_label = t.peak_label
                                      WHERE c.ms_file_label IN
                                            (SELECT ms_file_label FROM samples WHERE use_for_processing = TRUE)
                                        AND t.rt_min IS NOT NULL
                                        AND t.rt_max IS NOT NULL
                                        AND CASE
                                                WHEN ? -- bookmarked
                                                    THEN c.peak_label IN (SELECT peak_label FROM targets WHERE bookmark = TRUE)
                                                ELSE TRUE
                                          END
                                        AND (
                                          ? -- recompute
                                              OR NOT EXISTS (SELECT 1
                                                             FROM results r
                                                             WHERE r.peak_label = c.peak_label
                                                               AND r.ms_file_label = c.ms_file_label)
                                          )),
                 unnested AS (SELECT c.peak_label,
                                     c.ms_file_label,
                                     UNNEST(c.scan_time) AS scan_time,
                                     UNNEST(c.intensity) AS intensity
                              FROM chromatograms c
                                       JOIN pairs_to_process p ON c.peak_label = p.peak_label
                                  AND c.ms_file_label = p.ms_file_label),
-- Compute total_intensity (without rt filter)
                 total_stats AS (SELECT peak_label,
                                        ms_file_label,
                                        SUM(intensity) AS total_intensity
                                 FROM unnested
                                 GROUP BY peak_label, ms_file_label),
-- Filter by rt_min - rt_max window
                 filtered_range AS (SELECT u.peak_label,
                                           u.ms_file_label,
                                           u.scan_time,
                                           u.intensity
                                    FROM unnested u
                                             JOIN targets t ON u.peak_label = t.peak_label
                                    WHERE u.scan_time BETWEEN t.rt_min AND t.rt_max),
-- Compute trapezoid segments for each consecutive pair of points
                 trapezoid_segments AS (
                     SELECT peak_label,
                            ms_file_label,
                            scan_time,
                            intensity,
                            -- Trapezoid area for segment [i, i+1]: (y[i] + y[i+1])/2 * (x[i+1] - x[i])
                            CASE 
                                WHEN LEAD(scan_time) OVER w IS NOT NULL 
                                THEN (intensity + LEAD(intensity) OVER w) / 2.0 
                                     * (LEAD(scan_time) OVER w - scan_time)
                                ELSE 0
                            END AS segment_area
                     FROM filtered_range
                     WINDOW w AS (PARTITION BY peak_label, ms_file_label ORDER BY scan_time)
                 ),
-- Group the filtered data into lists
                 aggregated AS (SELECT peak_label,
                                       ms_file_label,
                                       LIST(scan_time ORDER BY scan_time) AS scan_time,
                                       LIST(intensity ORDER BY scan_time) AS intensity,
                                       -- Trapezoid integration (more accurate than simple sum)
                                       ROUND(SUM(segment_area), 0)        AS peak_area,
                                       ROUND(MAX(intensity), 0)           AS peak_max,
                                       ROUND(MIN(intensity), 0)           AS peak_min,
                                       ROUND(AVG(intensity), 0)           AS peak_mean,
                                       ROUND(MEDIAN(intensity), 0)        AS peak_median,
                                       COUNT(*)                           AS peak_n_datapoints
                                FROM trapezoid_segments
                                GROUP BY peak_label, ms_file_label),
-- Compute peak_area_top3
                 top3_calc AS (
                     SELECT 
                         peak_label,
                         ms_file_label,
                         ROUND(intensity + prev_intensity + next_intensity, 0) AS peak_area_top3
                     FROM (
                         SELECT 
                             peak_label,
                             ms_file_label,
                             intensity,
                             COALESCE(LAG(intensity) OVER (PARTITION BY peak_label, ms_file_label ORDER BY scan_time), 0) AS prev_intensity,
                             COALESCE(LEAD(intensity) OVER (PARTITION BY peak_label, ms_file_label ORDER BY scan_time), 0) AS next_intensity,
                             ROW_NUMBER() OVER (PARTITION BY peak_label, ms_file_label ORDER BY intensity DESC) AS rn
                         FROM filtered_range
                     ) ranked
                     WHERE rn = 1
                 ),
-- Find scan_time of peak_max
                 rt_of_max AS (SELECT peak_label,
                                      ms_file_label,
                                      scan_time AS peak_rt_of_max
                               FROM (SELECT peak_label,
                                            ms_file_label,
                                            scan_time,
                                            intensity,
                                            ROW_NUMBER() OVER (PARTITION BY peak_label, ms_file_label ORDER BY intensity DESC) AS rn
                                     FROM filtered_range) sub
                               WHERE rn = 1)
            SELECT a.peak_label,
                   a.ms_file_label,
                   ts.total_intensity,
                   a.peak_area,
                   t3.peak_area_top3,
                   a.peak_max,
                   a.peak_min,
                   a.peak_mean,
                   rm.peak_rt_of_max,
                   a.peak_median,
                   a.peak_n_datapoints,
                   a.scan_time,
                   a.intensity
            FROM aggregated a
                     JOIN total_stats ts ON a.peak_label = ts.peak_label AND a.ms_file_label = ts.ms_file_label
                     JOIN top3_calc t3 ON a.peak_label = t3.peak_label AND a.ms_file_label = t3.ms_file_label
                     JOIN rt_of_max rm ON a.peak_label = rm.peak_label AND a.ms_file_label = rm.ms_file_label
            ORDER BY a.peak_label, a.ms_file_label;
            """

    # Shared variable to accumulate progress
    accumulated_progress = [0.0]
    stop_monitoring = [False]

    def monitor_progress():
        """Monitor progress of the current query"""
        while not stop_monitoring[0]:
            try:
                qp = con.query_progress()
                if qp != -1 and qp > 0:
                    # Total progress depends on which query is executing
                    total_progress = qp

                    accumulated_progress[0] = total_progress
                    if set_progress:
                        set_progress(round(total_progress, 1))
                    logger.info(f"Progress: {total_progress:.1f}%")
                time.sleep(0.05)

            except (duckdb.InvalidInputException, duckdb.ConnectionException):
                break
            except Exception as e:
                logger.error(f"Progress monitoring error: {e}")
                break

    # Start monitoring
    if set_progress:
        progress_thread = Thread(target=monitor_progress, daemon=True)
        progress_thread.start()

    try:
        # Run MS1
        logger.info("Processing MS1 chromatograms...")
        con.execute(query, [bookmarked, recompute])
        accumulated_progress[0] = 100
        if set_progress:
            set_progress(round(accumulated_progress[0], 1))

        logger.info("Chromatograms computed and inserted into DuckDB.")

    finally:
        stop_monitoring[0] = True
        if set_progress:
            progress_thread.join(timeout=0.5)

    logger.info("Peak properties computed and inserted into DuckDB.")


def create_pivot(conn, rows=None, cols=None, value='peak_area', table='results'):
    """
    Create pivot from DuckDB for unique per-pair data
    """

    # Use fetchall() for faster list extraction (3.25x speedup vs DataFrame)
    ordered_pl = [row[0] for row in conn.execute(f"""
        SELECT DISTINCT r.peak_label
        FROM {table} r
        JOIN targets t ON r.peak_label = t.peak_label
        ORDER BY t.ms_type
    """).fetchall()]

    group_cols_sql = ",\n                ".join([f"s.{col}" for col in GROUP_COLUMNS])

    query = f"""
        PIVOT (
            SELECT
                s.ms_type,
                s.sample_type,
                {group_cols_sql},
                r.ms_file_label,
                r.peak_label,
                r.{value}
            FROM {table} r
            JOIN samples s ON s.ms_file_label = r.ms_file_label
            WHERE s.use_for_analysis = TRUE
            ORDER BY s.ms_type, r.peak_label
        )
        ON peak_label
        USING FIRST({value})
        -- GROUP BY ms_type
        ORDER BY ms_type
    """
    df = conn.execute(query).df()
    meta_cols = ['ms_type', 'sample_type', *GROUP_COLUMNS, 'ms_file_label']
    keep_cols = [col for col in meta_cols if col in df.columns] + ordered_pl
    return df[keep_cols]


def compute_and_insert_chromatograms_iteratively(con: duckdb.DuckDBPyConnection, set_progress=None):
    """
    Computes and inserts chromatograms iteratively by batching targets.

    :param con: An active DuckDB connection.
    :param set_progress: A callback function to update the progress bar.
    """
    # Use SQL filtering for faster list extraction (1.46x speedup vs DataFrame filtering)
    ms1_targets = [row[0] for row in con.execute(
        "SELECT peak_label FROM targets WHERE ms_type = 'ms1'"
    ).fetchall()]
    ms2_targets = [row[0] for row in con.execute(
        "SELECT peak_label FROM targets WHERE ms_type = 'ms2'"
    ).fetchall()]
    ms_files_count = con.execute("SELECT count(*) FROM samples WHERE use_for_optimization = TRUE").fetchone()[0]

    if ms_files_count == 0:
        if set_progress:
            set_progress(100)
        return

    n_total = len(ms1_targets) + len(ms2_targets)
    processed_count = 0

    def process_batch(targets_batch):
        nonlocal processed_count
        if not targets_batch:
            return

        placeholders = ', '.join(['?'] * len(targets_batch))
        query = f"""
        INSERT INTO chromatograms (peak_label, ms_file_label, scan_time, intensity)
        WITH precomputed AS (
            SELECT
                t.peak_label,
                m.ms_file_label,
                list(m.scan_time ORDER BY m.scan_time) AS scan_time,
                list(m.intensity ORDER BY m.scan_time) AS intensity
            FROM ms_data AS m
            JOIN targets AS t ON m.mz BETWEEN t.mz_mean - (t.mz_mean * t.mz_width / 1e6)
                                       AND t.mz_mean + (t.mz_mean * t.mz_width / 1e6)
            JOIN samples AS s ON m.ms_file_label = s.ms_file_label
            WHERE s.use_for_optimization = TRUE AND t.peak_label IN ({placeholders})
            GROUP BY t.peak_label, m.ms_file_label
        )
        SELECT peak_label, ms_file_label, scan_time, intensity
        FROM precomputed
        ON CONFLICT (ms_file_label, peak_label) DO UPDATE
            SET scan_time = excluded.scan_time,
                intensity = excluded.intensity;
        """
        con.execute(query, targets_batch)
        processed_count += len(targets_batch)
        if set_progress and n_total > 0:
            progress = round(processed_count / n_total * 100, 2)
            set_progress(progress)

    # Process MS2 targets in batches of 10
    for i in range(0, len(ms2_targets), 10):
        process_batch(ms2_targets[i:i + 10])

    # Determine batch size for MS1 targets
    ms1_batch_size = 1 if ms_files_count >= 50 else 5
    for i in range(0, len(ms1_targets), ms1_batch_size):
        process_batch(ms1_targets[i:i + ms1_batch_size])

    if set_progress and n_total > 0:
        set_progress(100)

    logger.info("Iterative chromatogram computation complete.")


def get_chromatogram_envelope(conn, target_label, ms_type='ms1', bins=500, full_range=False):
    """
    Computes Min/Max/Mean envelope for chromatograms, binned by time.
    Returns a Polars DataFrame with columns: 
    [sample_type, color, bin_idx, rt, min_int, max_int, mean_int, count, sample_count]
    """
    
    # Full-range downsampled chromatograms are only available for MS1.
    if full_range and ms_type != 'ms1':
        logger.warning(
            "Full-range envelope requested for ms_type=%s; falling back to sliced range.",
            ms_type,
        )
        full_range = False

    # Determine which columns to use
    if full_range:
        time_col = "scan_time_full_ds"
        int_col = "intensity_full_ds"
    else:
        time_col = "scan_time"
        int_col = "intensity"

    try:
        bins = int(bins)
    except (TypeError, ValueError):
        bins = 500
    bins = max(10, min(bins, 2000))

    query = f"""
    WITH picked_target AS (
        SELECT peak_label, intensity_threshold
        FROM targets
        WHERE peak_label = ?
    ),
    picked_samples AS (
        SELECT ms_file_label, sample_type, color
        FROM samples
        WHERE use_for_optimization = TRUE
    ),
    base AS (
        SELECT
            c.ms_file_label,
            s.sample_type,
            s.color,
            c.{time_col} AS scan_time_list,
            c.{int_col} AS intensity_list,
            COALESCE(t.intensity_threshold, 0) AS intensity_threshold
        FROM chromatograms c
        JOIN picked_samples s ON c.ms_file_label = s.ms_file_label
        JOIN picked_target t ON c.peak_label = t.peak_label
        WHERE c.peak_label = ?
          AND c.ms_type = ?
          AND c.{time_col} IS NOT NULL
          AND c.{int_col} IS NOT NULL
    ),
    zipped AS (
        SELECT
            ms_file_label,
            sample_type,
            color,
            intensity_threshold,
            list_transform(
                range(1, len(scan_time_list) + 1),
                i -> struct_pack(
                    rt := list_extract(scan_time_list, i),
                    intens := list_extract(intensity_list, i)
                )
            ) AS pairs
        FROM base
    ),
    raw_points AS (
        SELECT
            z.ms_file_label,
            z.sample_type,
            z.color,
            u.pair.rt AS rt,
            u.pair.intens AS intens
        FROM zipped z
        CROSS JOIN UNNEST(z.pairs) AS u(pair)
        WHERE u.pair.intens >= z.intensity_threshold
    ),
    sample_counts AS (
        SELECT
            sample_type,
            COUNT(DISTINCT ms_file_label) AS sample_count
        FROM raw_points
        GROUP BY sample_type
    ),
    bounds AS (
        -- Calculate bounds but ignore extreme outliers if any (though data should be sliced to 30s)
        SELECT MIN(rt) as min_rt, MAX(rt) as max_rt FROM raw_points
    ),
    binned AS (
        SELECT
            p.sample_type,
            p.color,
            p.rt,
            p.intens,
            -- Increase bins to 500 for higher resolution
            CAST(
                LEAST(
                    {bins - 1},
                    GREATEST(
                        0,
                        FLOOR((p.rt - b.min_rt) / (b.max_rt - b.min_rt + 1e-9) * {bins})
                    )
                ) AS INTEGER
            ) as bin_idx
        FROM raw_points p, bounds b
    ),
    aggregated AS (
        SELECT
            sample_type,
            color,
            bin_idx,
            AVG(rt) as rt,
            MIN(intens) as min_int,
            MAX(intens) as max_int,
            AVG(intens) as mean_int,
            COUNT(*) as count
        FROM binned
        GROUP BY sample_type, color, bin_idx
    )
    SELECT
        a.*,
        sc.sample_count
    FROM aggregated a
    LEFT JOIN sample_counts sc USING (sample_type)
    ORDER BY a.sample_type, a.bin_idx
    """
    
    params = [target_label, target_label, ms_type]
    df = conn.execute(query, params).pl()

    # Fallback: if no MS2 chromatograms exist for this target, try MS1 so UI isn't blank.
    if df.is_empty() and ms_type == 'ms2':
        df = conn.execute(query, [target_label, target_label, 'ms1']).pl()
        if not df.is_empty():
            logger.warning(
                "No MS2 chromatograms found for target '%s'; using MS1 envelope instead.",
                target_label,
            )

    return df
