#!/usr/bin/env python3
"""
Fuzzy Matcher GUI v3.11.0 - Modern Progress System
===================================================
Complete async refactor with modern progress tracking:
- All long-running operations through engine_bridge
- ProgressEvent-based progress reporting
- Phase-aware progress tracking (INIT/RUN/FINAL)
- No UI blocking
- Clean separation of concerns
- Type-safe boundaries
"""

# ── stdlib ──────────────────────────────────────────────────────────────────
from __future__ import annotations
import asyncio
import concurrent.futures as cf
import csv
import json
import logging
import math
import multiprocessing as mp
import os
import queue
import shutil
import tempfile
import threading
import gc
import uuid
import warnings
import inspect
from collections import deque
from dataclasses import dataclass, field
from pathlib import Path
import time
from typing import Any, Callable, Dict, List, Optional, Tuple, Union

# ── third-party ─────────────────────────────────────────────────────────────
import numpy as np
import pandas as pd
import tkinter as tk
from tkinter import ttk, filedialog, messagebox
import yaml
import re
import requests
import difflib
import rapidfuzz.fuzz as rapidfuzz_fuzz

# ── FoundryMatch imports ────────────────────────────────────────────────────
from fmatch.desktop import engine_bridge
from fmatch.desktop.models import (
    DuplicateResultModel,
    MatchResultModel,
)
from fmatch.desktop.results_popup import ResultsPopup
from fmatch.core.progress_event import ProgressEvent, ProgressPhase

try:
    from fmatch.core.telemetry import TelemetryCollector
except ImportError:
    # Define a placeholder if the telemetry module is not found
    # This makes the feature optional and prevents crashing.
    logging.warning("TelemetryCollector not found. Telemetry will be disabled.")

    class TelemetryCollector:
        def __init__(self, *args, **kwargs):
            self.enabled = False

        def record_icon_assignment(self, *args, **kwargs):
            pass

        def record_icon_override(self, *args, **kwargs):
            pass


from fmatch.core.sampling import (
    ReservoirSampler,
    DEFAULT_SAMPLE_SIZE,
)

from fmatch.core.utils import (
    sanitize_column_name,
    analyze_domain_matching_strategy,
)

from fmatch.core import (
    DedupeConfig,
    MatchConfig,
    BlockingMode,
    build_dedupe_config,
    build_match_config,
    DEFAULT_NORMALIZATION_RULES,
    AVAILABLE_RULESETS,
    detect_encoding,
    decide_processing_mode,
    _sanitize_name,
    determine_optimal_blocking_strategy,
)
from fmatch.core.algorithm_selector import (
    PerColumnAlgorithmSelector,
    EnhancedDedupeConfig,
    EnhancedMatchConfig,
)
from fmatch.core.engine import (
    _get_individual_column_stats as engine_get_column_stats,  # Import ML version
    auto_select_id_column,
    select_id_column_with_override,
    generate_id_selection_report,
    VARIANT_CATEGORIES,
    VARIANT_DESCRIPTIONS,
    PREPROC_REGISTRY,
    BlockingWeightProfile,
    BlockingColumnValidator,
)
from fmatch.shared.logger import configure_logging
from fmatch.integrations.salesforce.gui_modifications import (
    add_salesforce_support_to_app,
    add_salesforce_results_button,
    modify_refresh_ui_for_salesforce,
    save_salesforce_config_to_session,
    load_salesforce_config_from_session,
)

# Compact Salesforce dialog
from fmatch.desktop.ui_constants import UIState
from fmatch.desktop.blocking_transparency_dialog import BlockingTransparencyWindow

# Salesforce integration modules

# ── B2C Domain Filtering imports ────────────────────────────────────────────
from fmatch.desktop.b2c_gui_integration import (
    add_b2c_filter_controls,
    add_b2c_wizard_hint,
    save_b2c_config,
    load_b2c_config,
)
from fmatch.core.b2c_engine_integration import B2CFilterConfig

# ── Constants and Configuration ─────────────────────────────────────────────
log = logging.getLogger(__name__)
warnings.filterwarnings(
    "ignore", category=UserWarning, message="X does not have valid feature names"
)

# Configuration dictionary

_config_dir = Path.home() / ".foundrymatch"
_config_dir.mkdir(exist_ok=True)
(_config_dir / "cache").mkdir(exist_ok=True)

CONFIG = {
    "APP_VERSION": "3.11.0",
    "LOG_FILE": _config_dir / "fuzzy_matcher.log",
    "CONFIG_PATH": _config_dir / "fm_session.json",
    "CACHE_PATH": _config_dir / "cache",
    "DEFAULT_THRESHOLD": 80,
    "MIN_THRESHOLD": 0,
    "MAX_THRESHOLD": 100,
    "DUP_BLOCK_LIMIT": 1000,
    "DEFAULT_WEIGHT": 1,
    "MIN_WEIGHT": 1,
    "MAX_WEIGHT": 5,
    "MAX_WORKERS": 4,
    "MAX_ENGINE_WORKERS": 2,
    "MAX_GUI_LOAD_BYTES": 500 * 1024 * 1024,  # 500MB limit for GUI loading
    "COLORS": {
        "primary": "#2c3e50",
        "secondary": "#34495e",
        "success": "#27ae60",
        "danger": "#c0392b",
        "warning": "#f39c12",
        "info": "#2980b9",
        "bg": "#ecf0f1",
        "text": "#2c3e50",
        "placeholder": "#95a5a6",
        "disabled_bg": "#e0e0e0",
        "disabled_fg": "#a0a0a0",
    },
}

# Additional directories
CUSTOM_SEMANTIC_TYPES_DIR = _config_dir / "semantic_types"
MAPPING_DIR = _config_dir / "mappings"
MAPPING_DIR.mkdir(exist_ok=True)

_APP_VERSION = "3.11.0"

# UI Constants for Space Optimization
COMPACT_TILE_HEIGHT = 80
COMPACT_TILE_WIDTH = 180
PLACEHOLDER_FONT = ("TkDefaultFont", 10, "italic")
PLACEHOLDER_COLOR = "gray"
MAX_TREE_HEIGHT = 6

# Operator metadata for custom rules
OPERATORS = {
    "Similarity (WRatio) >=": {
        "id": "sim_wratio_ge",
        "needs_ref": True,
        "needs_threshold": True,
    },
    "Similarity (Token Set) >=": {
        "id": "sim_token_set_ge",
        "needs_ref": True,
        "needs_threshold": True,
    },
    "Is an exact match": {
        "id": "exact_match",
        "needs_ref": True,
        "needs_threshold": False,
    },
    "Is NOT an exact match": {
        "id": "not_exact_match",
        "needs_ref": True,
        "needs_threshold": False,
    },
    "Sounds like (Soundex)": {
        "id": "soundex_match",
        "needs_ref": True,
        "needs_threshold": False,
    },
}

# Algorithm definitions
ALGORITHMS = {
    "Ensemble Score": {
        "engine_val": "ensemble",
        "tooltip": "Intelligent multi-algorithm ensemble scoring (Recommended)",
    },
    "WRatio": {
        "engine_val": "fuzz.WRatio",
        "tooltip": "Weighted Ratio: Good all-around for typo correction",
    },
    "QRatio": {
        "engine_val": "fuzz.QRatio",
        "tooltip": "Quick Ratio: Faster, less comprehensive",
    },
    "Ratio": {
        "engine_val": "fuzz.ratio",
        "tooltip": "Simple Ratio: Basic Levenshtein distance",
    },
    "PartialRatio": {
        "engine_val": "fuzz.partial_ratio",
        "tooltip": "Partial Ratio: Good for substring matches",
    },
    "TokenSetRatio": {
        "engine_val": "fuzz.token_set_ratio",
        "tooltip": "Token Set Ratio: Ignores word order",
    },
    "TokenSortRatio": {
        "engine_val": "fuzz.token_sort_ratio",
        "tooltip": "Token Sort Ratio: Handles word order differences",
    },
    "Jaro": {
        "engine_val": "jaro_similarity",
        "tooltip": "Jaro Distance: Good for short strings",
    },
    "JaroWinkler": {
        "engine_val": "jaro_winkler_similarity",
        "tooltip": "Jaro-Winkler: Better for names",
    },
    "Soundex": {"engine_val": "soundex", "tooltip": "Soundex: Phonetic matching"},
    "Exact": {
        "engine_val": "exact_match",
        "tooltip": "Exact Match: 100 for identical, 0 otherwise",
    },
}

# Heuristic rules for algorithm suggestions
HEURISTIC_RULES = [
    {
        "condition": lambda s: s.get("avg_len", float("inf")) < 12
        and s.get("avg_tokens", float("inf")) <= 2.0
        and s.get("alpha_char_ratio", 0.0) > 0.7,
        "algo_display_name": "JaroWinkler",
        "reason": "Short alphabetic strings with few words",
    },
    {
        "condition": lambda s: s.get("avg_tokens", 0.0) > 3.5,
        "algo_display_name": "TokenSetRatio",
        "reason": "Multiple words where order might differ",
    },
    {
        "condition": lambda s: s.get("numeric_char_ratio", 0.0) > 0.6
        and s.get("avg_len", float("inf")) < 25,
        "algo_display_name": "Ratio",
        "reason": "Primarily numeric strings",
    },
    {
        "condition": lambda s: s.get("avg_len", 0.0) > 50
        and s.get("avg_tokens", 0.0) > 5,
        "algo_display_name": "WRatio",
        "reason": "Longer descriptive text",
    },
    {
        "condition": lambda s: True,
        "algo_display_name": "WRatio",
        "reason": "General-purpose default",
    },
]

# ── Helper Classes ──────────────────────────────────────────────────────────


def determine_optimal_blocking_strategy_from_paths(
    src_path: str,
    ref_path: Optional[str],
    src_encoding: Optional[str],
    ref_encoding: Optional[str],
    mappings: Optional[list[dict]],
    mode: Union[BlockingMode, str],
    ruleset: str,
    **kw: Any,
) -> dict:
    """
    Wrapper for determine_optimal_blocking_strategy that loads data from file paths.
    This function is picklable and suitable for process pool execution.
    """
    import pandas as pd
    import os

    # Normalize and validate paths
    if not src_path.startswith("Salesforce:"):
        src_path = os.path.abspath(os.path.expanduser(src_path))
        if not os.path.exists(src_path):
            raise FileNotFoundError(f"Source file not found: {src_path}")

    if ref_path and not ref_path.startswith("Salesforce:"):
        ref_path = os.path.abspath(os.path.expanduser(ref_path))
        if not os.path.exists(ref_path):
            raise FileNotFoundError(f"Reference file not found: {ref_path}")

    # Extract enum value properly
    mode_value = mode.value if hasattr(mode, "value") else mode

    # Load source data
    if src_path.startswith("Salesforce:"):
        # For MVP: Return a simple blocking strategy without full analysis
        # This prevents crashes while maintaining basic functionality
        return {
            "mode": mode_value,
            "strategy": "fallback",
            "stats": {
                "src_count": 0,
                "ref_count": 0,
                "pairs_to_compute": 0,
                "reduction_ratio": 1.0,
                "warning": "Salesforce data blocking analysis not available in current version",
            },
            "columns": [],
            "error": None,
        }
    else:
        # Optimized CSV reading with configurable engine
        csv_engine = kw.get(
            "csv_engine", os.environ.get("FMATCH_CSV_ENGINE", "pyarrow")
        )  # Default to pyarrow
        log.info(f"Loading source CSV with engine: {csv_engine}")
        try:
            if csv_engine == "pyarrow":
                src_df = pd.read_csv(
                    src_path,
                    encoding=src_encoding or "utf-8",
                    engine="pyarrow",
                    dtype_backend="pyarrow",
                )
                log.info("Successfully loaded source with pyarrow engine")
            else:
                # Use standard C engine
                src_df = pd.read_csv(
                    src_path,
                    encoding=src_encoding or "utf-8",
                    engine="c",
                    low_memory=False,
                    on_bad_lines="skip",
                )
                log.info("Successfully loaded source with C engine")
        except Exception as e:
            # Fallback to standard engine
            log.warning(
                f"Failed to load with {csv_engine} engine, falling back to C engine: {e}"
            )
            src_df = pd.read_csv(
                src_path,
                encoding=src_encoding or "utf-8",
                low_memory=False,
                on_bad_lines="skip",
            )
            log.info("Successfully loaded source with fallback C engine")

    # Load reference data if in match mode
    ref_df = None
    if ref_path:
        if ref_path.startswith("Salesforce:"):
            # Return fallback for Salesforce reference too
            return {
                "mode": mode_value,
                "strategy": "fallback",
                "stats": {
                    "src_count": len(src_df) if "src_df" in locals() else 0,
                    "ref_count": 0,
                    "pairs_to_compute": 0,
                    "reduction_ratio": 1.0,
                    "warning": "Salesforce data blocking analysis not available in current version",
                },
                "columns": [],
                "error": None,
            }
        else:
            # Optimized CSV reading with configurable engine
            csv_engine = kw.get(
                "csv_engine", os.environ.get("FMATCH_CSV_ENGINE", "pyarrow")
            )  # Default to pyarrow
            log.info(f"Loading reference CSV with engine: {csv_engine}")
            try:
                if csv_engine == "pyarrow":
                    ref_df = pd.read_csv(
                        ref_path,
                        encoding=ref_encoding or "utf-8",
                        engine="pyarrow",
                        dtype_backend="pyarrow",
                    )
                    log.info("Successfully loaded reference with pyarrow engine")
                else:
                    # Use standard C engine
                    ref_df = pd.read_csv(
                        ref_path,
                        encoding=ref_encoding or "utf-8",
                        engine="c",
                        low_memory=False,
                        on_bad_lines="skip",
                    )
                    log.info("Successfully loaded reference with C engine")
            except Exception as e:
                # Fallback to standard engine
                log.warning(
                    f"Failed to load with {csv_engine} engine, falling back to C engine: {e}"
                )
                ref_df = pd.read_csv(
                    ref_path,
                    encoding=ref_encoding or "utf-8",
                    low_memory=False,
                    on_bad_lines="skip",
                )
                log.info("Successfully loaded reference with fallback C engine")

    # Call the original function with loaded DataFrames
    # Remove csv_engine from kwargs as it's not needed by the engine function
    engine_kwargs = {k: v for k, v in kw.items() if k != "csv_engine"}
    return determine_optimal_blocking_strategy(
        src_df=src_df,
        ref_df=ref_df,
        mappings=mappings,
        mode=mode,
        ruleset=ruleset,
        **engine_kwargs,
    )


@dataclass
class MappingHeuristics:
    """Configuration for mapping heuristics."""

    raw_score_initial_baseline: float = 0.0
    max_name_sim_contribution: float = 30.0
    fill_rate_bonus_high: float = 20.0
    fill_rate_bonus_medium: float = 10.0
    fill_rate_bonus_low: float = 5.0
    semantic_boost_strong_id: float = 15.0
    semantic_boost_medium_enum: float = 7.0
    semantic_boost_weak_match: float = 3.0
    strong_id_semantic_types: List[str] = field(
        default_factory=lambda: ["email", "website_domain", "name_company"]
    )
    medium_enum_semantic_types: List[str] = field(
        default_factory=lambda: ["country", "state", "status", "tier"]
    )
    distinct_reward_low_cardinality_overlap: float = 10.0
    distinct_penalty_general: float = -10.0
    distinct_low_cardinality_threshold: int = 25
    distinct_low_cardinality_overlap_threshold: float = 0.7
    distinct_ratio_low_threshold: float = 0.05
    distinct_ratio_high_threshold: float = 0.99
    type_agreement_bonus: float = 5.0
    type_mismatch_penalty: float = -5.0
    length_similarity_max_bonus: float = 5.0
    token_count_similarity_max_bonus: float = 3.0
    similarity_high_threshold: float = 0.7
    raw_score_min_clamp: float = 0.0
    raw_score_max_clamp: float = 100.0
    raw_to_scaled_weight_tiers: List[Tuple[float, int]] = field(
        default_factory=lambda: [(90.0, 5), (75.0, 4), (60.0, 3), (40.0, 2), (0.0, 1)]
    )
    auto_map_raw_score_cutoff: float = 50.0


class BlockingConfigWrapper:
    """Wrapper for blocking configuration with convenient attribute access."""

    # Define local constant for block key
    BLOCK_KEY_COLUMN = "__block_key__"

    def __init__(self, config_dict: Dict[str, Any]):
        self._config_dict = config_dict
        self.best_column: str = self.BLOCK_KEY_COLUMN  # Use local constant
        self.original_src_col: Optional[str] = config_dict.get("original_src_col")
        self.original_ref_col: Optional[str] = config_dict.get("original_ref_col")
        self.block_key_length: Optional[int] = config_dict.get("block_key_length")

        metrics = config_dict.get("metrics", {})
        self.entropy: float = metrics.get("entropy", 0.0)
        self.estimated_reduction_pct: float = metrics.get("reduction_pct", 0.0)
        self.quality_score: float = metrics.get("quality_score", 0.0)
        self.details_preview: List[Dict[str, Any]] = metrics.get("preview", [])

    def get(self, key, default=None):
        return self._config_dict.get(key, default)


class ToolTip:
    """Simple tooltip widget for Tkinter."""

    def __init__(self, widget: tk.Widget, text: str, delay_ms: int = 400):
        self.widget = widget
        self.text = text
        self.delay_ms = delay_ms
        self._id: Optional[str] = None
        self._tip: Optional[tk.Toplevel] = None

        widget.bind("<Enter>", self._schedule_show, "+")
        widget.bind("<Leave>", self.hide, "+")
        widget.bind("<ButtonPress>", self.hide, "+")

    def _schedule_show(self, event: tk.Event) -> None:
        if self._id is None:
            self._id = self.widget.after(self.delay_ms, self.show)

    def show(self) -> None:
        self._cancel_timer()
        if self._tip is not None:
            return

        x = self.widget.winfo_rootx() + 10
        y = self.widget.winfo_rooty() + self.widget.winfo_height() + 2

        self._tip = tk.Toplevel(self.widget)
        self._tip.wm_overrideredirect(True)
        self._tip.wm_geometry(f"+{x}+{y}")

        label = ttk.Label(
            self._tip,
            text=self.text,
            background="#ffffe0",
            relief="solid",
            borderwidth=1,
            justify="left",
            padding=4,
            font=("TkDefaultFont", 9, "normal"),
        )
        label.pack(ipadx=1)

    def hide(self, event: Optional[tk.Event] = None) -> None:
        self._cancel_timer()
        if self._tip is not None:
            self._tip.destroy()
            self._tip = None

    def _cancel_timer(self) -> None:
        if self._id is not None:
            self.widget.after_cancel(self._id)
            self._id = None


class PreprocessingVariantSelector(ttk.Frame):
    def __init__(self, parent, column_name, detected_variants):
        super().__init__(parent)

        self.column_name = column_name
        self.selected_variants = tk.StringVar(value=detected_variants[0])

        # Header
        ttk.Label(
            self,
            text=f"Preprocessing for '{column_name}':",
            font=("TkDefaultFont", 10, "bold"),
        ).pack(anchor="w")

        # Auto-detected info
        ttk.Label(
            self,
            text=f"Auto-detected types: {', '.join(detected_variants)}",
            foreground="green",
        ).pack(anchor="w")

        # Variant dropdown
        variant_frame = ttk.Frame(self)
        variant_frame.pack(fill="x", pady=5)

        ttk.Label(variant_frame, text="Method:").pack(side="left", padx=(0, 5))

        # Group variants by category
        variant_menu = ttk.Combobox(
            variant_frame,
            textvariable=self.selected_variants,
            state="readonly",
            width=20,
        )

        # Build grouped values
        grouped_values = []
        for category, variants in VARIANT_CATEGORIES.items():
            grouped_values.append(f"--{category.upper()}--")
            grouped_values.extend([f"  {v}" for v in variants if v in PREPROC_REGISTRY])

        variant_menu["values"] = grouped_values
        variant_menu.pack(side="left")

        # Tooltip
        self.create_tooltip(variant_menu)

    def create_tooltip(self, widget):
        def on_select(event):
            selected = self.selected_variants.get().strip()
            if selected in VARIANT_DESCRIPTIONS:
                tooltip_text = VARIANT_DESCRIPTIONS[selected]
                # Show tooltip (implement your tooltip widget)

        widget.bind("<<ComboboxSelected>>", on_select)


class WeightProfileSelector(ttk.LabelFrame):
    def __init__(self, parent, default_profile="balanced"):
        super().__init__(parent, text="Blocking Strategy Weights", padding=10)

        self.profile_var = tk.StringVar(value=default_profile)
        self.custom_weights = {}
        self.weight_entries = {}

        # Profile dropdown
        profile_frame = ttk.Frame(self)
        profile_frame.pack(fill="x", pady=(0, 10))

        ttk.Label(profile_frame, text="Profile:").pack(side="left", padx=(0, 5))

        profile_combo = ttk.Combobox(
            profile_frame, textvariable=self.profile_var, state="readonly", width=25
        )
        profile_combo["values"] = [
            f"{name} - {prof['name']}"
            for name, prof in BlockingWeightProfile.PROFILES.items()
        ]
        profile_combo.pack(side="left", padx=(0, 10))
        profile_combo.bind("<<ComboboxSelected>>", self.on_profile_change)

        # Description label
        self.desc_label = ttk.Label(
            profile_frame,
            text=BlockingWeightProfile.PROFILES[default_profile]["description"],
            foreground="gray",
        )
        self.desc_label.pack(side="left")

        # Custom weights frame (hidden by default)
        self.custom_frame = ttk.Frame(self)

        # Weight sliders
        weight_info = {
            "rr": "Reduction Ratio - How much to reduce comparisons",
            "h": "Entropy - Diversity of blocking keys",
            "g": "Balance (Gini) - Even distribution of block sizes",
            "p1": "Large Block Penalty - Penalize dominant blocks",
            "s1": "Singleton Penalty - Penalize too many unique records",
            "k": "Key Length - Shorter vs longer keys",
        }

        for i, (key, desc) in enumerate(weight_info.items()):
            row_frame = ttk.Frame(self.custom_frame)
            row_frame.grid(row=i, column=0, sticky="ew", pady=2)

            ttk.Label(row_frame, text=f"{key.upper()}:").grid(
                row=0, column=0, sticky="w", padx=(0, 10)
            )

            slider = ttk.Scale(
                row_frame, from_=0, to=1, orient="horizontal", length=200
            )
            slider.grid(row=0, column=1, padx=(0, 10))

            value_label = ttk.Label(row_frame, text="0.00", width=5)
            value_label.grid(row=0, column=2)

            ttk.Label(row_frame, text=desc, foreground="gray").grid(
                row=0, column=3, sticky="w", padx=(10, 0)
            )

            # Connect slider to label
            def make_callback(lbl, k):
                def update_label(value):
                    lbl.config(text=f"{float(value):.2f}")
                    self.custom_weights[k] = float(value)

                return update_label

            slider.config(command=make_callback(value_label, key))
            self.weight_entries[key] = (slider, value_label)

        # Validation button
        ttk.Button(
            self.custom_frame,
            text="Validate Weights",
            command=self.validate_custom_weights,
        ).grid(row=len(weight_info), column=0, columnspan=4, pady=10)

    def on_profile_change(self, event=None):
        profile_key = self.profile_var.get().split(" - ")[0]

        if profile_key == "custom":
            self.custom_frame.pack(fill="both", expand=True, pady=10)
        else:
            self.custom_frame.pack_forget()

        # Update description
        profile = BlockingWeightProfile.PROFILES.get(
            profile_key, BlockingWeightProfile.PROFILES["balanced"]
        )
        self.desc_label.config(text=profile["description"])

        # Update sliders with profile weights
        weights = profile["weights"]
        for key, (slider, label) in self.weight_entries.items():
            value = weights.get(key, 0.5)
            slider.set(value)
            label.config(text=f"{value:.2f}")

    def validate_custom_weights(self):
        is_valid, msg = BlockingWeightProfile.validate_weights(self.custom_weights)

        if is_valid:
            messagebox.showinfo("Valid", "Custom weights are valid!")
        else:
            messagebox.showerror("Invalid Weights", msg)

    def get_selected_weights(self):
        """Returns the weight configuration for the engine."""
        profile_key = self.profile_var.get().split(" - ")[0]

        if profile_key == "custom":
            return {
                "weight_profile": "custom",
                "custom_weights": self.custom_weights.copy(),
            }
        else:
            return {"weight_profile": profile_key, "custom_weights": None}


class BlockingColumnValidationDialog(tk.Toplevel):
    def __init__(self, parent, df, column_name):
        super().__init__(parent)
        self.title(f"Validation Report: {column_name}")
        self.geometry("600x500")

        # Validate the column
        validator = BlockingColumnValidator()
        is_valid, issues, metrics = validator.validate_column(df[column_name])

        # Header with status
        header_frame = ttk.Frame(self)
        header_frame.pack(fill="x", padx=10, pady=10)

        status_color = "green" if is_valid else "red"
        status_text = "✓ Valid for blocking" if is_valid else "✗ Not recommended"

        ttk.Label(
            header_frame,
            text=status_text,
            font=("TkDefaultFont", 12, "bold"),
            foreground=status_color,
        ).pack(side="left")

        # Quality score
        quality = metrics.get("quality_score", 0)
        ttk.Label(
            header_frame,
            text=f"Quality Score: {quality:.0%}",
            font=("TkDefaultFont", 10),
        ).pack(side="right")

        # Metrics
        metrics_frame = ttk.LabelFrame(self, text="Column Metrics", padding=10)
        metrics_frame.pack(fill="x", padx=10, pady=5)

        metrics_text = [
            f"Unique values: {metrics.get('n_unique', 'N/A'):,}",
            f"Uniqueness ratio: {metrics.get('unique_ratio', 0):.1%}",
            f"Null ratio: {metrics.get('null_ratio', 0):.1%}",
            f"Top value ratio: {metrics.get('top_value_ratio', 0):.1%}",
            f"Empty value ratio: {metrics.get('empty_ratio', 0):.1%}",
        ]

        for text in metrics_text:
            ttk.Label(metrics_frame, text=text).pack(anchor="w")

        # Issues
        if issues:
            issues_frame = ttk.LabelFrame(self, text="Issues Found", padding=10)
            issues_frame.pack(fill="both", expand=True, padx=10, pady=5)

            issues_text = tk.Text(issues_frame, wrap="word", height=6)
            issues_text.pack(fill="both", expand=True)

            for issue in issues:
                issues_text.insert("end", f"• {issue}\n")

            issues_text.config(state="disabled")

            # Suggestions
            suggestions = validator.suggest_preprocessing(df[column_name], issues)
            if suggestions:
                sugg_frame = ttk.LabelFrame(self, text="Suggestions", padding=10)
                sugg_frame.pack(fill="both", expand=True, padx=10, pady=5)

                sugg_text = tk.Text(sugg_frame, wrap="word", height=4)
                sugg_text.pack(fill="both", expand=True)

                for i, sugg in enumerate(suggestions, 1):
                    sugg_text.insert("end", f"{i}. {sugg}\n")

                sugg_text.config(state="disabled")

        # Close button
        ttk.Button(self, text="Close", command=self.destroy).pack(pady=10)


class BlockingAnalysisProgress(tk.Toplevel):
    """Real-time progress window for blocking analysis."""

    def __init__(self, parent):
        super().__init__(parent)
        self.title("Blocking Strategy Analysis")
        self.geometry("700x500")
        self.transient(parent)

        # Progress section
        prog_frame = ttk.LabelFrame(self, text="Progress", padding=10)
        prog_frame.pack(fill="x", padx=10, pady=10)

        # Progress bar
        self.progress_var = tk.DoubleVar()
        self.progress_bar = ttk.Progressbar(
            prog_frame, variable=self.progress_var, maximum=100
        )
        self.progress_bar.pack(fill="x", pady=(0, 5))

        # Status labels
        self.status_label = ttk.Label(prog_frame, text="Initializing...")
        self.status_label.pack(anchor="w")

        self.eta_label = ttk.Label(prog_frame, text="")
        self.eta_label.pack(anchor="w")

        # Current best section
        best_frame = ttk.LabelFrame(self, text="Current Best Strategy", padding=10)
        best_frame.pack(fill="x", padx=10, pady=5)

        self.best_info = tk.Text(best_frame, height=4, wrap="word")
        self.best_info.pack(fill="x")

        # Live evaluation section
        live_frame = ttk.LabelFrame(self, text="Live Evaluation", padding=10)
        live_frame.pack(fill="both", expand=True, padx=10, pady=5)

        # Create treeview for live results
        columns = ("Column", "Variant", "Length", "Score")
        self.tree = ttk.Treeview(live_frame, columns=columns, show="headings")

        for col in columns:
            self.tree.heading(col, text=col)
            self.tree.column(col, width=150)

        # Scrollbar
        scrollbar = ttk.Scrollbar(
            live_frame, orient="vertical", command=self.tree.yview
        )
        self.tree.configure(yscrollcommand=scrollbar.set)

        self.tree.pack(side="left", fill="both", expand=True)
        scrollbar.pack(side="right", fill="y")

        # Control buttons
        button_frame = ttk.Frame(self)
        button_frame.pack(fill="x", padx=10, pady=10)

        self.pause_button = ttk.Button(
            button_frame, text="Pause", command=self.toggle_pause
        )
        self.pause_button.pack(side="left", padx=5)

        ttk.Button(button_frame, text="Cancel", command=self.cancel_analysis).pack(
            side="left"
        )

        self.is_paused = False
        self.is_cancelled = False

    def update_progress(self, data: dict):
        """Called by the engine to update progress."""
        if self.is_cancelled:
            return

        # Update progress bar
        self.progress_var.set(data["percent"])

        # Update status
        self.status_label.config(
            text=f"Evaluated {data['current']}/{data['total']} strategies "
            f"({data['rate']:.1f} strategies/sec)"
        )

        # Update ETA
        eta_min = data["eta"] / 60
        self.eta_label.config(text=f"Time remaining: {eta_min:.1f} minutes")

        # Update best strategy
        if data.get("best_strategy"):
            best = data["best_strategy"]
            self.best_info.delete(1.0, tk.END)
            self.best_info.insert(
                tk.END,
                f"Column: {best['src_col']}\n"
                f"Preprocessing: {best['preproc']}\n"
                f"Key Length: {best['k_len']}\n"
                f"Score: {data['best_score']:.3f}",
            )

        # Add to live evaluation (only if score is decent)
        if data["current_score"] > 0.1:
            current = data["current_strategy"]
            self.tree.insert(
                "",
                0,
                values=(
                    current["src_col"],
                    current["preproc"],
                    current["k_len"],
                    f"{data['current_score']:.3f}",
                ),
            )

            # Keep only top 20 visible
            if len(self.tree.get_children()) > 20:
                self.tree.delete(self.tree.get_children()[-1])

        # Force GUI update
        self.update_idletasks()

    def toggle_pause(self):
        self.is_paused = not self.is_paused
        self.pause_button.config(text="Resume" if self.is_paused else "Pause")

    def cancel_analysis(self):
        self.is_cancelled = True
        self.destroy()


class RuleRow:
    """Encapsulates a single custom rule row in the UI."""

    def __init__(
        self,
        parent: ttk.Frame,
        app: "FuzzyMatcherApp",
        idx: int,
        remove_callback: Callable,
    ):
        self.app = app
        self.idx = idx
        self.remove_callback = remove_callback

        self.frame = ttk.Frame(parent)
        self.frame.columnconfigure(2, weight=1)
        self.frame.columnconfigure(6, weight=1)

        # Variables
        self.src_var = tk.StringVar()
        self.op_var = tk.StringVar()
        self.ref_var = tk.StringVar()
        self.threshold_var = tk.IntVar(value=85)
        self.weight_var = tk.IntVar(value=100)

        # Widgets
        src_cols = (
            list(self.app.source_df.columns) if self.app.source_df is not None else []
        )
        self.src_combo = ttk.Combobox(
            self.frame,
            textvariable=self.src_var,
            values=src_cols,
            state="readonly",
            width=25,
        )
        self.src_combo.bind("<<ComboboxSelected>>", self._on_source_column_change)

        self.op_combo = ttk.Combobox(
            self.frame,
            textvariable=self.op_var,
            values=list(OPERATORS.keys()),
            state="readonly",
            width=22,
        )
        self.op_combo.bind("<<ComboboxSelected>>", self._on_operator_change)

        ref_cols = self._get_ref_columns()
        self.ref_combo = ttk.Combobox(
            self.frame,
            textvariable=self.ref_var,
            values=ref_cols,
            state="readonly",
            width=25,
        )

        self.threshold_spinbox = ttk.Spinbox(
            self.frame, from_=1, to=100, textvariable=self.threshold_var, width=5
        )

        self.weight_spinbox = ttk.Spinbox(
            self.frame, from_=1, to=100, textvariable=self.weight_var, width=5
        )

        self.remove_btn = ttk.Button(
            self.frame, text="–", command=lambda: self.remove_callback(self), width=2
        )
        ToolTip(self.remove_btn, "Remove this condition")

        # Layout
        self._layout_widgets()

        # Initialize
        self.op_combo.set(list(OPERATORS.keys())[0])
        self._on_operator_change()

    def _layout_widgets(self):
        """Layout widgets in the frame."""
        self.src_combo.grid(row=0, column=0, padx=(0, 5), pady=2)
        self.op_combo.grid(row=0, column=1, padx=5, pady=2)
        self.ref_combo.grid(row=0, column=2, padx=5, pady=2, sticky="ew")
        self.threshold_spinbox.grid(row=0, column=3, padx=5, pady=2)
        ttk.Label(self.frame, text="Weight:").grid(
            row=0, column=4, padx=(10, 2), pady=2
        )
        self.weight_spinbox.grid(row=0, column=5, padx=(0, 5), pady=2)
        ttk.Frame(self.frame).grid(row=0, column=6, sticky="ew")  # Spacer
        self.remove_btn.grid(row=0, column=7, padx=5, pady=2)
        ttk.Separator(self.frame).grid(
            row=1, column=0, columnspan=8, sticky="ew", pady=(5, 0)
        )

    def _get_ref_columns(self) -> List[str]:
        """Get appropriate reference columns based on mode."""
        if self.app.mode.get() == "match" and self.app.ref_df is not None:
            return list(self.app.ref_df.columns)
        elif self.app.source_df is not None:
            return list(self.app.source_df.columns)
        return []

    def _on_source_column_change(self, event=None):
        """Auto-select reference column based on mappings."""
        selected_src = self.src_var.get()
        for mapping in self.app.mappings:
            if mapping.get("source") == selected_src:
                mapped_ref = mapping.get("ref")
                if mapped_ref and mapped_ref in self.ref_combo["values"]:
                    self.ref_var.set(mapped_ref)
                    return

        if selected_src and selected_src in self.ref_combo["values"]:
            self.ref_var.set(selected_src)

    def _on_operator_change(self, event=None):
        """Enable/disable widgets based on operator."""
        op_key = self.op_var.get()
        op_meta = OPERATORS.get(op_key, {})

        if op_meta.get("needs_ref", False):
            self.ref_combo.config(state="readonly")
            if not self.ref_var.get():
                self._on_source_column_change()
        else:
            self.ref_combo.config(state="disabled")
            self.ref_var.set("")

        self.threshold_spinbox.config(
            state="normal" if op_meta.get("needs_threshold", False) else "disabled"
        )

    def refresh_state(self):
        """Refresh the state based on current data."""
        # Update source columns
        src_cols = (
            list(self.app.source_df.columns) if self.app.source_df is not None else []
        )
        current_src = self.src_var.get()
        self.src_combo.config(values=src_cols)
        if current_src not in src_cols:
            self.src_var.set("")

        # Update reference columns
        ref_cols = self._get_ref_columns()
        current_ref = self.ref_var.get()
        self.ref_combo.config(values=ref_cols)
        if current_ref not in ref_cols:
            self.ref_var.set("")

        self._on_operator_change()

    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary for serialization."""
        op_key = self.op_var.get()
        op_meta = OPERATORS.get(op_key, {})

        return {
            "source_field": self.src_var.get(),
            "operator": op_meta.get("id", "unknown"),
            "ref_field": self.ref_var.get() if op_meta.get("needs_ref") else None,
            "threshold": (
                self.threshold_var.get() if op_meta.get("needs_threshold") else None
            ),
            "weight": self.weight_var.get(),
        }

    def grid(self, **kwargs):
        self.frame.grid(**kwargs)

    def destroy(self):
        self.frame.destroy()


class AlgorithmTransparencyPopup(tk.Toplevel):
    """A tooltip-like popup to explain algorithm suggestions with enhanced features."""

    def __init__(self, parent, algorithm: str, reason: str, rules: List[str] = None):
        super().__init__(parent)

        # --- Window Setup ---
        self.overrideredirect(True)  # Frameless window
        self.attributes("-topmost", True)  # Keep on top

        # Position near the algorithm dropdown in the parent
        parent.update_idletasks()
        x = parent.cbo_alg.winfo_rootx()
        y = parent.cbo_alg.winfo_rooty() + parent.cbo_alg.winfo_height() + 5

        # Ensure it stays on screen
        screen_width = self.winfo_screenwidth()
        if x + 320 > screen_width:  # Approximate width
            x = screen_width - 320

        self.geometry(f"+{x}+{y}")

        # --- Styling ---
        # Create a frame for content and border
        main_frame = ttk.Frame(self, style="Tooltip.TFrame", padding=10)
        main_frame.pack(fill=tk.BOTH, expand=True)

        # --- Content ---
        title_label = ttk.Label(
            main_frame,
            text="✨ Why this algorithm?",  # Added icon
            font=("TkDefaultFont", 10, "bold"),
            style="Tooltip.TLabel",
        )
        title_label.pack(anchor="w", pady=(0, 5))

        reason_text = f"Algorithm: {algorithm}\n\nReason: {reason}"
        reason_label = ttk.Label(
            main_frame,
            text=reason_text,
            wraplength=300,
            justify="left",
            style="Tooltip.TLabel",
        )
        reason_label.pack(anchor="w")

        # Add rules if provided
        if rules:
            rules_text = f"\nRules applied: {', '.join(rules[:3])}"
            if len(rules) > 3:
                rules_text += f" (+{len(rules)-3} more)"

            rules_label = ttk.Label(
                main_frame,
                text=rules_text,
                font=("TkDefaultFont", 8),
                style="Tooltip.Sublabel.TLabel",  # Use the full, correct style name
            )
            rules_label.pack(anchor="w", pady=(5, 0))

        # --- Animation and Lifetime ---
        self._is_fading_out = False
        self.attributes("-alpha", 0.0)  # Start transparent
        self._fade_in()

        # Auto-hide after 5 seconds with fade out
        self._hide_timer = self._after(5000, self._fade_out)

        # Close on click
        self.bind("<Button-1>", self._on_click_close)
        for widget in main_frame.winfo_children():
            widget.bind("<Button-1>", self._on_click_close)
        main_frame.bind("<Button-1>", self._on_click_close)

        # Make sure window can be closed by clicking anywhere nearby
        self.bind("<FocusOut>", lambda e: self._after(100, self._check_and_close))

    def _check_and_close(self):
        """Check if widget should close due to lost focus."""
        if self.focus_get() is None:
            self._fade_out()

    def _on_click_close(self, event=None):
        """Handle click to close the popup."""
        if hasattr(self, "_hide_timer") and self._hide_timer:
            self.after_cancel(self._hide_timer)
            self._hide_timer = None
        self._fade_out()

    def _fade_in(self):
        """Fade in animation."""
        try:
            alpha = self.attributes("-alpha")
            if alpha < 0.95:
                alpha += 0.1
                self.attributes("-alpha", alpha)
                self.after(20, self._fade_in)
        except tk.TclError:
            # Window might have been destroyed
            pass

    def _fade_out(self):
        """Fade out animation before destroying."""
        if getattr(self, "_is_fading_out", False):
            return
        self._is_fading_out = True

        try:
            alpha = self.attributes("-alpha")
            if alpha > 0:
                alpha -= 0.1
                self.attributes("-alpha", max(0, alpha))
                self.after(20, self._fade_out)
            else:
                self.destroy()
        except tk.TclError:
            # Window might have already been destroyed
            pass


class OperationCancelledError(Exception):
    """Custom exception for cancelled operations."""

    pass


# ── Results Popup Class ─────────────────────────────────────────────────────
class ResultsPopup(tk.Toplevel):
    """
    Stand‑alone results viewer with paging, inline diff view and review tagging.

    Parameters
    ----------
    parent : tk.Tk | tk.Toplevel
        The root window so that .grab_set() works correctly.
    df : pd.DataFrame
        The results to display.  A copy is made internally.
    title : str
        Window caption.
    mappings_list : Optional[List[Dict[str, Any]]]
        Mappings originally used for the match; drives diff panel.
    source_id_col_name : Optional[str]
    ref_id_col_name : Optional[str]
    export_callback : Optional[callable]
        If provided, called with (full_df) when user presses *Export*.
        Defaults to internal CSV exporter.

    Notes
    -----
    *Column selection*: we take up to 12 columns in this order:
        • user‑supplied ID columns
        • 'confidence_score'
        • mapped `s_*/r_*` columns
        • 'review_status'
    """

    ROW_HEIGHT = 24
    PAGE_SIZE_OPTIONS = [50, 100, 250, 500, 1000]
    DEFAULT_PAGE_SIZE = 100
    TAGS = {
        "hi": dict(background="#d1f2eb"),
        "mid": dict(background="#fcf3cf"),
        "low": dict(background="#fadbd8"),
        "approved": dict(background="#d4edda"),
        "rejected": dict(background="#f8d7da"),
        "maybe": dict(background="#fff3cd"),
    }

    def __init__(
        self,
        parent: tk.Tk,
        df: pd.DataFrame,
        title: str,
        mappings_list: Optional[List[Dict[str, Any]]] = None,
        source_id_col_name: Optional[str] = None,
        ref_id_col_name: Optional[str] = None,
        export_callback=None,
    ):
        super().__init__(parent)
        self.title(title)
        # self.transient(parent)  # Commented out to enable maximize button on Windows
        self.grab_set()

        # ---- data ----------------------------------------------------------
        self.full_df = df.copy(deep=True).reset_index(drop=True)
        if "review_status" not in self.full_df:
            self.full_df["review_status"] = pd.NA
        self.mappings = mappings_list or []
        self.offset = 0  # paging cursor
        self.page_size = self.DEFAULT_PAGE_SIZE
        self.sort_column = None
        self.sort_reverse = False
        self.sorted_df = self.full_df.copy()  # Working copy for sorting

        # ---- screen geometry ----------------------------------------------
        sw, sh = self.winfo_screenwidth(), self.winfo_screenheight()
        self.geometry(f"{int(sw*0.8)}x{int(sh*0.85)}+20+20")
        self.minsize(900, 600)

        # Enable window resizing with maximize button
        self.resizable(True, True)

        # ---- layout --------------------------------------------------------
        # Create main container frame to hold paned window and buttons
        main_container = ttk.Frame(self)
        main_container.pack(fill="both", expand=True)
        main_container.rowconfigure(0, weight=1)  # Paned window row expands
        main_container.columnconfigure(0, weight=1)

        paned = ttk.PanedWindow(main_container, orient=tk.VERTICAL)
        paned.grid(row=0, column=0, sticky="nsew", padx=10, pady=(10, 5))

        list_frame = ttk.Frame(paned)
        detail_frame = ttk.Frame(paned)
        paned.add(list_frame, weight=3)
        paned.add(detail_frame, weight=2)

        # -- list frame ------------------------------------------------------
        list_frame.rowconfigure(1, weight=1)
        list_frame.columnconfigure(0, weight=1)

        nav_bar = ttk.Frame(list_frame)
        nav_bar.grid(row=0, column=0, sticky="ew")
        nav_bar.columnconfigure(4, weight=1)  # First spacer
        nav_bar.columnconfigure(6, weight=1)  # Second spacer

        # Navigation buttons
        ttk.Button(
            nav_bar, text="« Prev", command=lambda: self._show_page(-self.page_size)
        ).grid(row=0, column=0)
        ttk.Button(
            nav_bar, text="Next »", command=lambda: self._show_page(self.page_size)
        ).grid(row=0, column=1, padx=5)

        # Page size selector - remove weight from column 3 to fix floating issue
        ttk.Label(nav_bar, text="Results per page:").grid(row=0, column=2, padx=(10, 2))
        self.page_size_var = tk.StringVar(value=str(self.page_size))
        page_size_combo = ttk.Combobox(
            nav_bar,
            textvariable=self.page_size_var,
            values=[str(s) for s in self.PAGE_SIZE_OPTIONS],
            state="readonly",
            width=6,
        )
        page_size_combo.grid(row=0, column=3, padx=2, sticky="w")
        page_size_combo.bind("<<ComboboxSelected>>", self._on_page_size_changed)

        # Spacer to push page info to center
        ttk.Frame(nav_bar).grid(row=0, column=4, sticky="ew")

        # Page info label (centered)
        self.lbl_page = ttk.Label(nav_bar)
        self.lbl_page.grid(row=0, column=5, padx=10)

        # Another spacer
        ttk.Frame(nav_bar).grid(row=0, column=6, sticky="ew")

        # Page jumper dropdown
        ttk.Label(nav_bar, text="Jump to:").grid(row=0, column=7, padx=(10, 2))
        self.page_jumper_var = tk.StringVar()
        self.page_jumper = ttk.Combobox(
            nav_bar, textvariable=self.page_jumper_var, state="readonly", width=10
        )
        self.page_jumper.grid(row=0, column=8, padx=2)
        self.page_jumper.bind("<<ComboboxSelected>>", self._on_page_jump_dropdown)

        # tree
        tree_container = ttk.Frame(list_frame)
        tree_container.grid(row=1, column=0, sticky="nsew")
        tree_container.rowconfigure(0, weight=1)
        tree_container.columnconfigure(0, weight=1)

        self.display_cols = self._choose_columns(source_id_col_name, ref_id_col_name)
        self.tree = ttk.Treeview(
            tree_container,
            columns=self.display_cols,
            show="headings",
            selectmode="browse",
            height=25,
        )
        for c in self.display_cols:
            # Set up column with sort command
            self.tree.heading(
                c, text=c, command=lambda col=c: self._sort_by_column(col)
            )
            width = 120 if "score" in c else 180
            self.tree.column(c, width=width, anchor=tk.W, stretch=True)

        vsb = ttk.Scrollbar(tree_container, orient="vertical", command=self.tree.yview)
        hsb = ttk.Scrollbar(
            tree_container, orient="horizontal", command=self.tree.xview
        )
        self.tree.configure(yscrollcommand=vsb.set, xscrollcommand=hsb.set)
        self.tree.grid(row=0, column=0, sticky="nsew")
        vsb.grid(row=0, column=1, sticky="ns")
        hsb.grid(row=1, column=0, sticky="ew")

        # tag colours
        for name, opts in self.TAGS.items():
            self.tree.tag_configure(name, **opts)

        self.tree.bind("<<TreeviewSelect>>", self._on_select)

        # -- detail frame ----------------------------------------------------
        detail_frame.columnconfigure(0, weight=1)
        detail_frame.rowconfigure(0, weight=1)

        canvas = tk.Canvas(detail_frame, borderwidth=0)
        d_scroll = ttk.Scrollbar(detail_frame, orient="vertical", command=canvas.yview)
        self.detail_inner = ttk.Frame(canvas)
        inner_id = canvas.create_window((0, 0), window=self.detail_inner, anchor="nw")
        canvas.configure(yscrollcommand=d_scroll.set)
        canvas.grid(row=0, column=0, sticky="nsew")
        d_scroll.grid(row=0, column=1, sticky="ns")

        def _sync(event):
            canvas.configure(scrollregion=canvas.bbox("all"))
            canvas.itemconfigure(inner_id, width=canvas.winfo_width())

        self.detail_inner.bind("<Configure>", _sync)
        canvas.bind("<Configure>", _sync)

        # summary
        lf_sum = ttk.LabelFrame(self.detail_inner, text="Selected match")
        lf_sum.pack(fill="x", padx=5, pady=5)
        self.lbl_score = ttk.Label(lf_sum, text="Score: N/A")
        self.lbl_score.pack(anchor="w")

        self.score_tree = ttk.Treeview(
            lf_sum,
            columns=("source_field", "ref_field", "score"),
            show="headings",
            height=4,
        )
        for h, w in (("source_field", 200), ("ref_field", 200), ("score", 60)):
            self.score_tree.heading(h, text=h.replace("_", " ").title())
            self.score_tree.column(h, width=w, anchor=tk.W)
        self.score_tree.pack(fill="x", padx=2, pady=5)

        # comparison
        self.lf_cmp = ttk.LabelFrame(self.detail_inner, text="Field comparison")
        self.lf_cmp.pack(fill="x", padx=5, pady=5)

        # -- bottom buttons --------------------------------------------------
        btn_bar = ttk.Frame(main_container)
        btn_bar.grid(row=1, column=0, pady=(5, 10))

        for txt, tag in (
            ("Approve (A)", "approved"),
            ("Reject (R)", "rejected"),
            ("Maybe (M)", "maybe"),
        ):
            ttk.Button(
                btn_bar, text=txt, command=lambda t=tag: self._set_review_status(t)
            ).pack(side="left", padx=5)

        ttk.Separator(btn_bar, orient="vertical").pack(side="left", fill="y", padx=10)

        ttk.Button(
            btn_bar, text="Export", command=(export_callback or self._default_export)
        ).pack(side="left", padx=5)
        ttk.Button(btn_bar, text="Close", command=self.destroy).pack(
            side="left", padx=5
        )

        # keyboard shortcuts
        for key, tag in [("a", "approved"), ("r", "rejected"), ("m", "maybe")]:
            self.bind(key, lambda e, t=tag: self._set_review_status(t))
            self.bind(key.upper(), lambda e, t=tag: self._set_review_status(t))

        # first paint
        self._show_page(0)

    def _on_page_size_changed(self, event=None):
        """Handle page size change from dropdown."""
        try:
            new_size = int(self.page_size_var.get())
            if new_size in self.PAGE_SIZE_OPTIONS:
                self.page_size = new_size
                # Reset to first page to avoid confusion
                self.offset = 0
                self._show_page(0)
        except ValueError:
            # Reset to current size if invalid
            self.page_size_var.set(str(self.page_size))

    def _on_page_jump_dropdown(self, event=None):
        """Handle page navigation from dropdown selection."""
        selected = self.page_jumper_var.get()
        if selected:
            # Extract page number from "Page X" format
            try:
                page_num = int(selected.split()[-1])
                self._show_page(absolute_page=page_num)
            except (ValueError, IndexError):
                pass  # Invalid selection, ignore

    def _sort_by_column(self, col):
        """Sort the entire dataset by the specified column."""
        # Determine sort order
        if self.sort_column == col:
            # Toggle sort direction
            self.sort_reverse = not self.sort_reverse
        else:
            # New column, default to ascending
            self.sort_column = col
            self.sort_reverse = False

        # Sort the dataframe
        try:
            # Special handling for numeric columns
            if col == "confidence_score" or "score" in col.lower():
                # Ensure numeric sorting
                self.sorted_df[col] = pd.to_numeric(
                    self.sorted_df[col], errors="coerce"
                )

            self.sorted_df = self.sorted_df.sort_values(
                by=col, ascending=not self.sort_reverse, na_position="last"
            ).reset_index(drop=True)

            # Update column headers to show sort indicator
            for c in self.display_cols:
                if c == col:
                    arrow = " ▼" if self.sort_reverse else " ▲"
                    self.tree.heading(
                        c,
                        text=c + arrow,
                        command=lambda col=c: self._sort_by_column(col),
                    )
                else:
                    self.tree.heading(
                        c, text=c, command=lambda col=c: self._sort_by_column(col)
                    )

            # Refresh display (stay on current page if possible)
            current_page = (self.offset // self.page_size) + 1
            self._show_page(absolute_page=current_page)

        except Exception as e:
            log.error(f"Error sorting by column {col}: {e}")
            messagebox.showerror(
                "Sort Error", f"Could not sort by column '{col}': {e}", parent=self
            )

    # ── helpers ──────────────────────────────────────────────────────────
    def _choose_columns(
        self, source_id: Optional[str], ref_id: Optional[str]
    ) -> List[str]:
        """Return ordered list of columns to show."""
        cols: List[str] = []
        prefer = [source_id, ref_id, "confidence_score"]
        for c in prefer:
            if c and c in self.full_df.columns and c not in cols:
                cols.append(c)

        # mapped s_/r_
        mapped = {f"s_{self._san(m['source'])}" for m in self.mappings} | {
            f"r_{self._san(m['ref'])}" for m in self.mappings
        }
        for c in self.full_df.columns:
            if c in mapped and c not in cols:
                cols.append(c)
                if len(cols) >= 12:
                    break

        if "review_status" in self.full_df and "review_status" not in cols:
            cols.append("review_status")

        # guarantee at least three
        if len(cols) < 3:
            cols.extend([c for c in self.full_df.columns if c not in cols][:3])

        return cols[:12]

    @staticmethod
    def _san(name: str) -> str:
        """Internal helper for mapping translations."""
        return sanitize_column_name(name)

    # ── paging ───────────────────────────────────────────────────────────
    def _show_page(self, delta: int = 0, absolute_page: int = None):
        """Show a page of results."""
        if absolute_page is not None:
            # Jump to specific page
            self.offset = (absolute_page - 1) * self.page_size
        else:
            # Relative navigation
            self.offset = self.offset + delta

        # Ensure offset is within bounds
        self.offset = min(max(self.offset, 0), max(len(self.sorted_df) - 1, 0))
        if len(self.sorted_df) - self.offset < self.page_size:
            self.offset = max(0, len(self.sorted_df) - self.page_size)

        page = self.sorted_df.iloc[self.offset : self.offset + self.page_size]
        self._populate_tree(page)

        # Update page info
        end = min(self.offset + len(page), len(self.sorted_df))
        current_page = (self.offset // self.page_size) + 1
        total_pages = max(
            1, (len(self.sorted_df) + self.page_size - 1) // self.page_size
        )
        self.lbl_page.config(
            text=f"Page {current_page} of {total_pages} ({self.offset+1:,}–{end:,} of {len(self.sorted_df):,})"
        )

        # Update page jumper dropdown
        # For very large page counts, show a subset of pages around current page
        if total_pages > 100:
            # Show first few, last few, and pages around current
            page_nums = set()

            # First 5 pages
            page_nums.update(range(1, min(6, total_pages + 1)))

            # Last 5 pages
            page_nums.update(range(max(1, total_pages - 4), total_pages + 1))

            # 10 pages before and after current
            page_nums.update(
                range(
                    max(1, current_page - 10), min(total_pages + 1, current_page + 11)
                )
            )

            # Convert to sorted list and format
            page_values = [f"Page {i}" for i in sorted(page_nums)]
        else:
            # For smaller page counts, show all pages
            page_values = [f"Page {i}" for i in range(1, total_pages + 1)]

        # Only update values if they've changed (to avoid resetting during selection)
        if self.page_jumper["values"] != page_values:
            self.page_jumper["values"] = page_values

        # Update current selection
        self.page_jumper_var.set(f"Page {current_page}")

        # auto‑select top row for convenience
        children = self.tree.get_children()
        if children:
            self.tree.selection_set(children[0])
            self.tree.focus(children[0])
            self.tree.see(children[0])
            self._on_select()

    # ── tree population & selection ─────────────────────────────────────
    def _populate_tree(self, page: pd.DataFrame):
        self.tree.delete(*self.tree.get_children())
        for ridx, row in page.iterrows():
            vals = [self._fmt_cell(row, c) for c in self.display_cols]
            # Remove color coding from main results window
            # tags = self._row_tags(row)
            self.tree.insert("", "end", iid=str(ridx), values=vals, tags=())

    @staticmethod
    def _fmt_cell(row, col):
        v = row.get(col, "")

        # 1. FIX THE CRASH: Handle duplicate columns returning a Series, which is the direct cause of the error.
        if isinstance(v, (pd.Series, np.ndarray)):
            # Safely extract the first valid value from the Series to prevent the ValueError
            v = next((item for item in v if pd.notna(item)), "")

        # 2. ADD FEATURE: Intelligently format list/array types for better display (Your Idea)
        if isinstance(v, list):
            if not v:
                return "[]"
            # Special summary for the field_score_details column
            if col == "field_score_details" and all(isinstance(i, dict) for i in v):
                return f"[{len(v)} fields]"
            # Generic summary for any other lists to prevent clutter
            str_v = str(v)
            return str_v[:50] + "..." if len(str_v) > 50 else str_v

        # 3. Original formatting for scalar values
        if col == "confidence_score" and isinstance(v, (int, float)):
            return f"{v:.2f}"

        return "" if pd.isna(v) else str(v)

    def _row_tags(self, row):
        tags = []
        try:
            sc = row.get("confidence_score", np.nan)
            if not pd.isna(sc):
                tags.append("hi" if sc >= 90 else "mid" if sc >= 80 else "low")
        except Exception:
            pass
        try:
            rs = row.get("review_status", "")
            if isinstance(rs, str) and rs.lower() in self.TAGS:
                tags.append(rs.lower())
        except Exception:
            pass
        return tuple(tags)

    def _on_select(self, _e=None):
        sel = self.tree.selection()
        if not sel:
            return
        ridx = int(sel[0])
        row = self.sorted_df.iloc[ridx]

        # update summary
        sc = row.get("confidence_score", "N/A")
        self.lbl_score.config(
            text=f"Score: {sc:.2f}" if isinstance(sc, (int, float)) else f"Score: {sc}"
        )

        # field‑score details
        self.score_tree.delete(*self.score_tree.get_children())
        details = []
        try:
            col = row.get("field_score_details")
            if isinstance(col, str):
                details = json.loads(col)
            elif isinstance(col, list):
                details = col
        except Exception:
            pass
        for d in details:
            self.score_tree.insert(
                "",
                "end",
                values=(
                    d.get("source_field"),
                    d.get("ref_field"),
                    f"{d.get('score', 0):.1f}",
                ),
            )

        # diff pane
        for w in self.lf_cmp.winfo_children():
            w.destroy()

        for i, m in enumerate(self.mappings):
            sf, rf = m["source"], m["ref"]
            s_val = self._value_for_field(row, sf, "s_")
            r_val = self._value_for_field(row, rf, "r_")

            ttk.Label(self.lf_cmp, text=f"{sf}:").grid(
                row=i, column=0, sticky="ne", padx=5, pady=2
            )
            self._diff_widget(self.lf_cmp, s_val, r_val).grid(
                row=i, column=1, sticky="ew"
            )
            ttk.Label(self.lf_cmp, text=f"{rf}:").grid(
                row=i, column=2, sticky="ne", padx=5, pady=2
            )
            self._diff_widget(self.lf_cmp, r_val, s_val).grid(
                row=i, column=3, sticky="ew"
            )
        self.lf_cmp.columnconfigure(1, weight=1)
        self.lf_cmp.columnconfigure(3, weight=1)

    def _value_for_field(self, row, field, prefix):
        # try prefixed then plain
        candidates = [f"{prefix}{self._san(field)}", field]
        for c in candidates:
            if c in row.index:
                v = row[c]
                return "" if pd.isna(v) else str(v)
        return ""

    # ── diff widget -----------------------------------------------------
    def _diff_widget(self, parent, s1, s2):
        """Create a text widget showing character-level diffs."""
        # Get parent background - LabelFrame uses 'bg' not 'background'
        try:
            bg_color = parent.cget("bg")
        except tk.TclError:
            # Fallback to system default
            bg_color = "SystemButtonFace"

        # Calculate height needed (allow wrapping)
        max_len = max(len(s1), len(s2))
        height = min(3, max(1, max_len // 60 + 1))  # Rough estimate

        t = tk.Text(
            parent,
            height=height,
            wrap=tk.WORD,
            borderwidth=0,
            relief=tk.FLAT,
            font=("TkDefaultFont", 9),
            background=bg_color,
        )

        # Configure tags for diff highlighting
        t.tag_configure("addition", background="#d4f8d4")  # Soft green for additions
        t.tag_configure("deletion", background="#f8d4d4")  # Soft red for deletions

        if s1 == s2:
            # No differences
            t.insert("1.0", s1)
        else:
            # Use difflib to get character-level differences
            s = difflib.SequenceMatcher(None, s1, s2, autojunk=False)

            for tag, i1, i2, j1, j2 in s.get_opcodes():
                if tag == "equal":
                    # Text is the same - no highlighting
                    t.insert(tk.END, s1[i1:i2])
                elif tag == "replace":
                    # Text was replaced - show old text as deletion and new text as addition
                    t.insert(tk.END, s1[i1:i2], "deletion")
                    t.insert(tk.END, s2[j1:j2], "addition")
                elif tag == "delete":
                    # Text was deleted from s1 (present in s1 but not in s2)
                    t.insert(tk.END, s1[i1:i2], "deletion")
                elif tag == "insert":
                    # Text was inserted (present in s2 but not in s1)
                    t.insert(tk.END, s2[j1:j2], "addition")

        t.config(state=tk.DISABLED)
        return t

    # ── review ----------------------------------------------------------
    def _set_review_status(self, status):
        sel = self.tree.selection()
        if not sel:
            messagebox.showwarning("No selection", "Select a row first.", parent=self)
            return
        ridx = int(sel[0])
        # Update both full_df and sorted_df
        self.full_df.at[ridx, "review_status"] = status
        self.sorted_df.at[ridx, "review_status"] = status
        # repaint just that row
        vals = list(self.tree.item(sel[0], "values"))
        if "review_status" in self.display_cols:
            vals[self.display_cols.index("review_status")] = status
        self.tree.item(
            sel[0], values=vals, tags=self._row_tags(self.sorted_df.iloc[ridx])
        )
        # advance
        nxt = self.tree.next(sel[0])
        if nxt:
            self.tree.selection_set(nxt)
            self.tree.focus(nxt)
            self.tree.see(nxt)
            self._on_select()

    # ── export ----------------------------------------------------------
    def _default_export(self):
        f = filedialog.asksaveasfilename(
            defaultextension=".csv",
            filetypes=[("CSV", "*.csv")],
            title="Export results",
        )
        if f:
            try:
                self.full_df.to_csv(f, index=False)
                messagebox.showinfo("Export complete", f"Saved to:\n{f}", parent=self)
            except Exception as exc:
                messagebox.showerror("Export failed", str(exc), parent=self)


class SpaceOptimizedUI:
    """Mixin class with space-saving UI methods."""

    def _set_visible(self, widget, visible: bool):
        """Helper to show/hide frames cleanly."""
        if visible:
            if hasattr(widget, "_last_pack_info"):
                widget.pack(**widget._last_pack_info)
        else:
            widget._last_pack_info = widget.pack_info()
            widget.pack_forget()

    def _set_initial_sash_position(self, event=None):
        """
        Centre the sash the first time the paned window is painted.
        Afterwards the user can resize it freely.
        """
        if self._sash_set:  # only run once
            return

        # Make sure the geometry information is valid
        self.field_pane.update_idletasks()

        try:
            # Check if we're in dedupe mode
            if self.mode.get() == "dedupe":
                # No sash in dedupe mode (only one pane)
                self._sash_set = True
                return

            total = self.field_pane.winfo_width()
            panes = self.field_pane.panes()

            # Only set sash if there are 2 panes and we have valid width
            if len(panes) >= 2 and total > 0:
                self.field_pane.sashpos(0, total // 2)  # 50%
                self._sash_set = True
        except tk.TclError:
            # Ignore errors - might be in transition between modes
            pass

    def _build_top_section(self, parent: ttk.Frame) -> ttk.Frame:
        """Ultra‑compact, 50 / 50 loader row."""
        frame = ttk.LabelFrame(parent, text="Data Input", padding=4)

        # ── 0. Match‑mode selector ────────────────────────────────────────────
        mode_row = ttk.Frame(frame)
        mode_row.grid(row=0, column=0, columnspan=2, sticky="w")
        ttk.Radiobutton(
            mode_row,
            text="Match External File",
            variable=self.mode,
            value="match",
            command=self.refresh_ui,
        ).pack(side="left")
        ttk.Radiobutton(
            mode_row,
            text="Detect Duplicates",
            variable=self.mode,
            value="dedupe",
            command=self.refresh_ui,
        ).pack(side="left", padx=12)

        # ── 1. Source slot ────────────────────────────────────────────────────
        self.src_slot = self._create_loader_slot(frame, role="source")
        self.src_slot.grid(row=1, column=0, sticky="ew", padx=(0, 4))
        # ── 2. Reference slot ────────────────────────────────────────────────
        self.ref_slot = self._create_loader_slot(frame, role="reference")
        self.ref_slot.grid(row=1, column=1, sticky="ew", padx=(4, 0))

        frame.columnconfigure(0, weight=1)
        frame.columnconfigure(1, weight=1)
        return frame

    def _create_loader_slot(self, parent, *, role: str):
        """Return a frame with Source/Reference widgets in one row."""
        f = ttk.Frame(parent)

        # Use grid instead of pack for better control
        col = 0

        # Label
        # Label - make it wider to accommodate "Reference:"
        ttk.Label(f, text=f"{role.title()}:", width=10, anchor="w").grid(
            row=0, column=col, sticky="w", padx=(0, 5)
        )
        col += 1

        # Source chooser
        var_src = tk.StringVar(value="CSV/File")
        src_combo = ttk.Combobox(
            f,
            textvariable=var_src,
            state="readonly",
            values=["CSV/File", "Salesforce"],
            width=10,
        )
        src_combo.grid(row=0, column=col, padx=5)
        col += 1

        # Action button
        action = ttk.Button(f, text="Browse…")
        action.grid(row=0, column=col, padx=5)
        col += 1

        # File / object info - give it more space and weight
        info = ttk.Label(f, text="No data", anchor="w", foreground="gray")
        info.grid(row=0, column=col, sticky="ew", padx=5)
        f.columnconfigure(col, weight=1)  # This column expands
        col += 1

        # ID combobox - use the main ID variables
        if role == "source":
            id_var = self.source_id_var
        else:
            id_var = self.ref_id_var

        id_combo = ttk.Combobox(f, textvariable=id_var, state="disabled", width=20)
        id_combo.grid(row=0, column=col, padx=(5, 0))

        # Store for later updates
        setattr(
            self,
            f"{role}_widgets",
            dict(
                src_type=var_src,
                action=action,
                info=info,
                id_combo=id_combo,
                id_var=id_var,
            ),
        )

        # Wiring for click
        action.configure(command=lambda r=role: self._loader_action(r))
        return f

    # ──────────────────────────────────────────────────────────────────────────────
    # Loader-slot callback
    # ──────────────────────────────────────────────────────────────────────────────
    def _loader_action(self, role: str) -> None:
        """
        Respond to the Browse… / Connect… button in the compact banner.

        Parameters
        ----------
        role : 'source' | 'reference'
            Identifies which dataframe the user is loading.
        """
        is_source = role == "source"

        # 1. What source type did the user pick?
        widgets = getattr(self, f"{role}_widgets")
        src_type = widgets["src_type"].get()

        # 2. CSV / Excel path
        if src_type == "CSV/File":
            fname = filedialog.askopenfilename(
                title="Select CSV or Excel file",
                filetypes=[("CSV / Excel", "*.csv *.xlsx *.xls"), ("All files", "*.*")],
            )
            if not fname:
                return  # user cancelled
            self._load_data_file(fname, is_source=is_source)

        # 3. Salesforce object
        elif src_type == "Salesforce":
            from .gui_compact_sf import CompactSalesforceDialog

            if not hasattr(self, "sf_integration") or self.sf_integration is None:
                self._connect_to_salesforce()  # existing helper
                if self.sf_integration is None:
                    return  # user aborted login
            CompactSalesforceDialog(self, self.sf_integration, is_source)

        # -- future integrations go here --

        # 4. After load, refresh banner + trees
        self._update_loader_banner()
        self.refresh_ui()

    def _update_loader_banner(self):
        for role in ("source", "reference"):
            if not hasattr(self, f"{role}_widgets"):
                continue
            widgets = getattr(self, f"{role}_widgets")
            df = self.source_df if role == "source" else self.ref_df
            if df is None:
                widgets["info"].configure(text="No data", foreground="gray")
                widgets["id_combo"].configure(state="disabled", values=[])
                widgets["id_var"].set("")  # Clear the selection
            else:
                path = getattr(self, f"{role}_path", None)
                if path:
                    fname = Path(path).name
                else:
                    fname = "Data"
                widgets["info"].configure(
                    text=f"{fname} ({len(df):,} rows)", foreground="black"
                )
                widgets["id_combo"].configure(state="readonly", values=list(df.columns))

                # Auto-select ID column if not already set
                current_id = widgets["id_var"].get()
                if not current_id or current_id not in df.columns:
                    # FAST PATH: Run basic detection without dialogs or telemetry
                    # to avoid annoying the user on every UI refresh.
                    try:
                        file_path = getattr(self, f"{role}_path", None)
                        auto_id = auto_select_id_column(
                            df, file_source=file_path, enable_telemetry=False
                        )
                        if auto_id:
                            widgets["id_var"].set(auto_id)
                    except Exception as e:
                        log.warning(
                            f"Fast-path ID auto-detection failed for role '{role}': {e}"
                        )

    def _on_match_mode_change(self):
        """Handle match mode change without recursion."""
        mode = self.mode.get()
        # Update UI based on mode
        if hasattr(self, "data_source_picker"):
            # Could update tile states based on mode
            pass
        # Then refresh UI
        self.refresh_ui()

    def _build_field_mapping_section(self, parent: ttk.Frame) -> ttk.Frame:
        """Build field mapping with eager-created trees and fixed layout."""
        self.mapping_frame = ttk.LabelFrame(parent, text="Field Mapping", padding=5)
        self.mapping_frame.rowconfigure(0, weight=1)  # Trees row
        self.mapping_frame.rowconfigure(1, weight=0)  # Controls row (fixed height)
        self.mapping_frame.columnconfigure(0, weight=1)

        # Create PanedWindow for 50/50 split
        self.field_pane = ttk.PanedWindow(self.mapping_frame, orient="horizontal")
        self.field_pane.grid(row=0, column=0, sticky="nsew", padx=5, pady=5)

        # Source frame
        src_frame = ttk.Frame(self.field_pane)
        src_frame.rowconfigure(1, weight=1)
        src_frame.columnconfigure(0, weight=1)

        ttk.Label(
            src_frame, text="Source Columns", font=("TkDefaultFont", 10, "bold")
        ).grid(row=0, column=0, sticky="w")

        # Create source tree EAGERLY (always exists)
        columns = ("health", "col_name", "algorithm", "fill_rate", "distinct", "type")
        self.tree_src = ttk.Treeview(
            src_frame, columns=columns, show="headings", selectmode="browse", height=10
        )
        self._configure_tree_columns(self.tree_src)
        self.tree_src.grid(row=1, column=0, sticky="nsew")
        self._setup_tree_tooltips(self.tree_src)

        # Add placeholder text when empty
        self.tree_src.insert(
            "", "end", values=("", "<- Load source data to see columns", "", "", "")
        )
        self.tree_src.tag_configure(
            "placeholder", foreground=PLACEHOLDER_COLOR, font=PLACEHOLDER_FONT
        )
        self.tree_src.item(self.tree_src.get_children()[0], tags=("placeholder",))

        vsb_src = ttk.Scrollbar(
            src_frame, orient="vertical", command=self.tree_src.yview
        )
        vsb_src.grid(row=1, column=1, sticky="ns")
        self.tree_src.configure(yscrollcommand=vsb_src.set)

        # Reference frame
        ref_frame = ttk.Frame(self.field_pane)
        ref_frame.rowconfigure(1, weight=1)
        ref_frame.columnconfigure(0, weight=1)

        self.lbl_tree_ref = ttk.Label(
            ref_frame, text="Reference Columns", font=("TkDefaultFont", 10, "bold")
        )
        self.lbl_tree_ref.grid(row=0, column=0, sticky="w")

        # Create reference tree EAGERLY
        columns = ("health", "col_name", "algorithm", "fill_rate", "distinct", "type")
        self.tree_ref = ttk.Treeview(
            ref_frame, columns=columns, show="headings", selectmode="browse", height=10
        )
        self._configure_tree_columns(self.tree_ref)
        self.tree_ref.grid(row=1, column=0, sticky="nsew")
        self._setup_tree_tooltips(self.tree_ref)

        # Add placeholder
        self.tree_ref.insert(
            "", "end", values=("", "<- Load reference data to see columns", "", "", "")
        )
        self.tree_ref.item(self.tree_ref.get_children()[0], tags=("placeholder",))

        vsb_ref = ttk.Scrollbar(
            ref_frame, orient="vertical", command=self.tree_ref.yview
        )
        vsb_ref.grid(row=1, column=1, sticky="ns")
        self.tree_ref.configure(yscrollcommand=vsb_ref.set)

        # Add frames to PanedWindow with equal weight
        self.field_pane.add(src_frame, weight=1)
        self.field_pane.add(ref_frame, weight=1)

        # Bind the configure event to set the initial sash position
        self._sash_set = False  # Initialize our flag
        self.field_pane.bind("<Configure>", self._set_initial_sash_position)

        # Mapping controls in separate frame (Point 4)
        controls_frame = ttk.Frame(self.mapping_frame)
        controls_frame.grid(row=1, column=0, sticky="ew", padx=5, pady=5)
        self._build_mapping_controls(controls_frame)

        # Initialize state flag
        self._trees_built = True

        return self.mapping_frame

    def _configure_tree_columns(self, tree):
        """Configure tree column headers, widths, and tag styles."""
        # Update column setup with better column name
        tree["columns"] = (
            "health",
            "col_name",
            "algorithm",
            "fill_rate",
            "distinct",
            "type",
        )
        tree["show"] = "tree headings"

        # Configure display
        tree.heading("#0", text="", anchor="w")
        tree.heading("health", text="Health")  # Changed from "Icon" to "Health"
        tree.heading("col_name", text="Column Name")
        tree.heading("algorithm", text="Algorithm")
        tree.heading("fill_rate", text="Fill Rate")
        tree.heading("distinct", text="Unique Values")
        tree.heading("type", text="Type")

        # Set column widths
        tree.column("#0", width=0, stretch=False)
        tree.column("health", width=60, stretch=False, anchor="center")
        tree.column("col_name", width=180, stretch=True)
        tree.column("algorithm", width=100, stretch=False)
        tree.column("fill_rate", width=80, stretch=False, anchor="center")
        tree.column("distinct", width=120, stretch=False, anchor="center")
        tree.column("type", width=80, stretch=False)

        # Configure tags
        tree.tag_configure("warning", foreground="#000000")
        tree.tag_configure("high_quality", font=("TkDefaultFont", 10, "bold"))
        tree.tag_configure("primary_key", font=("TkDefaultFont", 10, "bold"))
        tree.tag_configure(
            "placeholder", foreground=PLACEHOLDER_COLOR, font=PLACEHOLDER_FONT
        )

    def _build_mapping_controls(self, parent: ttk.Frame):
        """Build the mapping control buttons including weight."""
        # Center the controls
        inner_frame = ttk.Frame(parent)
        inner_frame.pack(
            fill=tk.X, anchor="center", padx=5, pady=(2, 4)
        )  # FIX: Added fill=tk.X

        # Weight control
        weight_frame = ttk.Frame(inner_frame)
        weight_frame.pack(side=tk.LEFT, padx=10)
        ttk.Label(weight_frame, text="Weight:").pack(side=tk.LEFT)
        self.spin_weight = ttk.Spinbox(
            weight_frame,
            from_=1,
            to=10,
            increment=1,
            width=5,
            textvariable=tk.IntVar(value=CONFIG["DEFAULT_WEIGHT"]),
        )
        self.spin_weight.pack(side=tk.LEFT, padx=(5, 0))

        # Separator
        ttk.Separator(inner_frame, orient="vertical").pack(
            side=tk.LEFT, fill="y", padx=10
        )

        # Mapping buttons
        self.btn_auto_match = ttk.Button(
            inner_frame,
            text="🎯 Auto-Match",
            command=self.on_auto_map_fields_clicked,
            style="Primary.TButton",
        )
        self.btn_auto_match.pack(side=tk.LEFT, padx=5)

        self.btn_add_mapping = ttk.Button(
            inner_frame, text="➕ Add Mapping", command=self.add_mapping
        )
        self.btn_add_mapping.pack(side=tk.LEFT, padx=5)

        self.btn_remove_mapping = ttk.Button(
            inner_frame,
            text="➖ Remove Selected",
            command=self.remove_mapping,
            state=tk.DISABLED,
        )
        self.btn_remove_mapping.pack(side=tk.LEFT, padx=5)

        self.btn_clear_mappings = ttk.Button(
            inner_frame,
            text="🗑️ Clear All",
            command=self.clear_all_mappings,
            state=tk.DISABLED,
        )
        self.btn_clear_mappings.pack(side=tk.LEFT, padx=5)

    def _build_active_mappings_section(self, parent: ttk.Frame) -> ttk.Frame:
        """Build collapsible active mappings."""
        self.active_frame = ttk.LabelFrame(parent, text="Active Mappings", padding=5)
        # Don't pack initially - will show when mappings exist

        # Store reference to parent for later packing
        self.active_frame_parent = parent
        self.active_frame_visible = False

        # Header with confirm button
        header_frame = ttk.Frame(self.active_frame)
        header_frame.pack(fill=tk.X, padx=5, pady=5)

        self.btn_confirm_mappings = ttk.Button(
            header_frame,
            text="Confirm & Analyze Mappings",
            command=self.on_confirm_mappings_clicked,
            style="Primary.TButton",
        )
        self.btn_confirm_mappings.pack(side="right")

        # Add weight adjust button
        self.btn_adjust_weight = ttk.Button(
            header_frame,
            text="Adjust Weight",
            command=self._show_weight_dialog,
            state=tk.DISABLED,
        )
        self.btn_adjust_weight.pack(side="right", padx=5)

        # Build the listbox for mappings
        listbox_frame = ttk.Frame(self.active_frame)
        listbox_frame.pack(fill=tk.BOTH, expand=True, padx=5, pady=5)

        # Listbox with scrollbar - use pack consistently
        self.map_list = tk.Listbox(
            listbox_frame,
            height=4,
            selectmode=tk.SINGLE,  # Compact height
        )
        self.map_list.pack(side=tk.LEFT, fill=tk.BOTH, expand=True)

        # Scrollbar
        scrollbar = ttk.Scrollbar(
            listbox_frame, orient="vertical", command=self.map_list.yview
        )
        scrollbar.pack(side=tk.RIGHT, fill=tk.Y)
        self.map_list.configure(yscrollcommand=scrollbar.set)

        # Bind selection event
        self.map_list.bind("<<ListboxSelect>>", self._on_mapping_selected)

        return self.active_frame

    def _show_mapping_trees(self, force: bool = False):
        """Show trees and hide placeholders when data is loaded."""
        # Handle source
        if hasattr(self, "source_df") and self.source_df is not None:
            # Hide placeholder, show tree (if using placeholder-based approach)
            if hasattr(self, "src_placeholder_frame"):
                self.src_placeholder_frame.grid_remove()
            if hasattr(self, "src_frame"):
                self.src_frame.grid()

            # Populate tree
            if hasattr(self, "tree_src") and self.tree_src:
                self._populate_single_tree(self.tree_src, self.source_df)

        # Handle reference - this is the critical part
        if (
            self.mode.get() == "match"
            and hasattr(self, "ref_df")
            and self.ref_df is not None
        ):
            # Hide placeholder, show tree
            if hasattr(self, "ref_placeholder_frame"):
                self.ref_placeholder_frame.grid_remove()
            if hasattr(self, "ref_frame"):
                self.ref_frame.grid()

            # Populate tree
            if hasattr(self, "tree_ref") and self.tree_ref:
                self._populate_single_tree(self.tree_ref, self.ref_df)

            # Make sure the reference tree is visible
            if hasattr(self, "lbl_tree_ref"):
                self.lbl_tree_ref.config(text="Reference Columns")

        # Show mapping controls when data is loaded
        if hasattr(self, "mapping_controls_frame"):
            self.mapping_controls_frame.grid(
                row=1, column=0, sticky="ew", padx=5, pady=(5, 0)
            )

    def _build_preprocessing_tab(self, parent) -> None:
        """Build preprocessing tab with collapsible header."""
        parent.columnconfigure(0, weight=1)

        # Chevron header
        header_frame = ttk.Frame(parent)
        header_frame.grid(row=0, column=0, sticky="ew", padx=10, pady=5)

        self.pre_expanded = tk.BooleanVar(value=self._load_preprocessing_state())

        # Clickable header
        self.pre_header = ttk.Frame(header_frame, cursor="hand2")
        self.pre_header.pack(fill=tk.X)

        self.chevron = ttk.Label(self.pre_header, text="▶", font=("TkDefaultFont", 10))
        self.chevron.pack(side=tk.LEFT, padx=(0, 5))

        ttk.Label(
            self.pre_header,
            text="Modern Preprocessing Engine",
            font=("TkDefaultFont", 11, "bold"),
        ).pack(side=tk.LEFT)

        ttk.Label(
            self.pre_header,
            text="(enhanced performance, comprehensive validation)",
            font=("TkDefaultFont", 9),
            foreground="gray",
        ).pack(side=tk.LEFT, padx=10)

        # Content frame
        self.pre_content = ttk.Frame(parent)

        # Normalization rules
        norm_frame = ttk.LabelFrame(
            self.pre_content, text=" Text Normalization Rules ", padding=10
        )
        norm_frame.pack(fill=tk.X, padx=20, pady=10)

        # Create checkboxes for each rule
        for i, (name, var) in enumerate(sorted(self.norm_rule_vars.items())):
            cb = ttk.Checkbutton(
                norm_frame, text=name.replace("_", " ").title(), variable=var
            )
            cb.grid(row=i // 2, column=i % 2, sticky="w", padx=10, pady=2)

        # Personal email filtering
        add_b2c_filter_controls(parent, self)

        # Bind click to toggle
        def toggle_preprocessing(event=None):
            current = self.pre_expanded.get()
            self.pre_expanded.set(not current)
            self._update_preprocessing_display()
            self._save_preprocessing_state(not current)

        # Bind mouse and keyboard events
        self.pre_header.bind("<Button-1>", toggle_preprocessing)
        self.pre_header.bind("<Return>", toggle_preprocessing)
        self.pre_header.bind("<space>", toggle_preprocessing)

        for child in self.pre_header.winfo_children():
            child.bind("<Button-1>", toggle_preprocessing)

        # Apply initial state
        self._update_preprocessing_display()

    def _update_preprocessing_display(self):
        """Update preprocessing section display based on expanded state."""
        if self.pre_expanded.get():
            self.chevron.config(text="▼")
            self.pre_content.pack(fill=tk.BOTH, expand=True)
        else:
            self.chevron.config(text="▶")
            self.pre_content.pack_forget()

    def _load_preprocessing_state(self) -> bool:
        """Load preprocessing expanded state from session."""
        if hasattr(self, "session_data") and self.session_data:
            return self.session_data.get("preprocessing_expanded", False)
        return False

    def _save_preprocessing_state(self, expanded: bool):
        """Save preprocessing expanded state to session."""
        if not hasattr(self, "session_data"):
            self.session_data = {}
        self.session_data["preprocessing_expanded"] = expanded
        # Don't call _save_session here as it may not exist yet

    def _on_mapping_selected(self, event=None):
        """Handle mapping selection with debouncing."""
        # Cancel any pending refresh
        if hasattr(self, "_selection_after_id"):
            self.after_cancel(self._selection_after_id)

        # Schedule the actual handler
        self._selection_after_id = self.after_idle(self._do_mapping_selection)

    def _do_mapping_selection(self):
        """Actual mapping selection handler."""
        selection = self.map_list.curselection()
        if selection:
            self.btn_remove_mapping.config(state=tk.NORMAL)
            self.btn_adjust_weight.config(state=tk.NORMAL)
            # Highlight selected columns without refreshing trees
            idx = selection[0]
            if 0 <= idx < len(self.mappings):
                mapping = self.mappings[idx]
                # Just highlight, don't rebuild
        else:
            self.btn_remove_mapping.config(state=tk.DISABLED)
            self.btn_adjust_weight.config(state=tk.DISABLED)


# ── Main Application Class ──────────────────────────────────────────────────


class FuzzyMatcherApp(tk.Tk, SpaceOptimizedUI):
    """Main application window for the Fuzzy Matcher GUI."""

    def __init__(self):
        """Initialize the Fuzzy Matcher application."""
        super().__init__()

        self.title("FoundryMatch - Advanced Fuzzy Matching")
        self.geometry("1400x900")
        self.minsize(1200, 700)

        # Initialize attributes
        self.progress_queue = (
            queue.Queue()
        )  # Use thread-safe queue instead of mp.Manager().Queue()
        self._poll_id: Optional[str] = None
        self._after_tokens = set()  # Track all after() timers
        self.CSV_ENGINE = os.environ.get(
            "FMATCH_CSV_ENGINE", "pyarrow"
        )  # Configurable CSV engine
        self.algo_selector = (
            PerColumnAlgorithmSelector()
        )  # Algorithm selector for multi-algo
        self.start_time = 0.0
        self.job_start_time = 0.0  # For compatibility with progress tracking
        self.cumulative_items = 0.0
        self.smoothed_eta_s: Optional[float] = None
        self.max_progress_reported = 0.0
        self.cumulative_comparisons = 0.0  # Track total comparisons for CPS
        self.last_cps_update_time = 0.0
        self.smoothed_cps: Optional[float] = None
        self._final_shown = False  # Track if 100% completion has been shown

        # Phase tracking
        self.current_phase = None  # Current processing phase
        self._last_phase = None  # For phase transition detection
        self._final_phase = False  # Track if we're in final phase

        # Event validation tracking
        self._last_total_items = 0
        self._last_cumulative_items = 0
        self.last_event_received_ts = 0.0

        # Processing start time tracking
        self.processing_start_time = 0
        self.phase_start_times = {}

        # Process pool for parallel operations
        self.pool = None
        self.engine_process_pool = None  # Engine-specific process pool

        # Data attributes
        self.source_df = None
        self.ref_df = None
        self.results_df = None
        self.source_path = None
        self.ref_path = None
        self.src_path = None  # Alias for source_path
        self.ref_path = None  # Reference path
        self.source_encoding = None  # Encoding for source file
        self.ref_encoding = None  # Encoding for reference file
        self.mappings = []
        self.mappings_confirmed = False  # Track if mappings have been confirmed
        self.current_processing = None
        self.processing = False  # Track if processing is active
        self.auto_determined_locking_strategy = None  # Auto-determined locking strategy
        self._popup_ref = None  # Track popup window reference
        self.rule_rows = []  # Track custom rule rows

        # SpaceOptimizedUI specific attributes
        self._sash_set = False  # Track if sash position has been set
        self._trees_built = False  # Track if trees have been built
        self.active_frame_visible = False  # Track active mappings frame visibility
        self.USE_RESERVOIR_SAMPLING = True  # Enable reservoir sampling
        self.column_algorithm_hints = {}  # Store algorithm hints per column
        self.session_data = {}  # For storing UI state
        self.progress_history = []  # Track progress events
        self.mapping_heuristics = MappingHeuristics()  # Initialize mapping heuristics
        self.global_fill_rate_thresholds = {  # Default fill rate thresholds
            "low": 0.25,
            "medium": 0.50,
            "high": 0.75,
        }
        self.telemetry = TelemetryCollector()  # Initialize telemetry collector

        # Tkinter variables
        self._init_tk_variables()

        add_salesforce_support_to_app(self)

        # UI References (will be set during build)
        self._init_ui_references()

        # Build UI
        self._load_custom_semantic_types()
        self.build_ui()

        # Protocol handlers
        self.protocol("WM_DELETE_WINDOW", self._on_close)

        # Deferred initialization
        self._after(10, self._load_last_session_deferred)
        self._after(
            100,
            lambda: threading.Thread(
                target=self._check_for_updates, daemon=True
            ).start(),
        )

    def _init_tk_variables(self):
        """Initialize all Tkinter variables."""
        self.mode = tk.StringVar(value="match")
        self.threshold = tk.IntVar(value=CONFIG["DEFAULT_THRESHOLD"])
        self.block_limit = tk.IntVar(value=CONFIG["DUP_BLOCK_LIMIT"])
        self.source_id_var = tk.StringVar()
        self.ref_id_var = tk.StringVar()
        self.rules_var = tk.StringVar()
        self.source_block_col_var = tk.StringVar()
        self.ref_block_col_var = tk.StringVar()
        self.results_mode = tk.StringVar(value="all")
        self.block_key_length_var = tk.IntVar(value=3)
        self.norm_rules_expanded = tk.BooleanVar(value=True)
        self.current_processing_mode = tk.StringVar(value="scale")
        self.active_mappings_expanded = tk.BooleanVar(value=False)
        self.algorithm_var = tk.StringVar(value="Ensemble Score")
        self.webhook_var = tk.StringVar()
        self.status_var = tk.StringVar(value="Ready")

        # Ensemble and algorithm selection variables
        self.ensemble_var = tk.BooleanVar(value=True)
        self.enable_multi_algo = tk.BooleanVar(value=True)
        self.enable_auto_optimization = True  # Enable auto-optimization by default
        self.strict_domain_var = tk.BooleanVar(value=False)

        self.norm_rule_vars = {
            name: tk.BooleanVar(value=False)
            for name in sorted(DEFAULT_NORMALIZATION_RULES)
        }

        # B2C Domain Filtering variables
        self.b2c_enabled_var = tk.BooleanVar(value=True)
        self.b2c_extract_email_var = tk.BooleanVar(value=True)
        self.b2c_company_fallback_var = tk.BooleanVar(value=True)
        self.b2c_filter_config = {
            "enabled": True,
            "extract_from_email": True,
            "company_fallback": True,
            "track_stats": True,
        }

        self.sf_integration = None
        self.sf_credentials = None

    def _init_ui_references(self):
        """Initialize UI widget references."""
        # Progress and status
        self.bar = None
        self.lbl_status = None
        self.lbl_progress = None

        # Column statistics for heuristics
        self._src_col_stats = {}
        self._ref_col_stats = {}
        self._src_rowcount = 0
        self._ref_rowcount = 0

        # Race-safe bounded operation tracking
        self._completed_operations = deque(maxlen=8)
        self._completion_lock = threading.Lock()

        # Results spilling
        self.results_df = None
        self.results_path = None
        self.results_row_count = 0
        self.RESULTS_MEMORY_THRESHOLD = 100_000
        self.total_disk_usage = 0  # Track total temp disk usage

        self.lbl_eta = None

        # Buttons
        self.btn_start = None
        self.btn_cancel = None
        self.btn_export = None
        self.btn_src = None
        self.btn_ref = None
        self.btn_confirm_mappings = None

        # Trees
        self.source_tree = None
        self.ref_tree = None
        self.mappings_tree = None
        self.tree_src = None
        self.tree_ref = None

        # Labels (blocking controls)
        self.lbl_src_block = None
        self.lbl_ref_block = None

        # Comboboxes (blocking controls)
        self.cbo_src_block = None
        self.cbo_ref_block = None

        # SpaceOptimizedUI will create source_widgets and reference_widgets
        # which contain the ID combos and file info labels

        # Frames and containers
        self.mapping_frame = None
        self.custom_rules_container = None
        self.source_tiles = None
        self.ref_tiles = None
        self.limit_frame = None
        self.field_pane = None
        self.map_list = None
        self.file_menu = None
        self.rules_container = None
        self.active_mappings_content_frame = None
        self.active_mappings_outer_lf = None

    @property
    def ui_state(self) -> UIState:
        """Get current UI state based on loaded data."""
        has_source = self.source_df is not None
        has_ref = self.ref_df is not None

        if has_source and has_ref:
            return UIState.BOTH_LOADED
        elif has_source:
            return UIState.SOURCE_ONLY
        elif has_ref:
            return UIState.REFERENCE_ONLY
        else:
            return UIState.EMPTY

    def _update_ui_for_state(self):
        """Update UI elements based on current state."""
        state = self.ui_state

        # Update button states
        can_process = state == UIState.BOTH_LOADED and self.mappings_confirmed
        if self.btn_start:
            self.btn_start.config(state="normal" if can_process else "disabled")

    # ─────────────────────────────────────────────────────────────────────────
    # Thread-Safe UI Helper Methods
    # ─────────────────────────────────────────────────────────────────────────

    def _askyesno_mainthread(self, title, message) -> bool:
        """Thread-safe yes/no dialog."""
        from queue import Queue

        q = Queue(maxsize=1)

        def _ask():
            q.put(messagebox.askyesno(title, message, parent=self))

        self.after(0, _ask)
        return q.get()  # blocks worker thread until answer

    # ─────────────────────────────────────────────────────────────────────────
    # Disk Spilling Helper Methods
    # ─────────────────────────────────────────────────────────────────────────

    def _sanitize_cell(self, v):
        """Sanitize a cell value for CSV writing with formula injection protection."""
        if v is None or (isinstance(v, float) and math.isnan(v)):
            return ""
        if isinstance(v, (bytes, bytearray)):
            return v.decode("utf-8", "replace")
        if isinstance(v, (dict, list)):
            return json.dumps(v, ensure_ascii=False)

        s = str(v)
        # Neutralize potential formula injection
        if s and s[0] in ("=", "+", "-", "@", "\t", "\r", "\n"):
            return "'" + s  # Prefix with single quote to neutralize
        return s

    def _spill_matches_to_csv_stream(self, matches, chunk_size=100_000):
        """Stream matches directly to CSV without building DataFrame."""
        # Full pass to get all possible columns (already in memory anyway)
        cols = set()
        for r in matches:
            if isinstance(r, dict):
                cols.update(r.keys())
        cols = sorted(cols)  # Consistent column order

        # Generate unique file path
        tmp_dir = Path(tempfile.gettempdir())
        out_path = tmp_dir / f"fm_results_{uuid.uuid4().hex[:8]}.csv"

        try:
            with open(out_path, "w", newline="", encoding="utf-8") as f:
                writer = csv.DictWriter(f, fieldnames=cols, extrasaction="ignore")
                writer.writeheader()

                # Write in chunks to avoid memory buildup
                for r in matches:
                    writer.writerow({c: self._sanitize_cell(r.get(c)) for c in cols})

            # Log file size and track disk usage
            file_size = out_path.stat().st_size
            self.total_disk_usage += file_size
            log.info(
                f"Streamed {len(matches)} rows to CSV: {out_path} ({file_size / 1_000_000:.1f}MB)"
            )
            log.info(
                f"Total temp disk usage: {self.total_disk_usage / 1_000_000:.1f}MB"
            )
            return str(out_path)
        except Exception as e:
            log.error(f"Failed to stream to CSV: {e}")
            raise

    def _store_results_with_spilling(self, result, op_id):
        """Store results with automatic disk spilling for large datasets."""
        matches = result.matches or []
        self.results_row_count = len(matches)

        if len(matches) <= self.RESULTS_MEMORY_THRESHOLD:
            # Small results stay in memory
            self.results_df = pd.DataFrame(matches)
            self.results_path = None
            log.info(f"Kept {len(matches)} rows in memory")
            return

        # Large results need spilling
        log.info(
            f"Spilling {len(matches)} rows to disk (threshold: {self.RESULTS_MEMORY_THRESHOLD})"
        )

        # Try Parquet first if available
        try:
            import pyarrow as pa
            import pyarrow.parquet as pq

            # Try to build DataFrame and save as Parquet
            try:
                df = pd.DataFrame(matches)
                tmp_dir = Path(tempfile.gettempdir())
                tmp_path = tmp_dir / f"fm_results_{uuid.uuid4().hex[:8]}.parquet"

                df.to_parquet(tmp_path, engine="pyarrow", compression="snappy")

                # Log file size and track disk usage
                file_size = tmp_path.stat().st_size
                self.total_disk_usage += file_size

                self.results_df = None
                self.results_path = str(tmp_path)
                log.info(
                    f"Spilled to Parquet: {tmp_path} ({file_size / 1_000_000:.1f}MB)"
                )
                log.info(
                    f"Total temp disk usage: {self.total_disk_usage / 1_000_000:.1f}MB"
                )
                return

            except Exception as parquet_error:
                log.warning(f"Parquet spill failed: {parquet_error}, trying CSV")

        except ImportError:
            log.info("PyArrow not available, using CSV for spilling")

        # Fall back to streaming CSV (most memory efficient)
        try:
            self.results_path = self._spill_matches_to_csv_stream(matches)
            self.results_df = None
        except Exception as e:
            # Last resort: keep in memory but warn
            log.error(
                f"All spill methods failed: {e}, keeping in memory (may cause OOM)"
            )
            self.results_df = pd.DataFrame(matches)
            self.results_path = None

    # ─────────────────────────────────────────────────────────────────────────
    # Async Processing Methods
    # ─────────────────────────────────────────────────────────────────────────

    async def _dedupe_worker_async(self, cfg: DedupeConfig) -> None:
        """Async worker for deduplication via engine_bridge."""
        op_id = cfg.job_id
        try:
            # The callback now accepts a ProgressEvent directly
            async def _progress(event: ProgressEvent):
                self.progress_queue.put(event)

            log.info(f"Starting async dedupe via engine_bridge for job {op_id}")

            result: DuplicateResultModel = await engine_bridge.dedupe(
                cfg, progress_callback=_progress
            )

            log.info(f"Dedupe completed successfully for job {op_id}")

            # Store results with disk spilling for large datasets
            self._store_results_with_spilling(result, op_id)
            # NO LONGER NEEDED: The engine's RunStats is the source of truth for the FINAL event.
            # log.info(f"GUI worker for job {op_id} has queued the final event.")

        except Exception as exc:
            log.error(f"Error in async dedupe worker: {exc}", exc_info=True)
            self.progress_queue.put(
                ProgressEvent(
                    phase=ProgressPhase.FINAL.value,
                    message=f"❌ Deduplication failed: {exc}",
                    extras={"op_id": op_id, "error": str(exc)},
                )
            )

    async def _match_worker_async(self, cfg: MatchConfig) -> None:
        """Async worker for matching via engine_bridge."""
        op_id = cfg.job_id
        try:
            # The callback now accepts a ProgressEvent directly
            async def _progress(event: ProgressEvent):
                self.progress_queue.put(event)

            log.info(f"Starting async match via engine_bridge for job {op_id}")

            result: MatchResultModel = await engine_bridge.match(
                cfg, progress_callback=_progress
            )

            log.info(f"Match completed successfully for job {op_id}")

            # Store results with disk spilling for large datasets
            self._store_results_with_spilling(result, op_id)
            # NO LONGER NEEDED: The engine's RunStats is the source of truth for the FINAL event.
            # log.info(f"GUI worker for job {op_id} has queued the final event.")

        except Exception as exc:
            log.error(f"Error in async match worker: {exc}", exc_info=True)
            self.progress_queue.put(
                ProgressEvent(
                    phase=ProgressPhase.FINAL.value,
                    message=f"❌ Matching failed: {exc}",
                    extras={"op_id": op_id, "error": str(exc)},
                )
            )

    def _run_async_orchestrator(
        self, op_mode: str, config_obj: Union[DedupeConfig, MatchConfig]
    ) -> None:
        """
        Orchestrates async processing in a background thread.
        This runs in a thread and uses asyncio.run to execute the async worker.
        """
        job_id = config_obj.job_id
        log.info(f"Async orchestrator starting for {op_mode} job {job_id}")

        try:
            if self._is_processing_cancelled:
                raise OperationCancelledError("Cancelled before async worker start")

            # Run the appropriate async worker
            if op_mode == "dedupe":
                asyncio.run(self._dedupe_worker_async(config_obj))
            elif op_mode == "match":
                asyncio.run(self._match_worker_async(config_obj))
            else:
                raise ValueError(f"Unknown operation mode: {op_mode}")

            # *** THE FIX - PART 1 ***
            # After the async worker is completely finished and results are stored,
            # send the true "job finished" signal to the UI thread's queue.
            log.info(
                f"Orchestrator finished for job {job_id}, queuing 'job_finished' signal."
            )
            self.progress_queue.put(("job_finished", {"op_id": job_id}))

        except Exception as e:
            log.error(f"Critical error in async orchestrator: {e}", exc_info=True)
            # Send a failure signal instead
            self.progress_queue.put(
                ("job_failed", {"op_id": job_id, "error": f"Orchestrator Error: {e}"})
            )

    def start_processing(self) -> None:
        """
        Main entry point for starting a processing job.
        Gathers UI state, builds config, and launches async processing.
        """
        log.info("Starting processing job")

        # FIRST THING - reset state (preserve locking strategy for processing)
        self.reset_run_state(preserve_locking=True)

        # THEN check if we can start
        if not self._validate_can_start():
            return

        # Initialize processing state
        self.start_time = time.time()
        self.job_start_time = time.time()
        self.current_operation_id = str(uuid.uuid4())
        self.processing = True
        self.current_phase = ProgressPhase.INIT  # Initialize phase tracking
        self._is_processing_cancelled = False

        # Log run start for state leak detection
        log.info(
            f"RUN_START {self.current_operation_id}: "
            f"mode={self.mode.get()}, "
            f"mappings={len(self.mappings)}, "
            f"threshold={self.threshold.get()}"
        )

        # Reset all progress tracking state
        self.cumulative_items = 0.0  # Reset cumulative items to 0
        self.max_progress_reported = 0.0
        self.smoothed_eta_s = None  # Reset ETA smoothing
        self.cumulative_comparisons = 0.0  # Reset comparison counter
        self.smoothed_cps = None  # Reset CPS smoothing
        self._final_shown = False  # Reset final completion flag
        self._last_total_items = 0  # Reset total items tracking for validation
        self._last_cumulative_items = 0  # Reset cumulative tracking for validation
        self._final_phase = False  # Crucial fix for ETA bug

        # Set progress to 0% at start
        self._set_progress_determinate(maximum=100, value=0)
        if self.lbl_progress:
            self.lbl_progress.config(text="0%")
        if self.lbl_eta:
            self.lbl_eta.config(text="ETA: Calculating...")

        # Update UI
        self.refresh_ui()

        # CRITICAL: Ensure progress polling is started
        if not self._poll_id:
            log.info("Starting progress queue polling for processing")
            self._poll_progress_queue()
        else:
            log.info(f"Progress polling already active: {self._poll_id}")

        try:
            op_mode = self.mode.get()

            # Build configuration
            if op_mode == "dedupe":
                config = self._build_dedupe_config()
            else:
                config = self._build_match_config()

            # Belt-and-suspenders guard for blocking configuration
            if (
                self.auto_determined_locking_strategy
                and self.auto_determined_locking_strategy.get("apply_blocking")
            ):
                missing = [
                    k
                    for k in (
                        "source_block_col",
                        "ref_block_col" if op_mode == "match" else "source_block_col",
                    )
                    if not self.auto_determined_locking_strategy.get(k)
                ]
                if missing:
                    messagebox.showwarning(
                        "Blocking Incomplete",
                        f"Blocking is enabled but columns are missing: {', '.join(missing)}. Results may be empty.",
                    )
                    log.warning(f"Missing blocking columns: {missing}")

            # Update status
            self.threadsafe_update_status(
                "🚀 Starting intelligent ensemble matching engine..."
            )

            # Submit to background thread
            self._ensure_pool_active()
            self.pool.submit(
                _log_thread_exceptions(self._run_async_orchestrator), op_mode, config
            )

        except Exception as e:
            log.error(f"Error starting processing: {e}", exc_info=True)
            messagebox.showerror("Error", f"Failed to start processing: {e}")
            self.processing = False
            self.refresh_ui()

    def _validate_can_start(self) -> bool:
        """Validate that processing can start."""
        if self.processing:
            messagebox.showwarning("Busy", "Processing already in progress")
            return False

        if not self.mappings_confirmed or not self.mappings:
            messagebox.showwarning(
                "Not Ready", "Please confirm mappings before processing"
            )
            return False

        if self.source_df is None:
            messagebox.showwarning("No Data", "Please load source data")
            return False

        if self.mode.get() == "match" and self.ref_df is None:
            messagebox.showwarning(
                "No Data", "Please load reference data for match mode"
            )
            return False

        return True

    def _build_dedupe_config(self) -> EnhancedDedupeConfig:
        """Build enhanced deduplication configuration with per-column algorithms from UI state."""
        strategy = getattr(self, "auto_determined_locking_strategy", None) or {}

        # Log blocking configuration being used
        if strategy.get("apply_blocking"):
            log.info(
                "Using dedupe blocking config: %s (k=%s, preproc=%s)",
                strategy.get("source_block_col", "NOT SET"),
                strategy.get("block_key_len", 3),
                strategy.get("preproc", "alnum"),
            )
        else:
            log.warning(
                "apply_blocking=False or no strategy found for dedupe; engine will fall back to ID column."
            )

        output_dir = CONFIG.get(
            "RESULTS_DIR", Path(tempfile.gettempdir()) / "fuzzy_match_results"
        )
        if not isinstance(output_dir, Path):
            output_dir = Path(output_dir)
        output_dir.mkdir(parents=True, exist_ok=True)

        if strategy.get("apply_blocking", True) and strategy.get("source_block_col"):
            rec_id_for_blocking = strategy.get("source_block_col")
        else:
            rec_id_for_blocking = self.source_id_var.get()

        # --- NEW: Logic from the enhanced version ---
        # Convert mappings to include per-column algorithm selections
        enhanced_maps = []
        for m in self.mappings:
            enhanced_maps.append(
                {
                    "source": m.get("source"),
                    "ref": m.get("source"),  # Ref is the same as source for dedupe
                    "weight": m.get("weight", 1.0),
                    "preferred_algo": (
                        m.get("preferred_algo")
                        if self.enable_multi_algo.get()
                        else None
                    ),
                    "algo_confidence": m.get("algo_confidence", 0.0),
                }
            )

        # Apply optimization ONLY if multi-algo is enabled AND auto-optimization is enabled
        if self.enable_multi_algo.get() and self.enable_auto_optimization:
            log.info(
                f"Multi-algo enabled with auto-optimization, optimizing {len(enhanced_maps)} mappings for dedupe..."
            )
            # Only use DataFrames for optimization, they're already loaded for preview
            optimized_maps = self._optimize_mapping_algorithms(
                enhanced_maps, self.source_df, None
            )
        elif self.enable_multi_algo.get():
            log.info("Using saved ensemble recipe without auto-optimization for dedupe")
            optimized_maps = enhanced_maps
        else:
            log.info("Multi-algo disabled, using original mappings for dedupe")
            optimized_maps = enhanced_maps
        # --- END of new logic ---

        # Call the main config builder, now using 'enhanced_maps' and adding the new multi-algo flags
        return build_dedupe_config(
            maps=optimized_maps,
            enable_multi_algo=self.enable_multi_algo.get(),
            # Pass path as string to avoid Path object surprises
            input_path=str(self.src_path) if self.src_path else None,
            rec_id_col=rec_id_for_blocking,
            # Pass encoding and CSV engine info
            encoding=self.source_encoding or "utf-8",
            csv_engine=getattr(self, "csv_engine", "c"),  # Default to C engine (faster)
            job_id=self.current_operation_id,
            # Use the recommended algorithm from blocking analysis if available
            algorithm=(
                strategy.get("recommended_algorithm")
                if strategy.get("recommended_algorithm")
                else self.algorithm_var.get()
            ),  # Use recommended algorithm or fallback to user selection
            threshold=self.threshold.get(),
            ruleset_name=self.rules_var.get() or "default",
            dup_block_limit=self.block_limit.get(),
            cache_path=str(CONFIG["CACHE_PATH"]),
            processing_mode=strategy.get("determined_engine_mode", "scale"),
            result_mode=self.results_mode.get(),
            output_dir=str(output_dir),
            apply_blocking=strategy.get("apply_blocking", True),
            block_key_length=strategy.get("block_key_len", 3),
            preproc_variant=strategy.get("preproc", "alnum"),
            strip_b2c_domains=self.b2c_enabled_var.get(),
            b2c_config=B2CFilterConfig(
                enabled=self.b2c_enabled_var.get(),
                extract_from_email=self.b2c_filter_config.get(
                    "extract_from_email", True
                ),
                company_fallback_enabled=self.b2c_filter_config.get(
                    "company_fallback", True
                ),
                stats_tracking=self.b2c_filter_config.get("track_stats", True),
            ),
        )

    # In gui.py, replace this entire function

    def _build_match_config(self) -> EnhancedMatchConfig:
        """Build enhanced match configuration with per-column algorithms from UI state."""
        strategy = getattr(self, "auto_determined_locking_strategy", None) or {}

        # Log blocking configuration being used
        if strategy.get("apply_blocking"):
            log.info(
                "Using blocking config: %s <-> %s (k=%s, preproc=%s)",
                strategy.get("source_block_col", "NOT SET"),
                strategy.get("ref_block_col", "NOT SET"),
                strategy.get("block_key_len", 3),
                strategy.get("preproc", "alnum"),
            )
        else:
            log.warning(
                "apply_blocking=False or no strategy found; engine will fall back to ID columns."
            )

        output_dir = CONFIG.get(
            "RESULTS_DIR", Path(tempfile.gettempdir()) / "fuzzy_match_results"
        )
        if not isinstance(output_dir, Path):
            output_dir = Path(output_dir)
        output_dir.mkdir(parents=True, exist_ok=True)

        # --- NEW: Logic from the enhanced version ---
        # Convert mappings to include per-column algorithm selections
        enhanced_maps = []
        for m in self.mappings:
            enhanced_maps.append(
                {
                    "source": m.get("source"),
                    "ref": m.get("ref"),
                    "weight": m.get("weight", 1.0),
                    "preferred_algo": (
                        m.get("preferred_algo")
                        if self.enable_multi_algo.get()
                        else None
                    ),
                    "algo_confidence": m.get("algo_confidence", 0.0),
                }
            )

        # Apply optimization ONLY if multi-algo is enabled AND auto-optimization is enabled
        if self.enable_multi_algo.get() and self.enable_auto_optimization:
            log.info(
                f"Multi-algo enabled with auto-optimization, optimizing {len(enhanced_maps)} mappings..."
            )
            # Only use DataFrames for optimization, they're already loaded for preview
            optimized_maps = self._optimize_mapping_algorithms(
                enhanced_maps, self.source_df, self.ref_df
            )
        elif self.enable_multi_algo.get():
            log.info("Using saved ensemble recipe without auto-optimization")
            optimized_maps = enhanced_maps
        else:
            log.info("Multi-algo disabled, using original mappings")
            optimized_maps = enhanced_maps
        # --- END of new logic ---

        # Log if using recommended algorithm
        if strategy.get("recommended_algorithm"):
            log.info(
                f"Using recommended algorithm from blocking analysis: {strategy.get('recommended_algorithm')}"
            )

        # Call the main config builder, now using 'enhanced_maps' and adding the new multi-algo flags
        # Note: This now returns an EnhancedMatchConfig object
        return build_match_config(
            maps=optimized_maps,
            enable_multi_algo=self.enable_multi_algo.get(),
            # Pass paths as strings to avoid Path object surprises
            src_path=str(self.src_path) if self.src_path else None,
            ref_path=str(self.ref_path) if self.ref_path else None,
            src_id_col=self.source_id_var.get(),
            ref_id_col=self.ref_id_var.get(),
            # Pass encoding and CSV engine info
            encoding=self.source_encoding or "utf-8",
            csv_engine=getattr(self, "csv_engine", "c"),  # Default to C engine (faster)
            job_id=self.current_operation_id,
            # Use the recommended algorithm from blocking analysis if available
            algorithm=(
                strategy.get("recommended_algorithm")
                if strategy.get("recommended_algorithm")
                else self.algorithm_var.get()
            ),  # Use recommended algorithm or fallback to user selection
            threshold=self.threshold.get(),
            ruleset_name=self.rules_var.get() or "default",
            cache_path=str(CONFIG["CACHE_PATH"]),
            processing_mode=strategy.get("determined_engine_mode", "scale"),
            result_mode=self.results_mode.get(),
            output_dir=str(output_dir),
            # Plumb the actual blocking decision down to the engine
            apply_blocking=bool(strategy.get("apply_blocking", False)),
            source_block_col=strategy.get("source_block_col"),
            ref_block_col=strategy.get("ref_block_col"),
            block_key_length=int(strategy.get("block_key_len", 3)),
            preproc_variant=strategy.get("preproc", "alnum"),
            strip_b2c_domains=self.b2c_enabled_var.get(),
            b2c_config=B2CFilterConfig(
                enabled=self.b2c_enabled_var.get(),
                extract_from_email=self.b2c_filter_config.get(
                    "extract_from_email", True
                ),
                company_fallback_enabled=self.b2c_filter_config.get(
                    "company_fallback", True
                ),
                stats_tracking=self.b2c_filter_config.get("track_stats", True),
            ),
        )

    def cancel_processing(self) -> None:
        """Cancel current processing job."""
        if not self.processing:
            return

        log.info("Cancelling processing job")
        self._is_processing_cancelled = True
        self.threadsafe_update_status("Cancelling...")

        # The async workers will check this flag and exit gracefully

    def finish_processing(
        self, op_id: str, error_message: Optional[str] = None
    ) -> None:
        """
        Clean up after processing completes or fails.
        Called from the progress queue handler.

        Uses a completion lock to prevent race conditions from multiple
        completion signals.
        """
        # Thread-safe check if operation was already completed
        with self._completion_lock:
            if op_id in self._completed_operations:
                log.debug(
                    f"Operation {op_id} already completed, ignoring duplicate finish signal"
                )
                return

            if op_id != self.current_operation_id:
                log.warning(f"Ignoring finish for old operation {op_id}")
                return

            # Mark this operation as completed (deque automatically maintains size limit)
            self._completed_operations.append(op_id)

        self.processing = False
        self._is_processing_cancelled = False

        # Clear the current operation ID on terminal state
        self.current_operation_id = None

        # Force garbage collection on terminal state
        gc.collect()

        # Stop progress polling
        if self._poll_id:
            log.info(f"Stopping progress polling: {self._poll_id}")
            self.after_cancel(self._poll_id)
            self._poll_id = None

        # Update UI
        if error_message:
            messagebox.showerror("Processing Failed", error_message)
        else:
            self.after_idle(self._display_results)

        # No need for manual cleanup - deque(maxlen=8) handles it automatically

        # Log run end for state leak detection
        log.info(
            f"RUN_END {op_id}: "
            f"success={error_message is None}, "
            f"timers_remaining={len(getattr(self, '_after_tokens', []))}, "
            f"queue_empty={self.progress_queue.empty()}"
        )

    def _display_results(self, _event=None):
        """Display results with efficient loading from disk."""
        has_results = False
        display_df = None

        if self.results_df is not None and not self.results_df.empty:
            has_results = True
            display_df = self.results_df

        elif self.results_path:
            # Check file size before loading
            file_size = Path(self.results_path).stat().st_size
            if file_size > 500_000_000:  # 500MB
                log.warning(f"Large results file: {file_size / 1_000_000:.1f}MB")

            # Load sample from disk for display
            try:
                if self.results_path.endswith(".parquet"):
                    try:
                        import pyarrow.parquet as pq

                        # Efficient batched reading
                        pf = pq.ParquetFile(self.results_path)
                        target_rows = 10_000
                        got_rows = 0
                        batches = []

                        for batch in pf.iter_batches(batch_size=5_000):
                            batches.append(batch)
                            got_rows += batch.num_rows
                            if got_rows >= target_rows:
                                break

                        if batches:
                            import pyarrow as pa

                            display_df = (
                                pa.Table.from_batches(batches)
                                .to_pandas()
                                .head(target_rows)
                            )
                            has_results = True
                            log.info(
                                f"Loaded {len(display_df)} rows from Parquet for display"
                            )

                    except ImportError:
                        log.warning("PyArrow not available, falling back to full load")
                        display_df = pd.read_parquet(self.results_path).head(10_000)
                        has_results = True

                else:  # CSV
                    display_df = pd.read_csv(self.results_path, nrows=10_000)
                    has_results = not display_df.empty
                    log.info(f"Loaded {len(display_df)} rows from CSV for display")

            except Exception as e:
                log.error(f"Error loading results from disk: {e}")
                has_results = False

        if has_results and display_df is not None:
            # Add row count info to title if loading sample
            title = "Match Results"
            if self.results_path and self.results_row_count > 10_000:
                title = f"Match Results (showing first 10,000 of {self.results_row_count:,} total)"

            self.show_results_popup(
                parent_for_toplevel=self,
                df=display_df,
                title=title,
                mappings_list=self.mappings,
                source_id_col_name=self.source_id_var.get(),
                ref_id_col_name=self.ref_id_var.get()
                if self.mode.get() == "match"
                else None,
            )
        else:
            messagebox.showinfo("No Results", "No matches found")

    def show_results_popup(
        self,
        parent_for_toplevel: tk.Tk,
        df: pd.DataFrame,
        title: str,
        source_df_full: Optional[pd.DataFrame] = None,
        ref_df_full: Optional[pd.DataFrame] = None,
        mappings_list: Optional[List[Dict[str, Any]]] = None,
        source_id_col_name: Optional[str] = None,
        ref_id_col_name: Optional[str] = None,
    ):
        """
        Display results using the new ResultsPopup class.
        """

        # Close any existing popup first
        if hasattr(self, "results_popup") and self.results_popup:
            try:
                self.results_popup.destroy()
            except:
                pass
            self.results_popup = None

        # Create export callback
        def export_callback():
            if hasattr(self, "export_results"):
                self.export_results(from_popup=True)

        # Create the ResultsPopup instance
        self.results_popup = ResultsPopup(
            parent=parent_for_toplevel,
            df=df,
            title=title,
            mappings_list=mappings_list,
            source_id_col_name=source_id_col_name,
            ref_id_col_name=ref_id_col_name,
            export_callback=export_callback,  # Pass the callback
        )

        # Set up close handler
        if self.results_popup:
            self.results_popup.protocol(
                "WM_DELETE_WINDOW", lambda: self._close_results_popup()
            )

    def _close_results_popup(self):
        """Clean up results popup."""
        if hasattr(self, "results_popup") and self.results_popup:
            try:
                self.results_popup.destroy()
            except:
                pass
            self.results_popup = None

    # ─────────────────────────────────────────────────────────────────────────
    # Locking Analysis Methods
    # ─────────────────────────────────────────────────────────────────────────

    def initiate_locking_analysis(self) -> None:
        """Start the locking (blocking) analysis process."""
        log.info("Initiating locking analysis")

        # Check for Salesforce source or reference and show warning
        if (
            hasattr(self, "src_path")
            and self.src_path
            and str(self.src_path).startswith("Salesforce:")
        ) or (
            hasattr(self, "ref_path")
            and self.ref_path
            and str(self.ref_path).startswith("Salesforce:")
        ):
            messagebox.showwarning(
                "Salesforce Limitation",
                "Blocking analysis isn't available for Salesforce sources yet.\n\n"
                "The system will use a default blocking strategy for your run.",
            )
            # Set a fallback strategy
            self.auto_determined_locking_strategy = {
                "apply_blocking": False,
                "strategy": "fallback",
                "stats": {"warning": "Salesforce data blocking analysis not available"},
            }
            self.threadsafe_update_status("Using default strategy for Salesforce data")
            return

        # Reset state for analysis (clear locking strategy for fresh analysis)
        self.reset_run_state(preserve_locking=False)

        if not self._validate_locking_analysis():
            return

        self._ensure_pool_active()
        self.threadsafe_update_status("Analyzing data for optimal strategy...")

        # Set analyzing flag
        self._is_analyzing = True

        # Generate operation ID for this analysis
        self.current_operation_id = str(uuid.uuid4())
        log.info(
            f"Starting locking analysis with operation ID: {self.current_operation_id}"
        )

        # Initialize progress tracking for locking analysis
        self.job_start_time = time.time()
        self.cumulative_items = 0.0
        self.smoothed_eta_s = None
        self.max_progress_reported = 0.0

        # Set progress bar to determinate mode
        self._set_progress_determinate(maximum=100, value=0)

        # Start polling progress queue
        if not self._poll_id:
            self._poll_progress_queue()

        # Prepare parameters
        current_mappings = list(self.mappings)
        current_mode = self.mode.get()
        current_ruleset = self.rules_var.get() or "default"

        # Submit to thread pool
        future = self.pool.submit(
            _log_thread_exceptions(self._execute_locking_analysis),
            self.source_df,
            self.ref_df,
            current_mappings,
            current_ruleset,
            current_mode,
        )
        future.add_done_callback(self._handle_locking_result)

    def _validate_locking_analysis(self) -> bool:
        """Validate that locking analysis can proceed."""
        if not self.mappings_confirmed or not self.mappings:
            log.warning("Locking analysis requires confirmed mappings")
            return False

        if self.source_df is None or self.source_df.empty:
            messagebox.showwarning("No Data", "Please load source data")
            return False

        if self.mode.get() == "match" and (self.ref_df is None or self.ref_df.empty):
            messagebox.showwarning(
                "No Data", "Please load reference data for match mode"
            )
            return False

        return True

    def _execute_locking_analysis(
        self,
        src_df: pd.DataFrame,
        ref_df: Optional[pd.DataFrame],
        mappings: List[Dict[str, Any]],
        ruleset: str,
        mode: str,
    ) -> Dict[str, Any]:
        """
        Execute locking analysis in a background thread, including algorithm
        suggestion and blocking strategy determination.
        """
        try:
            # Phase 1: Algorithm suggestion (25% of the work)
            log.info("Phase 1: Analyzing data for ensemble algorithm selection")
            self.progress_queue.put(
                ProgressEvent(
                    items_delta=25,
                    total_items=100,  # Analysis is treated as 100 units of work
                    message="Analyzing data for intelligent algorithm selection...",
                    extras={"op_id": self.current_operation_id},
                )
            )

            suggested_algo, suggestion_reason = self._suggest_algorithm(
                src_df, ref_df, mappings, mode
            )

            if self._is_processing_cancelled:
                raise OperationCancelledError("Analysis cancelled")

            # Phase 2: Blocking strategy via engine (50% of the work)
            log.info("Phase 2: Determining optimal blocking strategy")
            self.progress_queue.put(
                ProgressEvent(
                    items_delta=50,
                    total_items=100,
                    message="Evaluating blocking strategies...",
                    extras={"op_id": self.current_operation_id},
                )
            )

            self._ensure_engine_process_pool_active()
            engine_mode = BlockingMode.MATCH if mode == "match" else BlockingMode.DEDUPE

            # Use file paths instead of DataFrames to avoid pickling large data
            future = self.engine_process_pool.submit(
                determine_optimal_blocking_strategy_from_paths,
                src_path=self.src_path,
                ref_path=self.ref_path if mode == "match" else None,
                src_encoding=self.source_encoding,
                ref_encoding=self.ref_encoding if mode == "match" else None,
                mappings=mappings,
                mode=engine_mode,
                ruleset=ruleset,
                csv_engine=self.CSV_ENGINE,  # Pass the CSV engine setting
            )

            if self._is_processing_cancelled:
                future.cancel()
                raise OperationCancelledError("Analysis cancelled")

            strategy_result = future.result()

            # Final progress update (remaining 25% of the work)
            self.progress_queue.put(
                ProgressEvent(
                    items_delta=25,
                    total_items=100,
                    phase=ProgressPhase.FINAL.value,
                    message="Analysis complete. Finalizing results...",
                    extras={"op_id": self.current_operation_id},
                )
            )

            # Combine results and return
            return {
                **strategy_result,
                "suggested_algorithm_display_name": suggested_algo,
                "suggestion_reason": suggestion_reason,
                "determined_engine_mode": decide_processing_mode(src_df, ref_df),
            }

        except Exception as e:
            log.error(f"Error in locking analysis: {e}", exc_info=True)
            # Emit a final error event
            self.progress_queue.put(
                ProgressEvent(
                    items_delta=100 - self.cumulative_items,
                    total_items=100,
                    phase=ProgressPhase.FINAL.value,
                    message=f"Analysis failed: {e}",
                    extras={"op_id": self.current_operation_id, "error": str(e)},
                )
            )

            # Return a default strategy with the error information
            default_col = self.source_id_var.get() or (
                src_df.columns[0] if not src_df.empty else ""
            )
            return {
                "apply_blocking": True,
                "determined_engine_mode": decide_processing_mode(src_df, ref_df),
                "source_block_col": default_col,
                "ref_block_col": (
                    ref_df.columns[0]
                    if ref_df is not None and not ref_df.empty
                    else default_col
                ),
                "block_key_len": 3,
                "preproc": "alnum",
                "error": str(e),
                "suggested_algorithm_display_name": "WRatio",
                "suggestion_reason": "Using defaults due to analysis error",
            }

    def _suggest_algorithm(
        self,
        src_df: pd.DataFrame,
        ref_df: Optional[pd.DataFrame],
        mappings: List[Dict[str, Any]],
        mode: str,
    ) -> Tuple[str, str]:
        """Suggest best algorithm based on data characteristics."""
        # Aggregate column statistics
        stats = self._aggregate_column_stats(src_df, ref_df, mappings, mode)

        # Apply heuristic rules
        for rule in HEURISTIC_RULES:
            if rule["condition"](stats):
                return rule["algo_display_name"], rule["reason"]

        return "WRatio", "General-purpose default"

    def _aggregate_column_stats(
        self,
        src_df: pd.DataFrame,
        ref_df: Optional[pd.DataFrame],
        mappings: List[Dict[str, Any]],
        mode: str,
    ) -> Dict[str, Any]:
        """Aggregate statistics across mapped columns."""
        all_stats = []

        for mapping in mappings:
            src_col = mapping.get("source")
            ref_col = mapping.get("ref") if mode == "match" else src_col

            if not src_col:
                continue

            # Get column data
            src_series = src_df.get(src_col) if src_col in src_df.columns else None
            ref_series = None

            if mode == "match" and ref_df is not None and ref_col:
                ref_series = ref_df.get(ref_col) if ref_col in ref_df.columns else None
            elif mode == "dedupe" and ref_col:
                ref_series = src_df.get(ref_col) if ref_col in src_df.columns else None

            # Calculate stats
            if src_series is not None or ref_series is not None:
                stats = self._calculate_column_stats(src_series, ref_series)
                if stats.get("has_data"):
                    all_stats.append(stats)

        # Weighted average
        if not all_stats:
            return {"has_data": False}

        return self._weighted_average_stats(all_stats)

    def _calculate_column_stats(
        self, series1: Optional[pd.Series], series2: Optional[pd.Series]
    ) -> Dict[str, Any]:
        """Calculate statistics for column data."""
        stats = {
            "avg_len": 0.0,
            "avg_tokens": 0.0,
            "numeric_char_ratio": 0.0,
            "alpha_char_ratio": 0.0,
            "distinct_ratio": 0.0,
            "has_data": False,
        }

        # Combine non-null data
        data_list = []
        if series1 is not None:
            clean1 = series1.dropna().astype(str)
            if not clean1.empty:
                data_list.append(clean1)

        if series2 is not None:
            clean2 = series2.dropna().astype(str)
            if not clean2.empty:
                data_list.append(clean2)

        if not data_list:
            return stats

        combined = pd.concat(data_list)
        if combined.empty:
            return stats

        stats["has_data"] = True

        # Calculate metrics
        stats["distinct_ratio"] = combined.nunique() / len(combined)
        stats["avg_len"] = combined.str.len().mean()
        stats["avg_tokens"] = combined.str.split().str.len().mean()

        total_chars = combined.str.len().sum()
        if total_chars > 0:
            stats["numeric_char_ratio"] = (
                combined.str.count(r"[0-9]").sum() / total_chars
            )
            stats["alpha_char_ratio"] = (
                combined.str.count(r"[a-zA-Z]").sum() / total_chars
            )

        # Handle NaN values
        for key in ["avg_len", "avg_tokens", "numeric_char_ratio", "alpha_char_ratio"]:
            if pd.isna(stats[key]):
                stats[key] = 0.0

        return stats

    def _weighted_average_stats(
        self, stats_list: List[Dict[str, Any]]
    ) -> Dict[str, Any]:
        """Calculate weighted average of statistics."""
        result = {
            "avg_len": 0.0,
            "avg_tokens": 0.0,
            "numeric_char_ratio": 0.0,
            "alpha_char_ratio": 0.0,
            "distinct_ratio": 0.0,
            "has_data": True,
        }

        total_weight = 0.0

        for stats in stats_list:
            # Use distinct_ratio as weight
            weight = max(stats.get("distinct_ratio", 0.0), 0.01)

            for key in [
                "avg_len",
                "avg_tokens",
                "numeric_char_ratio",
                "alpha_char_ratio",
            ]:
                result[key] += stats.get(key, 0.0) * weight

            result["distinct_ratio"] += stats.get("distinct_ratio", 0.0)
            total_weight += weight

        # Normalize
        if total_weight > 0:
            for key in [
                "avg_len",
                "avg_tokens",
                "numeric_char_ratio",
                "alpha_char_ratio",
            ]:
                result[key] /= total_weight

            result["distinct_ratio"] /= len(stats_list)

        return result

    # In gui.py, REPLACE the entire _handle_locking_result function with this:

    def _handle_locking_result(self, future: cf.Future) -> None:
        """Handle completion of locking analysis."""
        try:
            # Always clear analyzing flag first
            self._is_analyzing = False
            if self._poll_id:
                self.after_cancel(self._poll_id)
                self._poll_id = None

            self._set_progress_determinate(value=100)
            if self.lbl_progress:
                self.lbl_progress.config(text="100.0%")
            result = future.result()
            log.info(f"Locking analysis complete: {result}")

            # CRITICAL: Store the strategy FIRST, before any UI operations.
            self.auto_determined_locking_strategy = result

            # Check for fallback strategy (Salesforce or other issues)
            if result.get("strategy") == "fallback":
                warning_msg = result.get("stats", {}).get(
                    "warning", "Blocking analysis not available"
                )
                log.warning(f"Using fallback blocking strategy: {warning_msg}")
                self.threadsafe_update_status(f"Note: {warning_msg}")
                # Show warning banner to user
                messagebox.showwarning(
                    "Limited Analysis",
                    f"{warning_msg}\n\nUsing default blocking configuration.",
                )
                # Skip the detailed analysis window for fallback
                return

            # Update algorithm suggestion in the UI
            suggested_algo = result.get("suggested_algorithm_display_name")
            if suggested_algo and suggested_algo in ALGORITHMS:
                self.algorithm_var.set(suggested_algo)

            # --- START OF DIAGNOSTIC BLOCK ---
            try:
                # Ask Python where it found the BlockingTransparencyWindow class
                class_file_path = inspect.getfile(BlockingTransparencyWindow)
                log.info(
                    f"DIAGNOSTIC: Python is loading BlockingTransparencyWindow from: {class_file_path}"
                )
            except Exception as e:
                log.error(
                    f"DIAGNOSTIC: Could not inspect BlockingTransparencyWindow. Error: {e}"
                )
            # --- END OF DIAGNOSTIC BLOCK ---

            # Show the NEW transparency window with error handling
            try:
                try:
                    BlockingTransparencyWindow(
                        parent=self,
                        blocking_result=result,
                        source_df=self.source_df,
                        ref_df=self.ref_df if self.mode.get() == "match" else None,
                        mode=self.mode.get(),
                    )
                except tk.TclError as e:
                    if "bad window path name" in str(e):
                        # Window reference is stale, try after idle
                        self.after_idle(
                            lambda: BlockingTransparencyWindow(
                                parent=self,
                                blocking_result=result,
                                source_df=self.source_df,
                                ref_df=(
                                    self.ref_df if self.mode.get() == "match" else None
                                ),
                                mode=self.mode.get(),
                            )
                        )
                    else:
                        raise
            except Exception as e:
                # General catch-all for any UI errors
                log.warning(f"Failed to show transparency window: {e}")

            # Update blocking config and UI state (this part is correct)
            if result.get("apply_blocking"):
                self.blocking_config = BlockingConfigWrapper(
                    {
                        "original_src_col": result.get("source_block_col"),
                        "original_ref_col": result.get("ref_block_col"),
                        "block_key_length": result.get("block_key_len"),
                        "metrics": {
                            "entropy": result.get("raw_metrics", {}).get("H", 0.0),
                            "reduction_pct": result.get("raw_metrics", {}).get(
                                "RR", 0.0
                            )
                            * 100,
                            "quality_score": result.get("quality_score", 0.0),
                        },
                    }
                )
                self.source_block_col_var.set(result.get("source_block_col", ""))
                if self.mode.get() == "match":
                    self.ref_block_col_var.set(result.get("ref_block_col", ""))
                self.block_key_length_var.set(result.get("block_key_len", 3))
            else:
                self.blocking_config = None

            if result.get("raw_metrics", {}).get("domain_overlap", 0) == 0:
                # Special message for zero overlap
                self.threadsafe_update_status(
                    "✓ Analysis complete - Domain matching optimized for datasets with different naming conventions"
                )
            else:
                self.threadsafe_update_status(
                    "✓ Ensemble analysis complete - optimal algorithms selected"
                )

            self._set_progress_determinate(value=0)

        except Exception as e:
            log.error(f"Error handling locking result: {e}", exc_info=True)
            # Your existing error handling logic for failed analysis is fine here.
            # ...
        finally:
            self._is_analyzing = False
            if self._poll_id:
                self.after_cancel(self._poll_id)
                self._poll_id = None
            self.refresh_ui()

    # ─────────────────────────────────────────────────────────────────────────
    # Progress and Status Management
    # ─────────────────────────────────────────────────────────────────────────

    def _poll_progress_queue(self) -> None:
        """Poll the progress queue for ProgressEvent objects and custom signals."""
        if (not self.processing and not self._is_analyzing) or not self.winfo_exists():
            self._poll_id = None
            log.debug("Stopping progress queue polling - no active processing")
            return

        try:
            while not self.progress_queue.empty():
                try:
                    msg = self.progress_queue.get_nowait()

                    # Handle modern ProgressEvent objects
                    if isinstance(msg, ProgressEvent):
                        self._handle_progress_event_v2(msg)
                        continue

                    # *** THE FIX - PART 2 ***
                    # Handle custom tuple-based signals from the orchestrator
                    if isinstance(msg, tuple) and len(msg) == 2:
                        signal, data = msg
                        # Ensure data is a dictionary before calling .get()
                        if not isinstance(data, dict):
                            log.warning(
                                f"Received signal '{signal}' with non-dict data: {type(data)}"
                            )
                            data = {}

                        op_id = data.get("op_id")

                        if signal == "job_finished":
                            log.info(
                                f"GUI received 'job_finished' signal for op_id: {op_id}"
                            )
                            self.finish_processing(op_id)
                        elif signal == "job_failed":
                            log.error(
                                f"GUI received 'job_failed' signal for op_id: {op_id}"
                            )
                            self.finish_processing(
                                op_id, error_message=data.get("error")
                            )
                        continue
                    # *** END FIX PART 2 ***

                except queue.Empty:
                    break
                except Exception as e:
                    log.error(f"Error processing queue message: {e}", exc_info=True)

        except Exception as e:
            log.error(f"Error polling progress queue: {e}", exc_info=True)

        self._poll_id = self.after(250, self._poll_progress_queue)

    def _handle_progress_event_v2(self, event: ProgressEvent) -> None:
        """Handle V2 ProgressEvent with phase-aware status updates."""

        # Filter stale events from previous operations
        event_op_id = getattr(event, "op_id", None) or (
            event.extras.get("op_id")
            if hasattr(event, "extras") and event.extras
            else None
        )
        if (
            event_op_id
            and hasattr(self, "current_operation_id")
            and event_op_id != self.current_operation_id
        ):
            log.debug(
                f"Ignoring stale progress event from operation {event_op_id} (current: {self.current_operation_id})"
            )
            return

        # --- ADD THIS BLOCK ---
        # Initialize the start time on the very first event
        if not hasattr(self, "processing_start_time"):
            self.processing_start_time = event.timestamp_ns
        # --- END OF ADDED BLOCK ---

        # Update watchdog timestamp for ANY event
        self.last_event_received_ts = time.perf_counter()

        # Validate the progress event
        self._validate_progress_event(event)

        # Detect phase transitions
        if hasattr(self, "current_phase") and event.phase != self.current_phase:
            self._show_phase_transition(self.current_phase, event.phase)
            self.current_phase = event.phase

        # Store in history
        self.progress_history.append(
            {
                "timestamp": time.time(),
                "event": event,
                "cumulative_items": self.cumulative_items,
                "cumulative_comparisons": self.cumulative_comparisons,
            }
        )
        # Update cumulative totals
        self.cumulative_items += event.items_delta
        self.cumulative_comparisons += event.comparisons_delta
        self._last_cumulative_items = self.cumulative_items  # Track for validation

        # Total items is now tracked per event, not stored

        # Phase-specific handling
        if event.phase == ProgressPhase.INIT.value:
            self._handle_init_phase(event)
        elif event.phase == ProgressPhase.RUN.value:
            self._handle_run_phase(event)
        elif event.phase == ProgressPhase.FINAL.value:
            self._handle_final_phase(event)
        elif event.phase == ProgressPhase.HEARTBEAT.value:
            self._handle_heartbeat_phase(event)

        # Update common elements
        self._update_progress_display(event)
        self._update_eta_display(event)
        self._update_cps_display(event)

    def _handle_init_phase(self, event: ProgressEvent) -> None:
        """Handle initialization phase updates."""
        self.threadsafe_update_status(f"🚀 {event.message}")
        self._set_progress_determinate(maximum=100, value=0)

        # Store initialization metadata
        self.processing_start_time = time.perf_counter()
        self.phase_start_times = {"init": self.processing_start_time}
        # Reset the run phase timer
        self.run_phase_start_time = None

    def _handle_run_phase(self, event: ProgressEvent) -> None:
        """Handle running phase updates."""
        # Start the run-phase timer on the first RUN event
        if not hasattr(self, "run_phase_start_time") or not self.run_phase_start_time:
            self.run_phase_start_time = event.timestamp_ns
            self.run_phase_start_perf_counter = (
                time.perf_counter()
            )  # Store perf_counter for consistent timing
            self.phase_start_times["run"] = self.run_phase_start_time

        # Calculate progress percentage
        if event.total_items > 0:
            progress_pct = (self.cumulative_items / event.total_items) * 100.0
            # Clamp progress to 0-100%
            progress_pct = min(100.0, max(0.0, progress_pct))
        else:
            progress_pct = 0.0

        # Update progress bar
        self.bar["value"] = progress_pct
        self.lbl_progress.config(text=f"{progress_pct:.1f}%")

        # Log if progress exceeds 100%
        if self.cumulative_items > event.total_items:
            log.warning(
                f"Progress exceeds 100%: cumulative={self.cumulative_items}, total={event.total_items}"
            )

        # Status will be updated by _update_progress_display

    def _handle_final_phase(self, event: ProgressEvent) -> None:
        """
        Handle final phase updates. This now ONLY updates the UI to 100%
        and does NOT trigger the job completion logic.
        """
        # Safely handle extras (could be None from legacy events)
        extras = event.extras if event.extras is not None else {}
        if event.extras is None:
            log.debug("Normalized None extras to {} in FINAL phase event")

        op_id = extras.get("op_id")
        error = extras.get("error")

        log.info(
            f"_handle_final_phase: received op_id={op_id}, current_operation_id={self.current_operation_id}"
        )

        # A TRUE final event will have an op_id that matches the current job.
        if op_id and op_id == self.current_operation_id:
            if self._final_shown:
                return  # Avoid processing duplicate final events

            self._final_phase = True
            self._final_shown = True
            self.bar["value"] = 100.0
            self.lbl_progress.config(text="100.0%")
            self.lbl_eta.config(text="Complete")

            status_msg = f"✅ {event.message or 'Processing complete'}"
            if "b2c_stats" in extras:
                stats = extras["b2c_stats"]
                status_msg += f" | B2C filtered: {stats.get('filtered', 0):,}/{stats.get('total', 0):,} ({stats.get('percent', 0):.1f}%)"
            self.threadsafe_update_status(status_msg)

            # *** THE FIX - PART 3 ***
            # CRITICAL: DO NOT call finish_processing here.
            # The 'job_finished' signal from the orchestrator will handle it.
            # This prevents the race condition.
            if error:
                log.error(f"A final event was received with an error: {error}")

        else:
            # This handles intermediate FINAL events (e.g., from locking analysis).
            log.warning(
                f"FINAL event op_id mismatch or missing: got '{op_id}', expected '{self.current_operation_id}' - treating as intermediate step completion."
            )
            self.bar["value"] = 100.0
            self.lbl_progress.config(text="100.0%")
            self.threadsafe_update_status(f"✓ {event.message or 'Analysis complete'}")

    def _handle_heartbeat_phase(self, event: ProgressEvent) -> None:
        """Handle heartbeat updates for long operations."""
        # Safely handle extras (could be None from legacy events)
        extras = event.extras if event.extras is not None else {}

        # Don't update progress bar, just status
        status = event.message
        if "elapsed_s" in extras:
            status += f" ({extras['elapsed_s']:.0f}s elapsed)"

        self.threadsafe_update_status(f"💓 {status}")

    # In gui.py, inside the FuzzyMatcherApp class

    def _show_phase_transition(self, old_phase: str, new_phase: str) -> None:
        """Display phase transition in the UI."""
        phase_names = {
            ProgressPhase.INIT.value: "Initialization",
            ProgressPhase.RUN.value: "Processing",
            ProgressPhase.FINAL.value: "Finalizing",
            ProgressPhase.HEARTBEAT.value: "Heartbeat",
        }

        old_name = phase_names.get(old_phase, old_phase)
        new_name = phase_names.get(new_phase, new_phase)

        log.info(f"Phase transition: {old_name} -> {new_name}")

        # *** ADD THIS LINE TO UPDATE THE LABEL ***
        if hasattr(self, "lbl_phase") and self.lbl_phase.winfo_exists():
            self.lbl_phase.config(text=f"Phase: {new_name}")

        # Show transition in status bar with appropriate emoji
        if new_phase == ProgressPhase.RUN.value:
            self.threadsafe_update_status(f"⚡ Starting {new_name} phase...")
        elif new_phase == ProgressPhase.FINAL.value:
            self.threadsafe_update_status(f"🏁 Entering {new_name} phase...")

    # In gui.py

    def _update_eta_display(self, event: ProgressEvent) -> None:
        """Update ETA display with smoothing and correct time formatting."""
        if not hasattr(self, "lbl_eta") or not self.lbl_eta.winfo_exists():
            return

        if self._final_phase:
            self.lbl_eta.config(text="Complete")
            return

        # Check if the run phase has started and has a valid timestamp
        if (
            hasattr(self, "run_phase_start_time")
            and self.run_phase_start_time
            and self.run_phase_start_time > 0
        ):
            # Store perf_counter at run phase start for consistent timing
            if not hasattr(self, "run_phase_start_perf_counter"):
                self.run_phase_start_perf_counter = time.perf_counter()

            elapsed_s = time.perf_counter() - self.run_phase_start_perf_counter

            total_items = event.total_items
            if self.cumulative_items <= 0 or total_items <= 0 or elapsed_s < 1:
                self.lbl_eta.config(text="ETA: Calculating...")
                return

            progress_ratio = self.cumulative_items / total_items
            if progress_ratio >= 1.0:
                self.lbl_eta.config(text="Finalizing...")
                return

            if progress_ratio < 0.001:
                self.lbl_eta.config(text="ETA: Calculating...")
                return

            # Calculate raw remaining time
            total_estimated_time_s = elapsed_s / progress_ratio
            remaining_s = total_estimated_time_s - elapsed_s

            # Apply smoothing
            if self.smoothed_eta_s is None:
                self.smoothed_eta_s = remaining_s
            else:
                alpha = 0.05
                self.smoothed_eta_s = (
                    alpha * remaining_s + (1 - alpha) * self.smoothed_eta_s
                )

            eta_str = self._format_time_duration(self.smoothed_eta_s)
            self.lbl_eta.config(text=f"ETA: {eta_str}")
        else:
            # If run phase hasn't started or timestamp is invalid, keep calculating
            self.lbl_eta.config(text="ETA: Calculating...")

    def _update_cps_display(self, event: ProgressEvent) -> None:
        """Update comparisons per second display with formatting."""
        if not hasattr(self, "job_start_time") or self.job_start_time == 0:
            return

        elapsed_s = time.perf_counter() - self.job_start_time

        if elapsed_s > 0 and self.cumulative_comparisons > 0:
            instant_cps = self.cumulative_comparisons / elapsed_s

            # Smooth CPS for a more stable reading
            if self.smoothed_cps is None:
                self.smoothed_cps = instant_cps
            else:
                alpha = 0.3
                self.smoothed_cps = (
                    alpha * instant_cps + (1 - alpha) * self.smoothed_cps
                )

            # Format CPS text
            cps_text = self._format_rate(self.smoothed_cps, "CPS")
            total_text = self._format_number(self.cumulative_comparisons)

            if (
                hasattr(self, "lbl_cps")
                and self.lbl_cps
                and self.lbl_cps.winfo_exists()
            ):
                self.lbl_cps.config(text=f"{cps_text} | {total_text} total")

    def _update_progress_display(self, event: ProgressEvent) -> None:
        """Update progress bar and centralized status display."""
        # Guard against duplicate messages within 1 second
        current_time = time.time()
        if hasattr(self, "_last_status_message") and hasattr(self, "_last_status_time"):
            if (
                event.message == self._last_status_message
                and current_time - self._last_status_time < 1.0
            ):
                return  # Skip duplicate message

        # Update tracking
        self._last_status_message = event.message
        self._last_status_time = current_time

        # Only update status for RUN phase events with meaningful messages
        if event.phase == ProgressPhase.RUN.value and event.message:
            status_parts = [event.message]

            # Add rate information if available
            # Safely handle extras (could be None from legacy events)
            extras = event.extras if event.extras is not None else {}
            if "items_rate" in extras and extras["items_rate"] > 0:
                status_parts.append(f"{extras['items_rate']:.1f} items/s")

            self.threadsafe_update_status(" | ".join(status_parts))

        # Check if this was a batched event
        if event.extras and event.extras.get("batched", False):
            batch_size = event.extras.get("batch_size", 1)
            if batch_size > 5:  # Only show for significant batching
                # Add small indicator
                if hasattr(self, "lbl_batch_indicator"):
                    self.lbl_batch_indicator.config(
                        text=f"⚡{batch_size}x", foreground=CONFIG["COLORS"]["warning"]
                    )
                    # Hide after a moment
                    self._after(1000, lambda: self.lbl_batch_indicator.config(text=""))

    def _flash_progress_bar(self, color: str) -> None:
        """Briefly flash the progress bar with a color, safely."""
        # 1. Widget Safety Check: Immediately exit if the bar doesn't exist.
        if not hasattr(self, "bar") or not self.bar.winfo_exists():
            return

        # 2. Use after_idle for safer, non-blocking style restoration.
        original_style = self.bar["style"] or ""
        flash_style = "Flash.Horizontal.TProgressbar"

        # 3. Reuse style object for efficiency: create it only once.
        if not hasattr(self, "_flash_style_obj"):
            self._flash_style_obj = ttk.Style()

        self._flash_style_obj.configure(
            flash_style, troughcolor="white", background=color
        )

        # Apply the temporary flash style
        self.bar["style"] = flash_style

        # Schedule the style to be restored later, calling our new safe method
        self.after_idle(lambda: self._restore_bar_style(original_style))

    def _restore_bar_style(self, style: str) -> None:
        """Restore the progress bar's original style only if the widget still exists."""
        # This check is the reason for a separate method. It runs *after* the delay.
        if hasattr(self, "bar") and self.bar.winfo_exists():
            self.bar["style"] = style

    def _handle_progress_error(self, event: ProgressEvent) -> None:
        """Handle error information from progress events."""
        # Safely handle extras (could be None from legacy events)
        extras = event.extras if event.extras is not None else {}

        if "error" in extras:
            error_info = extras["error"]

            # Build contextual error message
            error_msg = f"Error during {event.phase}: {error_info}"

            # Add processing context
            if self.cumulative_items > 0:
                error_msg += f"\nProcessed {self.cumulative_items:,} items before error"

            if "elapsed_s" in extras:
                error_msg += f"\nElapsed time: {extras['elapsed_s']:.1f}s"

            log.error(error_msg)
            self.threadsafe_update_status(f"❌ {error_info}", is_error=True)

    def _validate_progress_event(self, event: ProgressEvent) -> None:
        """Validate progress event for suspicious patterns and log warnings."""
        # Check for zero items_delta in RUN phase
        if event.phase == ProgressPhase.RUN.value and event.items_delta == 0:
            log.warning(
                f"RUN phase event with items_delta=0: message='{event.message}', "
                f"comparisons_delta={event.comparisons_delta}, total_items={event.total_items}"
            )

        # Check for negative deltas (should never happen)
        if event.items_delta < 0:
            log.error(
                f"Negative items_delta detected: {event.items_delta} in phase {event.phase}"
            )

        if event.comparisons_delta < 0:
            log.error(
                f"Negative comparisons_delta detected: {event.comparisons_delta} in phase {event.phase}"
            )

        # Check total_items consistency
        if hasattr(self, "_last_total_items"):
            if event.total_items > 0 and self._last_total_items > 0:
                # Total items should not change during processing
                if event.total_items != self._last_total_items:
                    log.warning(
                        f"Total items changed from {self._last_total_items} to {event.total_items} "
                        f"during {event.phase} phase"
                    )

        # Update tracking
        if event.total_items > 0:
            self._last_total_items = event.total_items

        # Check for suspiciously large deltas
        if event.items_delta > 1000000:
            log.warning(
                f"Unusually large items_delta: {event.items_delta:,} in phase {event.phase}"
            )

        if event.comparisons_delta > 1000000000:
            log.warning(
                f"Unusually large comparisons_delta: {event.comparisons_delta:,} in phase {event.phase}"
            )

        # Check for progress going backwards
        if hasattr(self, "_last_cumulative_items"):
            new_cumulative = self._last_cumulative_items + event.items_delta
            if new_cumulative < self._last_cumulative_items and event.items_delta > 0:
                log.error(
                    f"Progress overflow detected: cumulative would be {new_cumulative} "
                    f"after adding {event.items_delta}"
                )

        # Validate phase transitions
        if event.phase == ProgressPhase.FINAL.value:
            if (
                event.total_items > 0
                and self.cumulative_items + event.items_delta < event.total_items * 0.95
            ):
                log.warning(
                    f"FINAL phase reached with only {self.cumulative_items + event.items_delta:,} "
                    f"items processed out of {event.total_items:,} expected"
                )

    def _set_progress_determinate(self, maximum: int = 100, value: int = 0) -> None:
        """Set progress bar to determinate mode."""
        if not self.bar or not self.bar.winfo_exists():
            return

        try:
            self.bar.stop()
        except tk.TclError:
            pass

        self.bar.config(mode="determinate", maximum=maximum, value=value)

    def _set_progress_indeterminate(self) -> None:
        """Set progress bar to indeterminate mode."""
        if not self.bar or not self.bar.winfo_exists():
            return

        self.bar.config(mode="indeterminate")
        self.bar.start(10)

    # ─────────────────────────────────────────────────────────────────────────
    # Data Loading Methods
    # ─────────────────────────────────────────────────────────────────────────

    def load_source(self) -> None:
        """Load source data file."""
        file_path = filedialog.askopenfilename(
            title="Select Source Data File",
            filetypes=[
                ("CSV files", "*.csv"),
                ("Excel files", "*.xlsx *.xls"),
                ("All files", "*.*"),
            ],
        )

        if not file_path:
            return

        self._load_data_file(file_path, is_source=True)

    def load_reference(self) -> None:
        """Load reference data file."""
        if self.mode.get() != "match":
            return

        file_path = filedialog.askopenfilename(
            title="Select Reference Data File",
            filetypes=[
                ("CSV files", "*.csv"),
                ("Excel files", "*.xlsx *.xls"),
                ("All files", "*.*"),
            ],
        )

        if not file_path:
            return

        self._load_data_file(file_path, is_source=False)

    def _load_data_file(self, file_path: str, is_source: bool) -> None:
        """Load a data file with enhanced reservoir sampling for column analysis."""
        try:
            # Check file size
            file_size = Path(file_path).stat().st_size
            if file_size > CONFIG["MAX_GUI_LOAD_BYTES"]:
                if not messagebox.askyesno(
                    "Large File",
                    f"This file is {file_size / 1e9:.1f} GB. Loading may be slow. Continue?",
                ):
                    return

            # Load file
            log.info(
                f"Loading {'source' if is_source else 'reference'} file: {file_path}"
            )

            if file_path.lower().endswith(".csv"):
                # Try multiple encodings
                encodings_to_try = ["utf-8", "latin-1", "cp1252", "iso-8859-1"]

                # First try to detect encoding
                try:
                    detected_encoding = detect_encoding(file_path)
                    if detected_encoding and detected_encoding not in encodings_to_try:
                        encodings_to_try.insert(0, detected_encoding)
                except Exception as e:
                    log.warning(f"Could not detect encoding: {e}")

                # Initialize reservoir samplers if enabled
                samplers = {}

                if self.USE_RESERVOIR_SAMPLING:
                    # First pass: identify domain columns and collect samples
                    domain_keywords = [
                        "domain",
                        "website",
                        "url",
                        "site",
                        "web",
                        "email",
                    ]

                    for encoding in encodings_to_try:
                        try:
                            # Read in chunks for reservoir sampling
                            for chunk_idx, chunk in enumerate(
                                pd.read_csv(
                                    file_path,
                                    encoding=encoding,
                                    chunksize=10000,
                                    low_memory=False,
                                )
                            ):
                                if chunk_idx == 0:
                                    # Initialize samplers for domain-like columns
                                    for col in chunk.columns:
                                        if any(
                                            kw in col.lower() for kw in domain_keywords
                                        ):
                                            samplers[col] = ReservoirSampler(
                                                k=DEFAULT_SAMPLE_SIZE
                                            )
                                            log.info(
                                                f"Initialized reservoir sampler for column: {col}"
                                            )

                                # Feed data to samplers
                                for col, sampler in samplers.items():
                                    if col in chunk.columns:
                                        for value in chunk[col].dropna():
                                            sampler.add(value)

                            # If we got here, encoding worked
                            break
                        except UnicodeDecodeError:
                            continue
                        except Exception as e:
                            log.debug(f"Chunked reading failed with {encoding}: {e}")
                            continue

                # Second pass: load full dataframe
                df = None
                encoding_used = None
                last_error = None

                for encoding in encodings_to_try:
                    try:
                        log.info(f"Trying to load CSV with encoding: {encoding}")
                        df = pd.read_csv(file_path, encoding=encoding, low_memory=False)
                        encoding_used = encoding
                        log.info(f"Successfully loaded CSV with encoding: {encoding}")
                        break
                    except UnicodeDecodeError as e:
                        last_error = e
                        log.debug(f"Failed with encoding {encoding}: {e}")
                        continue

                if df is None:
                    raise ValueError(
                        f"Could not load CSV file with any of the attempted encodings. Last error: {last_error}"
                    )

                encoding = encoding_used
            else:
                # Excel files
                df = pd.read_excel(file_path)
                encoding = None
                samplers = {}  # No sampling for Excel yet

            # Store samplers with the dataframe
            df.attrs["column_samplers"] = samplers

            # Successfully loaded data
            if is_source:
                self.source_df = df
                self.src_path = file_path
                self.source_encoding = encoding

                # Set row count and update label
                self._src_rowcount = (
                    int(self.source_df.shape[0]) if self.source_df is not None else 0
                )
                if hasattr(self, "lbl_src") and self.lbl_src:
                    self.lbl_src.config(text=f"Source: {self._src_rowcount:,} rows")

                # Compute column stats for heuristics
                self._src_col_stats = self._compute_col_stats(self.source_df)

                self._detect_and_apply_id(df, role="source", file_path=file_path)
                # Clear mappings if data changed
                if self.mappings and not getattr(self, "_loading_session", False):
                    if messagebox.askyesno(
                        "Clear Mappings?",
                        "Source data changed. Clear existing mappings?",
                    ):
                        self.clear_mappings()
            else:
                self.ref_df = df
                self.ref_path = file_path
                self.ref_encoding = encoding

                # Set row count and update label
                self._ref_rowcount = (
                    int(self.ref_df.shape[0]) if self.ref_df is not None else 0
                )
                if hasattr(self, "lbl_ref") and self.lbl_ref:
                    self.lbl_ref.config(text=f"Reference: {self._ref_rowcount:,} rows")

                # Compute column stats for heuristics
                self._ref_col_stats = (
                    self._compute_col_stats(self.ref_df)
                    if self.ref_df is not None
                    else {}
                )

                self._detect_and_apply_id(df, role="reference", file_path=file_path)

            # Update UI
            self._populate_column_trees()
            self.update_processing_mode_auto()
            self._calculate_global_fill_rate_thresholds()

            # Show mapping trees if using new UI
            if hasattr(self, "_show_mapping_trees"):
                self._show_mapping_trees()

            self._update_loader_banner()
            self.refresh_ui()

            # Check for mapping template
            if is_source:
                self._check_auto_load_mapping_template(file_path)

            log.info(f"Successfully loaded {len(df)} rows from {Path(file_path).name}")
            if samplers:
                log.info(
                    f"Collected reservoir samples for {len(samplers)} domain columns"
                )

        except Exception as e:
            log.error(f"Error loading file: {e}", exc_info=True)
            messagebox.showerror("Load Error", f"Failed to load file: {e}")

    # -- Public hooks expected by DataSourcePicker ---------------------
    def load_file(self, filename: str, is_source: bool = True) -> None:
        """Entry point used by the File tile to load a local dataset."""
        self._load_data_file(filename, is_source)

    def browse_file(self, is_source: bool = True) -> None:
        """Open a file-open dialog; keeps Picker logic simple."""
        from tkinter import filedialog

        filetypes = [
            ("All supported", "*.csv;*.xlsx;*.xls;*.parquet"),
            ("CSV", "*.csv"),
            ("Excel", "*.xlsx;*.xls"),
            ("Parquet", "*.parquet"),
            ("All files", "*.*"),
        ]
        fn = filedialog.askopenfilename(
            title=f"Select {'Source' if is_source else 'Reference'} File",
            filetypes=filetypes,
        )
        if fn:
            self.load_file(fn, is_source)

    def _ask_role_if_needed(self):
        """Ask user which slot to fill if both are already loaded."""
        if self.source_df is not None and self.ref_df is not None:
            # Create a simple dialog
            dialog = tk.Toplevel(self)
            dialog.title("Choose Data Role")
            dialog.geometry("300x150")
            dialog.transient(self)
            dialog.grab_set()

            # Configure dialog
            dialog.configure(bg=CONFIG["COLORS"]["bg"])
            dialog.resizable(False, False)

            result = {"role": None}

            ttk.Label(
                dialog,
                text="Both slots are filled. Replace which one?",
                font=("TkDefaultFont", 11),
            ).pack(pady=20)

            button_frame = ttk.Frame(dialog)
            button_frame.pack()

            def choose_source():
                result["role"] = True
                dialog.destroy()

            def choose_reference():
                result["role"] = False
                dialog.destroy()

            def on_key(event):
                """Handle keyboard shortcuts"""
                if event.keysym == "Return":
                    # Choose the focused button
                    focused = dialog.focus_get()
                    if focused and isinstance(focused, ttk.Button):
                        focused.invoke()
                elif event.keysym == "Escape":
                    dialog.destroy()

            src_btn = ttk.Button(
                button_frame,
                text="Replace Source",
                command=choose_source,
                style="Primary.TButton",
            )
            src_btn.pack(side=tk.LEFT, padx=10)

            ref_btn = ttk.Button(
                button_frame, text="Replace Reference", command=choose_reference
            )
            ref_btn.pack(side=tk.LEFT, padx=10)

            # Set initial focus and keyboard bindings
            dialog.focus_set()
            src_btn.focus_set()
            dialog.bind("<Key>", on_key)

            # Center the dialog
            dialog.update_idletasks()
            x = self.winfo_x() + (self.winfo_width() // 2) - (dialog.winfo_width() // 2)
            y = (
                self.winfo_y()
                + (self.winfo_height() // 2)
                - (dialog.winfo_height() // 2)
            )
            dialog.geometry(f"+{x}+{y}")

            self.wait_window(dialog)
            return result["role"]
        else:
            # Auto-detect which slot is empty
            return self.source_df is None

    # In gui.py, add these methods to the FuzzyMatcherApp class

    def _update_tree_intelligently(
        self, tree: ttk.Treeview, df: pd.DataFrame, tree_type: str = "source"
    ):
        """Update tree to avoid flicker by only changing, adding, or removing necessary items."""
        if not tree or not hasattr(tree, "winfo_exists") or not tree.winfo_exists():
            return

        current_items = {}
        try:
            for item in tree.get_children():
                values = tree.item(item)["values"]
                if len(values) > 1:
                    current_items[values[1]] = item  # Column 1 is col_name
        except Exception as e:
            log.debug(f"Error reading tree items: {e}")
            return

        seen_columns = set()

        for col in df.columns:
            seen_columns.add(col)
            stats = engine_get_column_stats(df[col])
            hints = self._get_column_ui_hints(stats)

            # Format display values
            icon_str = " ".join(hints["icons"])

            # Show the suggested algorithm based on column characteristics
            if hasattr(self, "enable_multi_algo") and self.enable_multi_algo.get():
                # In ensemble mode, show what algorithm would likely be used
                algo_name = self._suggest_algo_for_col(
                    col, is_source=(tree_type == "source")
                )
                log.debug(f"Column {col}: Suggested {algo_name} (ensemble mode)")
            else:
                # Show the selected global algorithm when not in ensemble mode
                algo_name = (
                    self.algorithm_var.get()
                    if hasattr(self, "algorithm_var")
                    else "WRatio"
                )
                log.debug(f"Column {col}: Using {algo_name} (non-ensemble mode)")

            # Store it for column hints
            if not hasattr(self, "column_algorithm_hints"):
                self.column_algorithm_hints = {}
            self.column_algorithm_hints[col] = algo_name

            fill_str = f"{stats['fill_rate'] * 100:.1f}%"
            distinct_str = (
                f"{stats['unique_count']:,} ({stats['distinct_ratio'] * 100:.1f}%)"
            )

            # Use friendly dtype instead of inferred_type
            dtype_str = self._friendly_dtype(
                self._src_col_stats.get(col, {}).get("dtype", "unknown")
                if tree_type == "source"
                else self._ref_col_stats.get(col, {}).get("dtype", "unknown")
            )

            new_values = (icon_str, col, algo_name, fill_str, distinct_str, dtype_str)

            # Determine tags for the item
            new_tags = []
            if hints.get("warnings"):
                new_tags.append("warning")
            elif "⭐" in hints["icons"]:
                new_tags.append("high_quality")
            elif "🔑" in hints["icons"]:
                new_tags.append("primary_key")

            if col in current_items:
                item_id = current_items[col]
                if (
                    list(tree.item(item_id)["values"]) != list(new_values)
                    or tree.item(item_id)["tags"] != new_tags
                ):
                    tree.item(item_id, values=new_values, tags=new_tags)
            else:
                item_id = tree.insert("", "end", values=new_values, tags=new_tags)

            # Store tooltip data with the item ID
            if not hasattr(self, "column_tooltips"):
                self.column_tooltips = {}
            if not hasattr(self, "column_hints"):
                self.column_hints = {}

            self.column_tooltips[item_id] = "\n".join(hints["tooltips"])
            self.column_hints[item_id] = {
                "column_name": col,
                "tree_type": tree_type,
                "hints": hints,
                "stats": stats,
            }

        # Remove columns that no longer exist
        for col_name, item_id in current_items.items():
            if col_name not in seen_columns:
                tree.delete(item_id)
                # Clean up tooltip data
                if hasattr(self, "column_tooltips") and item_id in self.column_tooltips:
                    del self.column_tooltips[item_id]
                if hasattr(self, "column_hints") and item_id in self.column_hints:
                    del self.column_hints[item_id]

    def _batch_record_telemetry(self, df: pd.DataFrame, tree_type: str):
        """Submits telemetry recording for all columns to a background thread."""
        if (
            hasattr(self, "telemetry")
            and self.telemetry.enabled
            and hasattr(self, "pool")
            and self.pool
        ):
            # Pass only the necessary data, not the whole 'self' object
            df_copy = df.copy()
            current_shape = self.current_dataset_shape
            self.pool.submit(
                self._record_telemetry_batch, df_copy, tree_type, current_shape
            )

    def _record_telemetry_batch(
        self, df: pd.DataFrame, tree_type: str, dataset_shape: tuple
    ):
        """Background method that performs the actual telemetry recording."""
        try:
            for col in df.columns:
                stats = engine_get_column_stats(df[col])
                hints = self._get_column_ui_hints(stats)
                self.telemetry.record_icon_assignment(
                    dataset_shape=dataset_shape,
                    column_stats=stats,
                    assigned_icons=hints["icons"],
                    confidence_scores=hints.get("confidence_scores", {}),
                    source_system=self._detect_source_system(),  # Assuming this method exists
                )
        except Exception as e:
            log.debug(f"Telemetry batch recording failed: {e}")

    def _detect_source_system(self) -> str:
        """Placeholder to detect the source system for telemetry."""
        # This can be enhanced later to detect Salesforce, HubSpot, etc.
        return "unknown"

    def _compute_col_stats(self, df):
        """Compute column statistics for heuristics."""
        from pandas.api.types import infer_dtype

        stats = {}
        if df is None or df.empty:
            return stats

        sample = df.head(2000)  # keep it fast and deterministic
        for col in sample.columns:
            s = sample[col].astype(str).fillna("")
            lens = s.str.len()
            tokens = s.str.split()
            dtype = infer_dtype(sample[col], skipna=True)
            stats[col] = {
                "avg_len": float(lens.mean() or 0.0),
                "avg_tokens": float(tokens.map(len).mean() or 0.0),
                "alpha_char_ratio": float(
                    s.str.replace(r"[^A-Za-z]", "", regex=True).str.len().sum()
                    / max(lens.sum(), 1)
                ),
                "numeric_char_ratio": float(
                    s.str.replace(r"[^0-9]", "", regex=True).str.len().sum()
                    / max(lens.sum(), 1)
                ),
                "dtype": dtype,  # 'string','integer','floating','boolean','datetime','mixed', etc.
            }
        return stats

    def _suggest_algo_for_col(self, col_name, is_source=True):
        """Choose algorithm per column using heuristic rules."""
        # Ensure we have the stats dict
        if not hasattr(self, "_src_col_stats"):
            self._src_col_stats = {}
        if not hasattr(self, "_ref_col_stats"):
            self._ref_col_stats = {}

        # Get the computed stats
        stats = (self._src_col_stats if is_source else self._ref_col_stats) or {}
        col_stats = stats.get(col_name, {})

        # If we don't have stats, compute them on the fly for this column
        if not col_stats and hasattr(self, "source_df" if is_source else "ref_df"):
            df = self.source_df if is_source else self.ref_df
            if df is not None and col_name in df.columns:
                # Quick stats computation for single column
                sample = df[col_name].head(1000)
                s_str = sample.astype(str).fillna("")
                lens = s_str.str.len()
                tokens = s_str.str.split()

                col_stats = {
                    "avg_len": float(lens.mean() or 20.0),
                    "avg_tokens": float(tokens.map(len).mean() or 2.0),
                    "alpha_char_ratio": float(
                        s_str.str.replace(r"[^A-Za-z]", "", regex=True).str.len().sum()
                        / max(lens.sum(), 1)
                    ),
                    "numeric_char_ratio": float(
                        s_str.str.replace(r"[^0-9]", "", regex=True).str.len().sum()
                        / max(lens.sum(), 1)
                    ),
                    "dtype": "string",  # Simplified for quick computation
                }
                # Cache it
                stats[col_name] = col_stats

        # If still no stats, use reasonable defaults
        if not col_stats:
            col_stats = {
                "avg_len": 20.0,
                "avg_tokens": 2.0,
                "alpha_char_ratio": 0.5,
                "numeric_char_ratio": 0.1,
                "dtype": "string",
            }

        # Build the stats dict for rule evaluation
        s = {
            "avg_len": col_stats.get("avg_len", 20.0),
            "avg_tokens": col_stats.get("avg_tokens", 2.0),
            "alpha_char_ratio": col_stats.get("alpha_char_ratio", 0.5),
            "numeric_char_ratio": col_stats.get("numeric_char_ratio", 0.1),
            "dtype": col_stats.get("dtype", "string"),
        }

        # Apply heuristic rules
        for rule in HEURISTIC_RULES:
            try:
                if rule["condition"](s):
                    return rule["algo_display_name"]
            except Exception as e:
                log.debug(f"Rule evaluation failed for {col_name}: {e}")
                continue

        return "WRatio"  # Default fallback

    def _friendly_dtype(self, raw):
        """Map dtype to a friendly label for the tree."""
        # pandas infer_dtype bucket -> display
        mapping = {
            "string": "string",
            "unicode": "string",
            "bytes": "string",
            "mixed": "string",
            "integer": "integer",
            "floating": "decimal",
            "boolean": "boolean",
            "datetime64": "datetime",
            "datetime": "datetime",
            "date": "date",
            "time": "time",
            "categorical": "category",
        }
        return mapping.get(str(raw), "string")

    def _populate_column_trees(self) -> None:
        """Populate the column trees intelligently to avoid flicker."""
        if self.source_df is not None and hasattr(self, "tree_src"):
            self.current_dataset_shape = self.source_df.shape
            self._update_tree_intelligently(
                self.tree_src, self.source_df, tree_type="source"
            )
            self._batch_record_telemetry(self.source_df, "source")

        if (
            self.mode.get() == "match"
            and self.ref_df is not None
            and hasattr(self, "tree_ref")
        ):
            self._update_tree_intelligently(
                self.tree_ref, self.ref_df, tree_type="reference"
            )

    def _detect_and_apply_id(
        self, df: pd.DataFrame, role: str, file_path: Optional[str] = None
    ):
        """
        Runs enterprise ID detection, updates the combobox, shows an override
        dialog when confidence is low, and pushes telemetry via the engine.
        """
        try:
            # 1. Generate a detailed report to get confidence and selection details
            report = generate_id_selection_report(df)
            chosen_id = report.get("selected")
            confidence = report.get("confidence", "low")

            # 2. For MVP, skip the dialog entirely - just use auto-detected ID
            # TODO: Implement proper ID selection dialog after MVP
            user_override = None

            # 3. Use the engine's override handler to finalize the choice and send telemetry
            final_choice = select_id_column_with_override(
                df=df,
                user_override=user_override,
                file_source=file_path,
                enable_telemetry=True,  # Ensure telemetry is active
            )

            # 4. Apply the final choice to the UI
            widgets = getattr(self, f"{role}_widgets")
            if final_choice and final_choice in df.columns:
                widgets["id_var"].set(final_choice)
                # Provide helpful status feedback to the user
                self.threadsafe_update_status(
                    f"Auto-detected ID: {final_choice} (Confidence: {confidence})"
                )

        except ValueError as e:
            log.error(f"Could not determine ID column for role '{role}': {e}")
            self.threadsafe_update_status(
                "Error: Could not determine an ID column.", is_error=True
            )
        except Exception as e:
            log.error(
                f"An unexpected error occurred during ID detection for role '{role}': {e}",
                exc_info=True,
            )
            self.threadsafe_update_status("Error during ID detection.", is_error=True)

    def _get_column_ui_hints(self, stats: dict) -> dict:
        """
        Central logic for deciding which icons and tooltips to display.
        Takes pure statistics and returns UI decisions.
        """
        icons = []
        tooltips = []
        confidence_scores = {}
        warnings = []

        # Priority 1: Primary ID Column
        if stats.get("is_primary_id"):
            icons.append("🔑")
            confidence = stats.get("id_confidence", 0.7)
            confidence_scores["🔑"] = confidence
            system = stats.get("detected_system", "generic")
            tooltips.append(
                f"Primary ID (confidence: {confidence:.0%}, system: {system})"
            )

        # Priority 2: Semantic Type Icons
        semantic_type = stats.get("inferred_type")

        SEMANTIC_ICONS = {
            "email": ("📧", "Email addresses"),
            "phone": ("📱", "Phone numbers"),
            "website_domain": ("🌐", "Website/Domain"),
            "name_company": ("🏢", "Company/Organization names"),
            "name_person": ("👤", "Person names"),
            "address_street": ("📍", "Street addresses"),
            "city": ("📍", "City names"),
            "state_province": ("📍", "State/Province"),
            "zip_postal": ("📍", "Postal codes"),
            "country": ("📍", "Country names"),
            "date": ("📅", "Date/Time values"),
        }

        if semantic_type in SEMANTIC_ICONS:
            icon, description = SEMANTIC_ICONS[semantic_type]
            if icon not in icons:
                icons.append(icon)
                confidence_scores[icon] = 0.85
                tooltips.append(description)

        # Priority 3: Data Quality Indicators
        fill_rate = stats.get("fill_rate", 0)
        distinct_ratio = stats.get("distinct_ratio", 0)

        if (
            fill_rate >= 0.95
            and distinct_ratio >= 0.8
            and not stats.get("is_primary_id")
        ):
            icons.append("⭐")
            confidence_scores["⭐"] = (fill_rate + distinct_ratio) / 2
            tooltips.append(
                f"High quality: {fill_rate:.0%} complete, {distinct_ratio:.0%} unique"
            )

        if fill_rate < 0.3:
            warnings.append(f"Sparse data: only {fill_rate:.0%} populated")

        if distinct_ratio < 0.01 and not stats.get("is_primary_id"):
            warnings.append(f"Low cardinality: {distinct_ratio:.1%} unique")

        if warnings:
            icons.append("⚠️")
            confidence_scores["⚠️"] = 1.0
            tooltips.append("Data quality issues: " + "; ".join(warnings))

        # Priority 4: Matching Suitability
        if not icons or ("⚠️" in icons and len(icons) == 1):
            if (
                0.7 <= fill_rate
                and 0.01 <= distinct_ratio <= 0.5
                and stats.get("entropy", 0) > 2.0
            ):
                icons.append("🧲")
                confidence_scores["🧲"] = 0.7
                tooltips.append("Good for record matching/blocking")

        # Default icon
        if not icons:
            icons.append("📊")
            tooltips.append("Data column")

        return {
            "icons": icons[:2],
            "tooltips": tooltips,
            "confidence_scores": confidence_scores,
            "warnings": warnings,
            "stats": stats,
        }

    def _populate_single_tree(
        self, tree: ttk.Treeview, df: pd.DataFrame, tree_type: str = "source"
    ) -> None:
        """Populate tree with algorithm recommendations."""
        tree.delete(*tree.get_children())

        if df is None or df.empty:
            return

        for col in df.columns:
            # Get column stats
            col_stats = self._get_individual_column_stats(df[col])

            # Get semantic type
            semantic_type = col_stats.get("inferred_type", "unknown")

            # Get algorithm hint based on column characteristics
            if hasattr(self, "enable_multi_algo") and self.enable_multi_algo.get():
                # In ensemble mode, show what algorithm would likely be used
                # First ensure we have column stats computed
                if not hasattr(self, "_src_col_stats"):
                    self._src_col_stats = (
                        self._compute_col_stats(df) if tree_type == "source" else {}
                    )
                if not hasattr(self, "_ref_col_stats"):
                    self._ref_col_stats = (
                        {} if tree_type == "source" else self._compute_col_stats(df)
                    )

                algo_hint = self._suggest_algo_for_col(
                    col, is_source=(tree_type == "source")
                )
            else:
                # Show the globally selected algorithm
                algo_hint = (
                    self.algorithm_var.get()
                    if hasattr(self, "algorithm_var")
                    else "WRatio"
                )

            # Store the hint
            if not hasattr(self, "column_algorithm_hints"):
                self.column_algorithm_hints = {}
            self.column_algorithm_hints[col] = algo_hint

            # Get UI hints
            hints = self._get_column_ui_hints(col_stats)
            icon_str = " ".join(hints.get("icons", []))

            # Format the stats for display
            fill_str = f"{col_stats.get('fill_rate', 0) * 100:.1f}%"
            distinct_str = f"{col_stats.get('unique_count_in_sample', 0):,} ({col_stats.get('distinct_ratio', 0) * 100:.1f}%)"
            type_str = col_stats.get("inferred_type", "unknown")

            # Insert with separate icon and algorithm columns
            item_id = tree.insert(
                "",
                "end",
                values=(
                    icon_str,
                    col,
                    algo_hint,  # Algorithm in its own column
                    fill_str,
                    distinct_str,
                    type_str,
                ),
            )

            # Store tooltip data
            if not hasattr(self, "column_tooltips"):
                self.column_tooltips = {}
            self.column_tooltips[item_id] = "\n".join(hints.get("tooltips", []))

    def _calculate_global_fill_rate_thresholds(self) -> None:
        """Calculate global fill rate thresholds for mapping scoring."""
        all_fill_rates = []

        # Collect from source
        if self.source_df is not None:
            for col in self.source_df.columns:
                if pd.api.types.is_string_dtype(
                    self.source_df[col]
                ) or pd.api.types.is_object_dtype(self.source_df[col]):
                    fill_rate = self.source_df[col].notna().mean()
                    all_fill_rates.append(fill_rate)

        # Collect from reference
        if self.mode.get() == "match" and self.ref_df is not None:
            for col in self.ref_df.columns:
                if pd.api.types.is_string_dtype(
                    self.ref_df[col]
                ) or pd.api.types.is_object_dtype(self.ref_df[col]):
                    fill_rate = self.ref_df[col].notna().mean()
                    all_fill_rates.append(fill_rate)

        # Calculate percentiles
        if all_fill_rates:
            fill_rates_np = np.array(all_fill_rates)
            self.global_fill_rate_thresholds = {
                "low": min(np.percentile(fill_rates_np, 25), 0.70),
                "medium": min(np.percentile(fill_rates_np, 50), 0.85),
                "high": min(np.percentile(fill_rates_np, 75), 0.95),
            }
        else:
            # Defaults
            self.global_fill_rate_thresholds = {
                "low": 0.25,
                "medium": 0.50,
                "high": 0.75,
            }

    # ─────────────────────────────────────────────────────────────────────────
    # Salesforce Data Loading Methods
    # ─────────────────────────────────────────────────────────────────────────

    def _connect_to_salesforce(self):
        """Handle Salesforce connection."""
        try:
            from fmatch.integrations.salesforce.gui_integration import (
                SalesforceConnectionDialog,
            )
            from fmatch.integrations.salesforce.core import SalesforceIntegration

            dialog = SalesforceConnectionDialog(self, self.sf_credentials)
            self.wait_window(dialog)

            if dialog.result:
                self.sf_credentials = dialog.result
                self.sf_integration = SalesforceIntegration(self.sf_credentials)

                # Update UI
                if hasattr(self, "data_loader"):
                    self.data_loader.update_displays()
        except ImportError:
            messagebox.showwarning(
                "Module Not Found",
                "Salesforce integration modules not found.\n"
                "Please ensure the salesforce integration package is installed.",
            )

    def _setup_salesforce_data_source(self):
        """Build the UI for selecting a Salesforce object to load."""
        from fmatch.integrations.salesforce.gui_integration import (
            SalesforceObjectSelector,
        )

        # Clear any existing widgets from the data source frame
        for widget in self.sf_data_frame.winfo_children():
            widget.destroy()

        # Create and display the object selector
        self.sf_object_selector = SalesforceObjectSelector(
            self.sf_data_frame, self.sf_integration
        )
        self.sf_object_selector.pack(fill=tk.BOTH, expand=True)

        # Add action buttons for loading data
        action_frame = ttk.Frame(self.sf_data_frame)
        action_frame.pack(fill=tk.X, pady=(10, 0))

        ttk.Button(
            action_frame,
            text="Load as Source Data",
            command=self._load_salesforce_data_as_source,
            style="Primary.TButton",
        ).pack(side=tk.LEFT, padx=5)

        ttk.Button(
            action_frame,
            text="Load as Reference Data",
            command=self._load_salesforce_data_as_reference,
        ).pack(side=tk.LEFT, padx=5)

    def _load_salesforce_data_as_source(self):
        """Load selected Salesforce object as source data."""
        self._load_salesforce_data(is_source=True)

    def _load_salesforce_data_as_reference(self):
        """Load selected Salesforce object as reference data."""
        self._load_salesforce_data(is_source=False)

    def _load_salesforce_data(self, is_source: bool):
        """Generic method to load data from the selected Salesforce object."""
        import threading
        import asyncio
        from tkinter import messagebox

        if not hasattr(self, "sf_object_selector"):
            messagebox.showerror("Error", "Object selector not found.")
            return

        selected_object = self.sf_object_selector.get_selected_object()
        if not selected_object:
            messagebox.showwarning(
                "No Selection", "Please select a Salesforce object from the list first."
            )
            return

        # For simplicity, load the first 10 fields, ensuring Id is included
        fields_to_load = [f.name for f in selected_object.fields[:10]]
        if "Id" not in fields_to_load:
            id_field = next(
                (f.name for f in selected_object.fields if f.name == "Id"), None
            )
            if id_field:
                fields_to_load.insert(0, "Id")

        # Build SOQL query
        soql = (
            f"SELECT {','.join(fields_to_load)} FROM {selected_object.name} LIMIT 10000"
        )

        # Show loading status
        data_type = "source" if is_source else "reference"
        self.status_var.set(f"Loading {data_type} data from {selected_object.name}...")

        def load_async():
            try:
                # Run the async query in a new event loop
                loop = asyncio.new_event_loop()
                asyncio.set_event_loop(loop)
                df = loop.run_until_complete(self.sf_integration.query_data(soql))
                loop.close()

                # Update UI on main thread
                self.after(
                    0,
                    self._handle_salesforce_data_loaded,
                    df,
                    is_source,
                    selected_object.name,
                )
            except Exception as exc:
                self.after(
                    0,
                    lambda err=exc: messagebox.showerror(
                        "Load Error", f"Failed to load data: {err}"
                    ),
                )
                self._after(0, lambda: self.status_var.set("Ready"))

        threading.Thread(target=load_async, daemon=True).start()

    def _handle_salesforce_data_loaded(self, df, is_source, object_name):
        """Callback to handle the loaded DataFrame."""
        if df is not None and not df.empty:
            if is_source:
                self.source_df = df
                self.src_path = f"Salesforce:{object_name}"
                if hasattr(self, "lbl_src"):
                    self.lbl_src.config(
                        text=f"Salesforce: {object_name} ({len(df):,} records)"
                    )
                if "Id" in df.columns and hasattr(self, "source_id_var"):
                    self.source_id_var.set("Id")
            else:
                self.ref_df = df
                self.ref_path = f"Salesforce:{object_name}"
                if hasattr(self, "lbl_ref"):
                    self.lbl_ref.config(
                        text=f"Salesforce: {object_name} ({len(df):,} records)"
                    )
                if "Id" in df.columns and hasattr(self, "ref_id_var"):
                    self.ref_id_var.set("Id")

            # Refresh UI to show new data
            if hasattr(self, "_populate_column_trees"):
                self._populate_column_trees()
            self.refresh_ui()

            messagebox.showinfo(
                "Success", f"Loaded {len(df):,} records from {object_name}."
            )
            self.status_var.set("Ready")
        else:
            messagebox.showwarning("No Data", "Query returned no data from Salesforce.")
            self.status_var.set("Ready")

    # ─────────────────────────────────────────────────────────────────────────
    # Mapping Methods
    # ─────────────────────────────────────────────────────────────────────────

    def add_mapping(self) -> None:
        """Add a field mapping, with separate logic for dedupe and match modes."""
        # Ensure mapping trees are visible before adding
        if hasattr(self, "tree_src") and self.tree_src:
            if not self.tree_src.winfo_viewable():
                log.debug("tree_src not viewable, forcing visibility")
                self._show_mapping_trees()

        # Handle dedupe mode separately
        if self.mode.get() == "dedupe":
            # For dedupe, just track selected columns
            src_sel = self.tree_src.selection() if self.tree_src else []
            if not src_sel:
                messagebox.showwarning(
                    "No Selection", "Please select a column to check for duplicates."
                )
                return

            src_item = self.tree_src.item(src_sel[0])
            src_col = src_item["values"][1]  # Column name is second value

            # Check for duplicates
            if any(m["source"] == src_col for m in self.mappings):
                messagebox.showwarning(
                    "Already Selected",
                    f"'{src_col}' is already selected for deduplication.",
                )
                return

            # Add as a "self-mapping" for dedupe
            mapping = {
                "source": src_col,
                "ref": src_col,  # Same column for dedupe
                "weight": 1,  # Weight is less relevant for simple dedupe column selection
            }
            self.mappings.append(mapping)
            self._refresh_mappings_list()
            self.refresh_ui()
            return

        # --- Match mode logic ---
        # Get selections
        src_sel = self.tree_src.selection() if self.tree_src else []
        ref_sel = self.tree_ref.selection() if self.tree_ref else []

        if not src_sel:
            messagebox.showwarning("No Selection", "Please select a source column")
            return

        if not ref_sel:
            messagebox.showwarning("No Selection", "Please select a reference column")
            return

        # Get column names
        src_item = self.tree_src.item(src_sel[0])
        src_col = src_item["values"][1]  # Column name is second value

        ref_item = self.tree_ref.item(ref_sel[0])
        ref_col = ref_item["values"][1]

        # Check for duplicates
        if any(m["source"] == src_col for m in self.mappings):
            messagebox.showwarning(
                "Duplicate Mapping", f"Source column '{src_col}' is already mapped"
            )
            return

        # Get weight
        try:
            weight = int(
                self.spin_weight.get()
                if hasattr(self, "spin_weight")
                else CONFIG["DEFAULT_WEIGHT"]
            )
            weight = max(CONFIG["MIN_WEIGHT"], min(CONFIG["MAX_WEIGHT"], weight))
        except (ValueError, tk.TclError):
            weight = CONFIG["DEFAULT_WEIGHT"]

        # Add mapping
        mapping = {"source": src_col, "ref": ref_col, "weight": weight}
        self.mappings.append(mapping)

        # Update UI
        self._refresh_mappings_list()
        self.refresh_ui()

    def remove_mapping(self) -> None:
        """Remove selected mapping."""
        if not self.map_list:
            return

        selection = self.map_list.curselection()
        if not selection:
            return

        idx = selection[0]
        if 0 <= idx < len(self.mappings):
            self.mappings.pop(idx)
            self._refresh_mappings_list()
            self.refresh_ui()

    def clear_mappings(self) -> None:
        """Clear all mappings and reset UI to placeholder state."""
        if self.mappings and not messagebox.askyesno(
            "Clear Mappings", "Remove all field mappings?"
        ):
            return

        self.mappings.clear()
        self.mappings_confirmed = False
        self.auto_determined_locking_strategy = None
        self.blocking_config = None

        # Hide trees and show placeholders
        if hasattr(self, "tree_src") and isinstance(self.tree_src, ttk.Treeview):
            self.tree_src.master.grid_forget()
            # Recreate source placeholder
            self.source_placeholder = ttk.Label(
                self.mapping_frame,
                text="<- Load source data to see columns",
                font=PLACEHOLDER_FONT,
                foreground=PLACEHOLDER_COLOR,
            )
            self.source_placeholder.grid(row=0, column=0, pady=30, padx=20, sticky="ew")

        if hasattr(self, "tree_ref") and isinstance(self.tree_ref, ttk.Treeview):
            self.tree_ref.master.grid_forget()
            # Recreate reference placeholder
            self.ref_placeholder = ttk.Label(
                self.mapping_frame,
                text="<- Load reference data to see columns",
                font=PLACEHOLDER_FONT,
                foreground=PLACEHOLDER_COLOR,
            )
            self.ref_placeholder.grid(row=0, column=1, pady=30, padx=20, sticky="ew")

        self._refresh_mappings_list()
        self.refresh_ui()

    def clear_all_mappings(self) -> None:
        """Clear all mappings with confirmation."""
        if not self.mappings:
            return

        if messagebox.askyesno("Clear Mappings", "Remove all column mappings?"):
            self.mappings = []
            self._refresh_mappings_list()
            self.refresh_ui()
            self.threadsafe_update_status("All mappings cleared")

    def _refresh_mappings_list(self) -> None:
        """Refresh the mappings listbox with algorithm info."""
        if not hasattr(self, "map_list") or not self.map_list:
            return

        self.map_list.delete(0, tk.END)

        for mapping in self.mappings:
            # Get algorithm info
            algo = mapping.get("preferred_algo", self.algorithm_var.get())
            confidence = mapping.get("algo_confidence", 0.0)
            weight = mapping.get("weight", 1.0)

            # Check if this was optimized for domains
            source_col = mapping.get("source", "").lower()
            ref_col = mapping.get("ref", "").lower()
            domain_keywords = [
                "domain",
                "website",
                "url",
                "site",
                "web",
                ".com",
                "email",
            ]
            # Check if this was optimized to Exact
            is_optimized = mapping.get("preferred_algo") == "Exact" and (
                any(keyword in source_col for keyword in domain_keywords)
                or any(keyword in ref_col for keyword in domain_keywords)
            )

            # Format display text
            if self.mode.get() == "dedupe":
                if self.enable_multi_algo.get() and mapping.get("preferred_algo"):
                    optimization_marker = " ⚡" if is_optimized else ""
                    text = f"{mapping['source']} ({algo}{optimization_marker}, weight: {weight:.1f})"
                else:
                    text = f"{mapping['source']} (weight: {weight:.1f})"
            else:
                if self.enable_multi_algo.get() and mapping.get("preferred_algo"):
                    optimization_marker = " ⚡" if is_optimized else ""
                    text = f"{mapping['source']} -> {mapping['ref']} ({algo}{optimization_marker} {confidence:.0%}, weight: {weight:.1f})"
                else:
                    text = f"{mapping['source']} -> {mapping['ref']} (weight: {weight:.1f})"

            self.map_list.insert(tk.END, text)

    def on_confirm_mappings_clicked(self) -> None:
        """Handle mapping confirmation."""
        if not self.mappings:
            messagebox.showwarning("No Mappings", "Please add at least one mapping")
            return

        if self.processing:
            messagebox.showwarning("Busy", "Cannot modify mappings while processing")
            return

        self.mappings_confirmed = True
        self.current_operation_id = str(uuid.uuid4())

        log.info(
            f"Mappings confirmed. Starting locking analysis (op_id: {self.current_operation_id})"
        )

        # ADD B2C WIZARD HINT HERE
        add_b2c_wizard_hint(self, self.mappings, self.source_df, self.ref_df)

        # Start analysis
        self.initiate_locking_analysis()
        self.refresh_ui()

    def on_unlock_mappings_clicked(self) -> None:
        """Handle mapping unlock."""
        if self.processing:
            messagebox.showwarning("Busy", "Cannot modify mappings while processing")
            return

        self.mappings_confirmed = False
        self.auto_determined_locking_strategy = None
        self.blocking_config = None

        log.info("Mappings unlocked")
        self.threadsafe_update_status(
            "Mappings unlocked - please re-confirm after changes"
        )
        self.refresh_ui()

    def _show_weight_dialog(self):
        """Show dialog to adjust mapping weight."""
        selection = self.map_list.curselection()
        if not selection:
            messagebox.showwarning(
                "No Selection", "Please select a mapping to adjust weight"
            )
            return

        idx = selection[0]
        if 0 <= idx < len(self.mappings):
            current_weight = self.mappings[idx].get("weight", 1)

            # Simple weight dialog
            dialog = tk.Toplevel(self)
            dialog.title("Adjust Weight")
            dialog.geometry("250x150")
            dialog.transient(self)
            dialog.grab_set()

            ttk.Label(dialog, text="Mapping Weight:").pack(pady=10)

            weight_var = tk.IntVar(value=current_weight)
            weight_spin = ttk.Spinbox(
                dialog, from_=1, to=10, textvariable=weight_var, width=10
            )
            weight_spin.pack(pady=5)

            def apply_weight():
                self.mappings[idx]["weight"] = weight_var.get()
                self._refresh_mappings_list()
                dialog.destroy()

            ttk.Button(dialog, text="Apply", command=apply_weight).pack(pady=10)

    # In gui.py, paste this new code block starting at line 1656

    def on_auto_map_fields_clicked(self) -> None:
        """Enhanced auto-map fields with background processing."""
        if self.source_df is None:
            messagebox.showwarning("No Data", "Please load source data first")
            return

        if self.mode.get() == "match" and self.ref_df is None:
            messagebox.showwarning(
                "No Data", "Please load reference data for match mode"
            )
            return

        # Check if there are unmapped columns
        mapped_src = {m["source"] for m in self.mappings}
        unmapped_src = [c for c in self.source_df.columns if c not in mapped_src]

        if not unmapped_src:
            messagebox.showinfo("Auto-Map", "All source columns are already mapped")
            return

        # Disable button and update status
        if hasattr(self, "btn_auto_map"):
            self.btn_auto_map.config(state="disabled")

        self.threadsafe_update_status("Analyzing columns for auto-mapping...")
        self._set_progress_indeterminate()
        # Hide the percentage during auto-mapping
        if hasattr(self, "lbl_progress") and self.lbl_progress:
            self.lbl_progress.config(text="Analyzing...")

        # Submit to thread pool
        self._ensure_pool_active()
        future = self.pool.submit(
            _log_thread_exceptions(self._perform_auto_mapping_logic)
        )
        future.add_done_callback(self._handle_auto_map_result)

    def _handle_auto_map_result(self, future: cf.Future) -> None:
        """Handle completion of auto-mapping."""
        self._set_progress_determinate(value=100)
        # Reset progress label
        if hasattr(self, "lbl_progress") and self.lbl_progress:
            self.lbl_progress.config(text="")

        try:
            new_mappings_count = future.result()

            # Update UI
            self._refresh_mappings_list()
            self.refresh_ui()

            # Update clear button state
            if hasattr(self, "btn_clear_mappings") and self.mappings:
                self.btn_clear_mappings.config(state=tk.NORMAL)

            # Show result
            if new_mappings_count > 0:
                messagebox.showinfo(
                    "Auto-Map Complete",
                    f"Successfully added {new_mappings_count} automatic mappings",
                )
                self.threadsafe_update_status(
                    f"Auto-mapping complete - added {new_mappings_count} mappings"
                )
            else:
                # Check if mappings already exist
                if self.mappings:
                    messagebox.showinfo(
                        "Auto-Map", "All suitable columns are already mapped"
                    )
                    self.threadsafe_update_status(
                        "Auto-mapping complete - all columns already mapped"
                    )
                else:
                    messagebox.showinfo("Auto-Map", "No suitable mappings found")
                    self.threadsafe_update_status(
                        "Auto-mapping complete - no suitable mappings found"
                    )

        except Exception as e:
            log.error(f"Error in auto-mapping: {e}", exc_info=True)
            messagebox.showerror("Auto-Map Error", f"Failed to auto-map fields: {e}")
            self.threadsafe_update_status("Auto-mapping failed", is_error=True)

        finally:
            # Re-enable button (both references for compatibility)
            if hasattr(self, "btn_auto_map"):
                self.btn_auto_map.config(state="normal")
            if hasattr(self, "btn_auto_match"):
                self.btn_auto_match.config(state="normal")
            self.refresh_ui()

    def _perform_auto_mapping_logic(self) -> int:
        """Enhanced version with per-column algorithm selection."""
        log.info("Performing auto-mapping with algorithm optimization...")

        # Get column lists
        source_cols = list(self.source_df.columns)
        ref_cols = (
            list(self.ref_df.columns) if self.mode.get() == "match" else source_cols
        )

        current_mapped_src_cols = {m["source"] for m in self.mappings}
        new_mappings_added_count = 0

        for s_col in source_cols:
            if s_col in current_mapped_src_cols:
                continue

            # Find best matching reference column
            best_ref_match_candidate = None
            highest_name_sim = 0.0

            for r_col in ref_cols:
                # Skip if already mapped
                if r_col in {m["ref"] for m in self.mappings}:
                    continue

                name_sim = rapidfuzz_fuzz.WRatio(
                    self._process_col_name_for_fuzzy(s_col),
                    self._process_col_name_for_fuzzy(r_col),
                )
                if name_sim > highest_name_sim:
                    highest_name_sim = name_sim
                    best_ref_match_candidate = r_col

            if best_ref_match_candidate and highest_name_sim >= 55.0:
                # Get data for algorithm selection
                src_data = self.source_df[s_col]
                ref_data = (
                    self.ref_df[best_ref_match_candidate]
                    if self.mode.get() == "match"
                    else self.source_df[best_ref_match_candidate]
                )

                # Determine semantic type
                semantic_type = self._get_semantic_type(s_col, src_data.head(100))

                # Select optimal algorithm if multi-algo is enabled
                if self.enable_multi_algo.get():
                    algo_name, algo_confidence = self.algo_selector.select_algorithm(
                        src_data, ref_data, semantic_type
                    )

                    # Store hint for display
                    self.column_algorithm_hints[s_col] = algo_name
                else:
                    # Use default algorithm
                    algo_name = self.algorithm_var.get()
                    algo_confidence = 0.7  # Default confidence

                # Calculate weight using existing logic
                raw_score, component_scores = self._calculate_mapping_weight(
                    highest_name_sim,
                    self._get_individual_column_stats(src_data),
                    self._get_individual_column_stats(ref_data),
                    semantic_type,
                    self._get_semantic_type(
                        best_ref_match_candidate, ref_data.head(100)
                    ),
                )

                if raw_score >= 50.0:  # Threshold
                    # Create enhanced mapping
                    mapping = {
                        "source": s_col,
                        "ref": best_ref_match_candidate,
                        "weight": self._scale_weight(raw_score),
                        "preferred_algo": (
                            algo_name if self.enable_multi_algo.get() else None
                        ),
                        "algo_confidence": algo_confidence,
                        "raw_score": round(raw_score, 2),
                        "component_scores": component_scores,
                        "auto_mapped": True,
                    }

                    self.mappings.append(mapping)
                    new_mappings_added_count += 1

                    log.info(
                        f"Auto-mapped: '{s_col}' -> '{best_ref_match_candidate}' "
                        f"using {algo_name} (confidence: {algo_confidence:.2f})"
                    )

        return new_mappings_added_count

    def _calculate_column_similarity(self, col1: str, col2: str) -> float:
        """Calculate similarity between column names."""
        # Normalize names
        norm1 = _sanitize_name(col1).lower()
        norm2 = _sanitize_name(col2).lower()

        # Exact match
        if norm1 == norm2:
            return 1.0

        # Substring match
        if norm1 in norm2 or norm2 in norm1:
            return 0.8

        # Token overlap
        tokens1 = set(norm1.split("_"))
        tokens2 = set(norm2.split("_"))

        if tokens1 and tokens2:
            overlap = len(tokens1 & tokens2)
            total = len(tokens1 | tokens2)
            if total > 0:
                return overlap / total * 0.7

        return 0.0

    def _get_semantic_type(
        self, column_name: str, series_sample: pd.Series
    ) -> Optional[str]:
        """Infers the semantic type of a column based on its name and data patterns."""

        # Semantic keyword patterns
        semantic_patterns = {
            "email": re.compile(r"\b(email|mail)\b", re.IGNORECASE),
            "phone": re.compile(r"\b(phone|mobile|fax|contact.?num)\b", re.IGNORECASE),
            "website_domain": re.compile(
                r"\b(website|url|domain|site)\b", re.IGNORECASE
            ),
            "name_person": re.compile(
                r"\b(name|contact|person|owner|user|first.?name|last.?name)\b",
                re.IGNORECASE,
            ),
            "name_company": re.compile(
                r"\b(company|organization|account|customer|business|client|supplier|vendor)\b",
                re.IGNORECASE,
            ),
            "country": re.compile(r"\b(country)\b", re.IGNORECASE),
            "state": re.compile(r"\b(state|province|county|region)\b", re.IGNORECASE),
            "status": re.compile(
                r"\b(status|state|type|category|tier|level)\b", re.IGNORECASE
            ),
        }

        # Data pattern regexes
        data_patterns = {
            "email": re.compile(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"),
            "website_domain": re.compile(
                r"^(?:https?:\/\/)?(?:www\.)?([a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*\.[a-zA-Z]{2,})"
            ),
            "phone": re.compile(
                r"^\(?([0-9]{3})\)?[-.●\s]?([0-9]{3})[-.●\s]?([0-9]{4})$"
            ),
        }

        # Check column name first
        col_name_lower = str(column_name).lower()
        for type_name, pattern in semantic_patterns.items():
            if pattern.search(col_name_lower):
                # Special case: avoid name_person if it looks like company
                if type_name == "name_person" and semantic_patterns[
                    "name_company"
                ].search(col_name_lower):
                    continue
                return type_name

        # Check data patterns if name doesn't match
        if not series_sample.empty:
            str_sample = series_sample.dropna().astype(str).head(100)
            if not str_sample.empty:
                for type_name, pattern in data_patterns.items():
                    # Check if at least 30% of values match the pattern
                    matches = str_sample.str.match(pattern)
                    if matches.mean() >= 0.30:
                        return type_name

        return None

    def _get_individual_column_stats(self, series: pd.Series) -> dict:
        """Calculate statistics for a single column."""
        stats = {
            "name": series.name,
            "fill_rate": 0.0,
            "distinct_ratio": 0.0,
            "unique_count_in_sample": 0,
            "sample_uniques": frozenset(),
            "inferred_type": "empty_unknown",
            "median_char_length": 0.0,
            "median_token_count": 0.0,
        }

        if series.empty:
            return stats

        # Fill rate
        stats["fill_rate"] = series.notna().mean()

        # For meaningful analysis, work with non-null values
        non_null = series.dropna()
        if non_null.empty:
            return stats

        # Distinct ratio
        unique_count = non_null.nunique()
        stats["distinct_ratio"] = unique_count / len(non_null)
        stats["unique_count_in_sample"] = unique_count

        # If cardinality is low, store unique values
        if unique_count <= 25:
            stats["sample_uniques"] = frozenset(non_null.unique())

        # Infer data type
        if pd.api.types.is_numeric_dtype(series):
            stats["inferred_type"] = "numeric"
        elif pd.api.types.is_datetime64_any_dtype(series):
            stats["inferred_type"] = "datetime"
        elif pd.api.types.is_bool_dtype(series):
            stats["inferred_type"] = "boolean"
        else:
            # String type - calculate additional stats
            stats["inferred_type"] = "string"
            str_series = non_null.astype(str)

            # Character length statistics
            lengths = str_series.str.len()
            stats["median_char_length"] = lengths.median() if not lengths.empty else 0.0

            # Token count statistics (split by whitespace)
            token_counts = str_series.str.split().str.len()
            stats["median_token_count"] = (
                token_counts.median() if not token_counts.empty else 0.0
            )

        return stats

    def _process_col_name_for_fuzzy(self, name: str) -> str:
        """Process column name for fuzzy matching by removing special characters."""
        processed_name = str(name).lower()
        # Remove special characters and spaces
        processed_name = processed_name.replace(" / ", "").replace("/", "")
        processed_name = processed_name.replace(" & ", "").replace("&", "")
        processed_name = processed_name.replace("_", "").replace(" ", "")
        processed_name = processed_name.replace("-", "")
        return processed_name

    def _calculate_mapping_weight(
        self,
        name_similarity_score: float,  # Expected 0-100
        s_stats: dict,  # Output from _get_individual_column_stats for source
        r_stats: dict,  # Output from _get_individual_column_stats for reference
        s_semantic_type: Optional[str],
        r_semantic_type: Optional[str],
    ) -> Tuple[float, Dict[str, float]]:  # Returns raw_score, component_scores
        """
        Calculates a raw mapping score (0-100) based on various heuristics
        and returns the raw score and its constituent component scores.
        Uses parameters from self.mapping_heuristics and self.global_fill_rate_thresholds.
        """
        # Ensure log, pd, np are accessible if direct pandas/numpy operations are done here
        # For now, relying on stats dicts being pre-processed.

        heuristics = (
            self.mapping_heuristics
        )  # Access the centralized heuristics from self
        raw_score = (
            heuristics.raw_score_initial_baseline
        )  # Start at 0.0 as per feedback
        component_scores = {"baseline": heuristics.raw_score_initial_baseline}

        # 1. Name Similarity Contribution
        clamped_name_sim = max(0.0, min(name_similarity_score, 100.0))  # Clamp to 0-100
        name_sim_points = (
            clamped_name_sim / 100.0
        ) * heuristics.max_name_sim_contribution
        raw_score += name_sim_points
        component_scores["name_similarity"] = round(name_sim_points, 2)
        log.debug(
            f"_calc_weight for S:'{s_stats.get('name', 'N/A')}' R:'{r_stats.get('name', 'N/A')}': "
            f"name_sim_score={clamped_name_sim:.1f} -> points={name_sim_points:.1f}"
        )

        # 2. Fill Rate Contribution (Uses self.global_fill_rate_thresholds)
        avg_fill_rate = (
            s_stats.get("fill_rate", 0.0) + r_stats.get("fill_rate", 0.0)
        ) / 2.0
        fill_rate_points = 0.0
        # Access global_fill_rate_thresholds from self
        current_fill_thresholds = self.global_fill_rate_thresholds

        if avg_fill_rate >= current_fill_thresholds.get("high", 0.75):
            fill_rate_points = heuristics.fill_rate_bonus_high
        elif avg_fill_rate >= current_fill_thresholds.get("medium", 0.50):
            fill_rate_points = heuristics.fill_rate_bonus_medium
        elif avg_fill_rate >= current_fill_thresholds.get("low", 0.25):
            fill_rate_points = heuristics.fill_rate_bonus_low
        raw_score += fill_rate_points
        component_scores["fill_rate"] = round(fill_rate_points, 2)
        log.debug(
            f"  avg_fill_rate={avg_fill_rate:.2f} (thresholds: {current_fill_thresholds}) -> points={fill_rate_points:.1f}"
        )

        # 3. Graduated Semantic Boost
        semantic_adjustment = 0.0
        if s_semantic_type and s_semantic_type == r_semantic_type:
            # Note: heuristics.strong_id_semantic_types etc. would need to be updated
            # if custom semantic types are to be categorized for these boosts.
            # For now, it uses the defaults in MappingHeuristics.
            if s_semantic_type in heuristics.strong_id_semantic_types:
                semantic_adjustment = heuristics.semantic_boost_strong_id
            elif s_semantic_type in heuristics.medium_enum_semantic_types:
                semantic_adjustment = heuristics.semantic_boost_medium_enum
            else:  # Check if it's any other known (e.g. custom) or just a generic match
                all_known_semantic_types = (
                    heuristics.strong_id_semantic_types
                    + heuristics.medium_enum_semantic_types
                )
                # This could be expanded if custom types are loaded into MappingHeuristics lists
                if (
                    s_semantic_type not in all_known_semantic_types
                ):  # A matched type, but not strong/medium
                    semantic_adjustment = heuristics.semantic_boost_weak_match

        raw_score += semantic_adjustment
        component_scores["semantic_boost"] = round(semantic_adjustment, 2)
        log.debug(
            f"  s_type='{s_semantic_type}', r_type='{r_semantic_type}' -> points={semantic_adjustment:.1f}"
        )

        # 4. Nuanced Distinctiveness Handling
        distinct_adjustment = 0.0
        s_distinct_ratio = s_stats.get("distinct_ratio", 0.5)
        r_distinct_ratio = r_stats.get("distinct_ratio", 0.5)
        s_unique_count = s_stats.get("unique_count_in_sample", float("inf"))
        r_unique_count = r_stats.get("unique_count_in_sample", float("inf"))
        s_sample_uniques = s_stats.get(
            "sample_uniques", frozenset()
        )  # Expected from _get_individual_column_stats
        r_sample_uniques = r_stats.get("sample_uniques", frozenset())

        log.debug(
            f"  Distinctiveness - Inputs: s_distinct_ratio={s_distinct_ratio:.3f}, r_distinct_ratio={r_distinct_ratio:.3f}, "
            f"s_uniques={s_unique_count}, r_uniques={r_unique_count}, "
            f"s_sample_uniques_len={len(s_sample_uniques)}, r_sample_uniques_len={len(r_sample_uniques)}"
        )

        # Low-Cardinality Reward (only if both have few uniques and we have their sample_uniques)
        if (
            s_unique_count <= heuristics.distinct_low_cardinality_threshold
            and r_unique_count <= heuristics.distinct_low_cardinality_threshold
            and s_sample_uniques
            and r_sample_uniques
        ):  # sample_uniques not empty means uniques were few enough to store
            log.debug(
                f"  Distinctiveness - Qualifies for Low Cardinality Check: s_uniques ({s_unique_count}) <= thresh ({heuristics.distinct_low_cardinality_threshold}) AND r_uniques ({r_unique_count}) <= thresh AND samples exist."
            )
            intersection_len = len(s_sample_uniques.intersection(r_sample_uniques))
            union_len = len(s_sample_uniques.union(r_sample_uniques))
            overlap_ratio = intersection_len / union_len if union_len > 0 else 0.0
            log.debug(
                f"  Distinctiveness - Low Cardinality Overlap: intersection={intersection_len}, union={union_len}, overlap_ratio={overlap_ratio:.3f}"
            )

            if overlap_ratio > heuristics.distinct_low_cardinality_overlap_threshold:
                distinct_adjustment = heuristics.distinct_reward_low_cardinality_overlap
                log.debug(
                    f"  Distinctiveness - REWARD applied: Low cardinality overlap ({overlap_ratio:.3f} > {heuristics.distinct_low_cardinality_overlap_threshold}). Adjustment = {distinct_adjustment:.1f}"
                )
            else:
                log.debug(
                    f"  Distinctiveness - No reward: Low cardinality overlap ({overlap_ratio:.3f}) not > threshold ({heuristics.distinct_low_cardinality_overlap_threshold})."
                )
        else:
            log.debug(
                f"  Distinctiveness - Did NOT qualify for Low Cardinality Reward check. s_uniques={s_unique_count}, r_uniques={r_unique_count}, s_sample_uniques_empty={not s_sample_uniques}, r_sample_uniques_empty={not r_sample_uniques}"
            )

        # High/Low Distinctiveness Penalty (if not rewarded above)
        if distinct_adjustment == 0.0:  # Only apply penalty if no reward was given
            log.debug(
                "  Distinctiveness - Checking for High/Low Distinctiveness Penalty (no reward given yet)."
            )
            is_strong_id_match_for_distinct_exempt = (
                s_semantic_type
                and s_semantic_type == r_semantic_type
                and s_semantic_type in heuristics.strong_id_semantic_types
            )
            log.debug(
                f"  Distinctiveness - Penalty exemption check: is_strong_id_match_for_distinct_exempt = {is_strong_id_match_for_distinct_exempt} (s_type: {s_semantic_type}, r_type: {r_semantic_type})"
            )

            if (
                s_distinct_ratio < heuristics.distinct_ratio_low_threshold
                or s_distinct_ratio > heuristics.distinct_ratio_high_threshold
                or r_distinct_ratio < heuristics.distinct_ratio_low_threshold
                or r_distinct_ratio > heuristics.distinct_ratio_high_threshold
            ) and not (
                is_strong_id_match_for_distinct_exempt
                and (
                    s_distinct_ratio > heuristics.distinct_ratio_high_threshold
                    or r_distinct_ratio > heuristics.distinct_ratio_high_threshold
                )
            ):
                log.debug(
                    f"  Distinctiveness - PENALTY conditions MET: "
                    f"s_distinct_ratio ({s_distinct_ratio:.3f}) outside [{heuristics.distinct_ratio_low_threshold}, {heuristics.distinct_ratio_high_threshold}] OR "
                    f"r_distinct_ratio ({r_distinct_ratio:.3f}) outside [{heuristics.distinct_ratio_low_threshold}, {heuristics.distinct_ratio_high_threshold}]. "
                    f"AND NOT (exempted_strong_id_high_distinctness)."
                )
                distinct_adjustment = heuristics.distinct_penalty_general
            else:
                log.debug(
                    "  Distinctiveness - No penalty applied (conditions not met or exempted)."
                )

        raw_score += distinct_adjustment
        component_scores["distinctness"] = round(distinct_adjustment, 2)
        log.debug(
            f"  Distinctiveness - Final distinct_adjustment = {distinct_adjustment:.1f}. Raw score now = {raw_score:.2f}"
        )

        # 5. Data Type Agreement Adjustment
        type_agreement_adjustment = 0.0
        s_inferred_type = s_stats.get("inferred_type", "empty_unknown")
        r_inferred_type = r_stats.get("inferred_type", "empty_unknown")

        if s_inferred_type == r_inferred_type and s_inferred_type not in [
            "empty_unknown",
            "mixed_other",
        ]:
            type_agreement_adjustment = heuristics.type_agreement_bonus
        elif (
            s_inferred_type != "empty_unknown"
            and r_inferred_type != "empty_unknown"
            and (
                (s_inferred_type == "numeric" and r_inferred_type == "string")
                or (s_inferred_type == "string" and r_inferred_type == "numeric")
            )
        ):
            # Basic check for string vs numeric mismatch; could be more nuanced
            type_agreement_adjustment = heuristics.type_mismatch_penalty
        raw_score += type_agreement_adjustment
        component_scores["type_agreement"] = round(type_agreement_adjustment, 2)
        log.debug(
            f"  s_inferred_type='{s_inferred_type}', r_inferred_type='{r_inferred_type}' -> points={type_agreement_adjustment:.1f}"
        )

        # 6. Token/Length Similarity (Placeholders - Option A)
        length_similarity_points = 0.0  # Placeholder
        token_similarity_points = 0.0  # Placeholder

        # Access median_char_length and median_token_count from s_stats and r_stats
        s_median_len = s_stats.get("median_char_length", 0.0)
        r_median_len = r_stats.get("median_char_length", 0.0)
        s_median_tokens = s_stats.get("median_token_count", 0.0)
        r_median_tokens = r_stats.get("median_token_count", 0.0)

        # --- Actual Calculation for Length Similarity (if data available) ---
        if s_median_len > 0 and r_median_len > 0:
            len_diff = abs(s_median_len - r_median_len)
            max_len = max(s_median_len, r_median_len)
            # Similarity: 1 = identical, 0 = one is zero length other is not, or vastly different
            # This formula gives higher values for closer lengths.
            length_similarity_metric = (
                1.0 - (len_diff / max_len) if max_len > 0 else 0.0
            )
            if length_similarity_metric > heuristics.similarity_high_threshold:
                # Scale bonus by how similar they are beyond the threshold
                bonus_factor = (
                    (length_similarity_metric - heuristics.similarity_high_threshold)
                    / (1.0 - heuristics.similarity_high_threshold)
                    if (1.0 - heuristics.similarity_high_threshold) > 0
                    else 1.0
                )
                length_similarity_points = (
                    heuristics.length_similarity_max_bonus * bonus_factor
                )
                log.debug(
                    f"  Median length similarity ({s_median_len:.1f} vs {r_median_len:.1f}, sim={length_similarity_metric:.2f}) -> points={length_similarity_points:.1f}"
                )

        # --- Actual Calculation for Token Similarity (if data available) ---
        if s_median_tokens > 0 and r_median_tokens > 0:
            token_diff = abs(s_median_tokens - r_median_tokens)
            max_tokens = max(s_median_tokens, r_median_tokens)
            token_similarity_metric = (
                1.0 - (token_diff / max_tokens) if max_tokens > 0 else 0.0
            )
            if token_similarity_metric > heuristics.similarity_high_threshold:
                bonus_factor = (
                    (token_similarity_metric - heuristics.similarity_high_threshold)
                    / (1.0 - heuristics.similarity_high_threshold)
                    if (1.0 - heuristics.similarity_high_threshold) > 0
                    else 1.0
                )
                token_similarity_points = (
                    heuristics.token_count_similarity_max_bonus * bonus_factor
                )
                log.debug(
                    f"  Median token similarity ({s_median_tokens:.1f} vs {r_median_tokens:.1f}, sim={token_similarity_metric:.2f}) -> points={token_similarity_points:.1f}"
                )

        raw_score += length_similarity_points
        raw_score += token_similarity_points
        component_scores["length_similarity"] = round(length_similarity_points, 2)
        component_scores["token_similarity"] = round(token_similarity_points, 2)

        # Final Clamping of Raw Score
        raw_score = max(
            heuristics.raw_score_min_clamp,
            min(raw_score, heuristics.raw_score_max_clamp),
        )
        log.debug(
            f"  Final Raw Score for ('{s_stats.get('name', 'S')}' vs '{r_stats.get('name', 'R')}') (after clamping {heuristics.raw_score_min_clamp}-{heuristics.raw_score_max_clamp}): {raw_score:.2f}"
        )

        return raw_score, component_scores

    def _optimize_mapping_algorithms(
        self,
        mappings: List[Dict[str, Any]],
        source_df: pd.DataFrame = None,
        ref_df: pd.DataFrame = None,
    ) -> List[Dict[str, Any]]:
        """
        Optimize algorithm selection based on column semantics.
        Now uses reservoir samples for faster domain analysis.
        """
        log.info(f"=== OPTIMIZATION START: Processing {len(mappings)} mappings ===")
        optimized_mappings = []

        # Use provided dataframes or fall back to instance attributes
        source_df = source_df if source_df is not None else self.source_df
        ref_df = ref_df if ref_df is not None else self.ref_df

        # Get samplers if available
        source_samplers = getattr(source_df, "attrs", {}).get("column_samplers", {})
        ref_samplers = (
            getattr(ref_df, "attrs", {}).get("column_samplers", {})
            if ref_df is not None
            else {}
        )

        for mapping in mappings:
            optimized_map = mapping.copy()
            source_col = mapping.get("source", "").lower()
            ref_col = mapping.get("ref", "").lower()

            log.info(f"Checking: {mapping.get('source')} -> {mapping.get('ref')}")

            # Check if this is a domain column
            domain_keywords = [
                "domain",
                "website",
                "url",
                "site",
                "web",
                ".com",
                "email",
            ]
            is_domain_mapping = any(
                keyword in source_col for keyword in domain_keywords
            ) or any(keyword in ref_col for keyword in domain_keywords)

            if is_domain_mapping:
                # Use reservoir samples if available, otherwise fall back to full data
                if mapping["source"] in source_samplers:
                    src_data = source_samplers[mapping["source"]].get_sample_series()
                    log.info(
                        f"  Using reservoir sample for {mapping['source']} ({len(src_data)} samples)"
                    )
                else:
                    src_data = source_df[mapping["source"]]

                if self.mode.get() == "match" and mapping["ref"] in ref_samplers:
                    ref_data = ref_samplers[mapping["ref"]].get_sample_series()
                    log.info(
                        f"  Using reservoir sample for {mapping['ref']} ({len(ref_data)} samples)"
                    )
                else:
                    ref_data = (
                        ref_df[mapping["ref"]]
                        if self.mode.get() == "match"
                        else source_df[mapping["ref"]]
                    )

                # Analyze domain matching strategy with enhanced sampling
                strategy = analyze_domain_matching_strategy(
                    src_data, ref_data, use_adaptive=True
                )

                # Apply the strategy
                optimized_map["preferred_algo"] = strategy["algorithm"]
                optimized_map["algo_confidence"] = strategy["confidence"]
                optimized_map["weight"] = strategy.get(
                    "weight", mapping.get("weight", 1.0)
                )

                if strategy.get("preprocessing"):
                    optimized_map["preprocessing"] = strategy["preprocessing"]

                # Store metadata for transparency
                optimized_map["analysis_metadata"] = strategy["metrics"]

                log.info(f"  Domain auto-analysis: {strategy['reason']}")
                log.info(
                    f"  -> Selected: {strategy['algorithm']} (confidence: {strategy['confidence']}, weight: {strategy.get('weight', 1.0)})"
                )

                # Log metrics for analysis
                try:
                    from fmatch.core.utils import log_domain_analysis_metrics

                    log_domain_analysis_metrics(
                        strategy, mapping["source"], mapping["ref"]
                    )
                except Exception as e:
                    log.debug(f"Could not log domain metrics: {e}")

            else:
                log.info(
                    f"  - Not a domain column, keeping {mapping.get('preferred_algo', 'default')}"
                )

            optimized_mappings.append(optimized_map)

        log.info("=== OPTIMIZATION COMPLETE ===")
        return optimized_mappings

    # ─────────────────────────────────────────────────────────────────────────
    # Results and Export Methods
    # ─────────────────────────────────────────────────────────────────────────

    def _scale_weight(self, raw_score: float) -> int:
        """Scale raw score (0-100) to weight tier (1-5)."""
        # Use the tiers from MappingHeuristics
        for threshold, weight in self.mapping_heuristics.raw_to_scaled_weight_tiers:
            if raw_score >= threshold:
                return weight
        return CONFIG["MIN_WEIGHT"]  # Default to minimum weight

    def export_results(self, from_popup: bool = False) -> None:
        """Export results with optimized file handling."""
        if (self.results_df is None or self.results_df.empty) and not self.results_path:
            messagebox.showinfo("No Results", "No results to export")
            return

        file_path = filedialog.asksaveasfilename(
            title="Export Results",
            defaultextension=".csv",
            filetypes=[
                ("CSV files", "*.csv"),
                ("Excel files", "*.xlsx"),
                ("Parquet files", "*.parquet"),
            ],
        )

        if not file_path:
            return

        # Ensure thread pool is active for async operations
        self._ensure_pool_active()

        # Disable export button during export
        if hasattr(self, "btn_export") and self.btn_export:
            self.btn_export.config(state="disabled")

        # Export in background thread
        def export_async():
            try:
                # Optimize: Direct copy for same format
                if self.results_path:
                    # Parquet to Parquet
                    if self.results_path.endswith(
                        ".parquet"
                    ) and file_path.lower().endswith(".parquet"):
                        shutil.copyfile(self.results_path, file_path)
                        log.info(f"Direct copied Parquet results to {file_path}")
                        self.after(
                            0,
                            lambda: messagebox.showinfo(
                                "Export Complete",
                                f"Exported {self.results_row_count:,} results to:\n{Path(file_path).name}",
                            ),
                        )
                        return

                    # CSV to CSV
                    elif self.results_path.endswith(
                        ".csv"
                    ) and file_path.lower().endswith(".csv"):
                        shutil.copyfile(self.results_path, file_path)
                        log.info(f"Direct copied CSV results to {file_path}")
                        self.after(
                            0,
                            lambda: messagebox.showinfo(
                                "Export Complete",
                                f"Exported {self.results_row_count:,} results to:\n{Path(file_path).name}",
                            ),
                        )
                        return

                # Load data for conversion
                if self.results_df is not None and not self.results_df.empty:
                    export_df = self.results_df
                elif self.results_path:
                    # Check file size before loading
                    file_size = Path(self.results_path).stat().st_size
                    if file_size > 500_000_000:  # 500MB
                        log.warning(
                            f"Large results file for export: {file_size / 1_000_000:.1f}MB"
                        )

                    if self.results_path.endswith(".parquet"):
                        try:
                            import pyarrow.parquet as pq

                            export_df = pq.read_table(self.results_path).to_pandas()
                        except ImportError:
                            export_df = pd.read_parquet(self.results_path)
                    else:  # CSV
                        export_df = pd.read_csv(self.results_path)
                    log.info(f"Loaded {len(export_df)} rows from disk for export")
                else:
                    export_df = pd.DataFrame()

                # Export based on format
                if file_path.lower().endswith(".parquet"):
                    try:
                        export_df.to_parquet(file_path, compression="snappy")
                    except Exception:
                        # Fallback to CSV if Parquet fails
                        csv_path = Path(file_path).with_suffix(".csv")
                        export_df.to_csv(csv_path, index=False)
                        self.after(
                            0,
                            lambda: messagebox.showinfo(
                                "Format Changed",
                                f"Parquet export failed. Saved as CSV:\n{csv_path.name}",
                            ),
                        )
                        log.info(f"Parquet failed, exported as CSV: {csv_path}")
                        return

                elif file_path.lower().endswith(".xlsx"):
                    # Handle Excel row limit with thread-safe dialog
                    if len(export_df) > 1_048_576:
                        response = self._askyesno_mainthread(
                            "Large Dataset",
                            f"Excel has a 1,048,576 row limit.\n"
                            f"Your data has {len(export_df):,} rows.\n\n"
                            f"Export first 1M rows to Excel?\n"
                            f"(Choose 'No' to select a different format)",
                        )
                        if response:
                            export_df.head(1_048_576).to_excel(file_path, index=False)
                        else:
                            return
                    else:
                        export_df.to_excel(file_path, index=False)
                else:  # CSV
                    export_df.to_csv(file_path, index=False)

                log.info(f"Exported {len(export_df)} results to {file_path}")

                self.after(
                    0,
                    lambda: messagebox.showinfo(
                        "Export Complete",
                        f"Exported {len(export_df):,} results to:\n{Path(file_path).name}",
                    ),
                )

            except Exception as e:
                log.error(f"Export failed: {e}", exc_info=True)
                err_msg = str(e)
                self.after(
                    0,
                    lambda msg=err_msg: messagebox.showerror(
                        "Export Failed", f"Failed to export: {msg}"
                    ),
                )
            finally:
                # Always re-enable UI
                self.after(
                    0,
                    lambda: (
                        self.status_var.set("Ready"),
                        self.btn_export.config(state="normal")
                        if hasattr(self, "btn_export") and self.btn_export
                        else None,
                    ),
                )

        # Start the export thread
        threading.Thread(target=export_async, daemon=True).start()

    def load_preferences(self) -> dict:
        """Loads user preferences from a separate config file."""
        prefs_path = _config_dir / "fm_gui_prefs.json"
        if not prefs_path.exists():
            return {"telemetry_enabled": False}  # Default to False if no file

        try:
            with open(prefs_path, "r") as f:
                prefs = json.load(f)
            return prefs if isinstance(prefs, dict) else {"telemetry_enabled": False}
        except (json.JSONDecodeError, IOError) as e:
            log.warning(f"Could not load preferences file: {e}")
            return {"telemetry_enabled": False}

    # ─────────────────────────────────────────────────────────────────────────
    # Session Management Methods
    # ─────────────────────────────────────────────────────────────────────────

    def save_session(self) -> None:
        """Save current session state."""
        try:
            session_data = {
                "version": CONFIG["APP_VERSION"],
                "mode": self.mode.get(),
                "source_path": self.src_path,
                "reference_path": self.ref_path,
                "source_encoding": self.source_encoding,
                "ref_encoding": self.ref_encoding,
                "source_id": self.source_id_var.get(),
                "ref_id": self.ref_id_var.get(),
                "mappings": self.mappings,
                "mappings_confirmed": self.mappings_confirmed,
                "algorithm": self.algorithm_var.get(),
                "threshold": self.threshold.get(),
                "block_limit": self.block_limit.get(),
                "ruleset": self.rules_var.get(),
                "results_mode": self.results_mode.get(),
                "webhook_url": self.webhook_var.get(),
                "locking_strategy": self.auto_determined_locking_strategy,
                "norm_rules": {k: v.get() for k, v in self.norm_rule_vars.items()},
                "custom_rules": [row.to_dict() for row in self.rule_rows],
            }

            save_salesforce_config_to_session(self, session_data)

            # ADD B2C CONFIGURATION TO SESSION
            session_data = save_b2c_config(session_data, self)

            # Save to file
            session_path = CONFIG["CONFIG_PATH"]
            with open(session_path, "w") as f:
                json.dump(session_data, f, indent=2)

            log.info(f"Session saved to {session_path}")
            self.threadsafe_update_status("Session saved")

        except Exception as e:
            log.error(f"Error saving session: {e}", exc_info=True)
            messagebox.showerror("Save Error", f"Failed to save session: {e}")

    def load_last_session(self) -> None:
        """Load last saved session."""
        self._loading_session = True
        try:
            session_path = CONFIG["CONFIG_PATH"]

            if not session_path.exists():
                log.info("No previous session found")
                return

            try:
                with open(session_path, "r") as f:
                    session_data = json.load(f)

                load_salesforce_config_from_session(self, session_data)

                # Restore state
                self.mode.set(session_data.get("mode", "match"))
                self.source_id_var.set(session_data.get("source_id", ""))
                self.ref_id_var.set(session_data.get("ref_id", ""))
                self.algorithm_var.set(session_data.get("algorithm", "WRatio"))
                self.threshold.set(
                    session_data.get("threshold", CONFIG["DEFAULT_THRESHOLD"])
                )
                self.block_limit.set(
                    session_data.get("block_limit", CONFIG["DUP_BLOCK_LIMIT"])
                )
                self.rules_var.set(session_data.get("ruleset", ""))
                self.results_mode.set(session_data.get("results_mode", "all"))
                self.webhook_var.set(session_data.get("webhook_url", ""))

                # Restore mappings
                self.mappings = session_data.get("mappings", [])
                # Don't restore confirmation state - always require re-confirmation
                self.mappings_confirmed = False  # Changed from session_data.get()
                self.auto_determined_locking_strategy = (
                    None  # Reset locking strategy too
                )

                # Restore norm rules
                for name, value in session_data.get("norm_rules", {}).items():
                    if name in self.norm_rule_vars:
                        self.norm_rule_vars[name].set(value)

                # LOAD B2C CONFIGURATION
                load_b2c_config(session_data, self)

                # Restore files if they exist
                if (
                    session_data.get("source_path")
                    and Path(session_data["source_path"]).exists()
                ):
                    self._load_data_file(session_data["source_path"], is_source=True)

                if (
                    session_data.get("reference_path")
                    and Path(session_data["reference_path"]).exists()
                ):
                    self._load_data_file(
                        session_data["reference_path"], is_source=False
                    )

                log.info("Session loaded successfully")

            except Exception as e:
                log.error(f"Error loading session: {e}", exc_info=True)
        finally:
            self._loading_session = False

    def reset_session(self) -> None:
        """Reset to a new session."""
        if self.processing:
            messagebox.showwarning("Busy", "Cannot reset while processing")
            return

        if messagebox.askyesno("Reset Session", "Clear all data and start over?"):
            # Shutdown and recreate process pool for clean state
            if hasattr(self, "engine_process_pool") and self.engine_process_pool:
                log.info("Resetting: Shutting down engine process pool")
                try:
                    # Process pool - immediate shutdown with cancel
                    self.engine_process_pool.shutdown(wait=False, cancel_futures=True)
                except Exception as e:
                    log.warning(f"Engine pool shutdown issue: {e}")
                finally:
                    self.engine_process_pool = None

            # Clear data
            self.source_df = None
            self.ref_df = None
            self.results_df = None

            # Clean up disk-spilled results if any
            if hasattr(self, "results_path") and self.results_path:
                try:
                    # Get file size before deletion to subtract from total
                    file_path = Path(self.results_path)
                    if file_path.exists():
                        file_size = file_path.stat().st_size
                        file_path.unlink()
                        self.total_disk_usage = max(
                            0, self.total_disk_usage - file_size
                        )
                        log.info(
                            f"Deleted temporary results file: {self.results_path} ({file_size / 1_000_000:.1f}MB)"
                        )
                        log.info(
                            f"Total temp disk usage: {self.total_disk_usage / 1_000_000:.1f}MB"
                        )
                except Exception as e:
                    log.warning(f"Could not delete temporary results file: {e}")
                self.results_path = None

            self.results_row_count = 0

            self.src_path = None
            self.ref_path = None
            self.source_encoding = None
            self.ref_encoding = None

            # Force garbage collection to free memory
            gc.collect()

            # Clear mappings
            self.mappings.clear()
            self.mappings_confirmed = False
            self.auto_determined_locking_strategy = None
            self.blocking_config = None

            # Reset UI
            self.mode.set("match")
            self.threshold.set(CONFIG["DEFAULT_THRESHOLD"])
            self.algorithm_var.set("WRatio")
            self.source_id_var.set("")
            self.ref_id_var.set("")

            # Clear trees properly
            if hasattr(self, "tree_src") and self.tree_src:
                self.tree_src.delete(*self.tree_src.get_children())
                # Add placeholder
                self.tree_src.insert(
                    "",
                    "end",
                    values=("", "<- Load source data to see columns", "", "", ""),
                )

            if hasattr(self, "tree_ref") and self.tree_ref:
                self.tree_ref.delete(*self.tree_ref.get_children())
                # Add placeholder
                self.tree_ref.insert(
                    "",
                    "end",
                    values=("", "<- Load reference data to see columns", "", "", ""),
                )

            # Clear rules
            for row in self.rule_rows:
                row.destroy()
            self.rule_rows.clear()

            # Update UI labels
            if hasattr(self, "lbl_src") and self.lbl_src:
                self.lbl_src.config(text="No file selected")
            if hasattr(self, "lbl_ref") and self.lbl_ref:
                self.lbl_ref.config(text="No file selected")

            # Row count state & labels
            self._src_rowcount = 0
            self._ref_rowcount = 0
            if hasattr(self, "lbl_src") and self.lbl_src:
                self.lbl_src.config(text="Source: — rows")
            if hasattr(self, "lbl_ref") and self.lbl_ref:
                self.lbl_ref.config(text="Reference: — rows")

            # Clear any cached column stats so heuristics don't render stale algos/dtypes
            self._src_col_stats = {}
            self._ref_col_stats = {}

            # Reset total disk usage tracker
            self.total_disk_usage = 0

            # Update loader banner
            if hasattr(self, "_update_loader_banner"):
                self._update_loader_banner()

            self._refresh_mappings_list()

            # Force a UI refresh so buttons/menus reflect empty state
            self.refresh_ui()

            self.threadsafe_update_status("Session reset")

    # ─────────────────────────────────────────────────────────────────────────
    # UI Building Methods
    # ─────────────────────────────────────────────────────────────────────────

    def _configure_styles(self) -> None:
        """Configure ttk styles."""
        style = ttk.Style(self)
        style.theme_use("clam")

        # Button styles
        style.configure(
            "Primary.TButton",
            foreground="white",
            background=CONFIG["COLORS"]["success"],
            font=("Helvetica", 10, "bold"),
        )
        style.map(
            "Primary.TButton",
            background=[
                ("active", CONFIG["COLORS"]["success"]),
                ("disabled", CONFIG["COLORS"]["disabled_bg"]),
            ],
            foreground=[("disabled", CONFIG["COLORS"]["disabled_fg"])],
        )

        style.configure(
            "Secondary.TButton",
            foreground="white",
            background=CONFIG["COLORS"]["info"],
            font=("Helvetica", 10),
        )

        # Other styles
        style.configure("TLabelframe.Label", font=("Helvetica", 10, "bold"))
        style.configure("Status.TLabel", font=("Helvetica", 9))
        style.configure("Heading.TLabel", font=("Helvetica", 11, "bold"))

        # Fix for visual stripes
        style.configure("TFrame", relief="flat", borderwidth=0)
        style.configure("TLabelframe", relief="groove", borderwidth=1)

        # Styles for the new AlgorithmTransparencyPopup
        style.configure(
            "Tooltip.TFrame",
            background="#2c3e50",  # Dark background
            borderwidth=1,
            relief="solid",
        )
        style.configure(
            "Tooltip.TLabel",
            background="#2c3e50",
            foreground="white",  # White text
        )
        # Add this new style for the rules sub-label
        style.configure(
            "Tooltip.Sublabel.TLabel",
            background="#2c3e50",
            foreground="#bdc3c7",  # Lighter gray for secondary text
        )

    def _create_icon_assets(self):
        """Create or load icon assets."""
        self._icons = {}

        # Try to load PNG icons first
        icon_dir = Path(__file__).parent / "icons"
        if icon_dir.exists():
            for name, filename in [("file", "file_24.png"), ("cloud", "cloud_24.png")]:
                icon_path = icon_dir / filename
                if icon_path.exists():
                    try:
                        self._icons[name] = tk.PhotoImage(file=str(icon_path))
                    except:
                        pass

        # Fallback: Create canvas-based icons
        if "file" not in self._icons:
            self._icons["file"] = self._create_file_icon()
        if "cloud" not in self._icons:
            self._icons["cloud"] = self._create_cloud_icon()

    def _create_file_icon(self):
        """Create a simple file icon using Canvas."""
        icon = tk.Canvas(self, width=24, height=24, highlightthickness=0)
        # Draw file shape
        icon.create_rectangle(4, 2, 20, 22, fill="#e0e0e0", outline="#666", width=2)
        icon.create_polygon(
            12, 2, 20, 8, 20, 2, fill="#e0e0e0", outline="#666", width=2
        )
        # Draw lines
        for y in [10, 13, 16]:
            icon.create_line(7, y, 17, y, fill="#666", width=1)
        return icon

    def build_ui(self) -> None:
        """Build the main UI."""
        # Configure styles
        self._configure_styles()

        # Menu
        self._build_menu()

        # Configure grid
        self.columnconfigure(0, weight=1)

        row = 0

        # Top section (mode and files)
        top_frame = self._build_top_section(self)  # SpaceOptimizedUI method
        top_frame.grid(row=row, column=0, sticky="ew", padx=10, pady=5)
        row += 1

        # Mapping section
        mapping_frame = self._build_field_mapping_section(
            self
        )  # SpaceOptimizedUI method
        mapping_frame.grid(row=row, column=0, sticky="nsew", padx=10, pady=5)
        self.rowconfigure(row, weight=1)
        row += 1

        # Active mappings
        self.active_mappings_outer_lf = self._build_active_mappings_section(
            self
        )  # SpaceOptimizedUI method
        # Note: Don't grid it here - let refresh_ui handle visibility
        row += 1

        # Configuration tabs
        notebook = self._build_config_notebook(self)
        notebook.grid(row=row, column=0, sticky="nsew", padx=10, pady=5)
        self.rowconfigure(row, weight=1)
        row += 1

        # Controls toolbar
        controls = self._build_controls_toolbar(self)
        controls.grid(row=row, column=0, sticky="ew", padx=10, pady=5)
        row += 1

        # Status bar
        status_bar = self._build_status_bar(self)
        status_bar.grid(row=row, column=0, sticky="sew", padx=10, pady=(5, 10))

        # Initial refresh
        self.refresh_ui()

    def _build_menu(self) -> None:
        """Build the menu bar."""
        menubar = tk.Menu(self)
        self.config(menu=menubar)

        # File menu
        file_menu = tk.Menu(menubar, tearoff=0)
        file_menu.add_command(
            label="New Session", command=self.reset_session, accelerator="Ctrl+N"
        )
        file_menu.add_command(label="Load Last Session", command=self.load_last_session)
        file_menu.add_command(
            label="Save Session", command=self.save_session, accelerator="Ctrl+S"
        )
        file_menu.add_separator()
        file_menu.add_command(
            label="Save Mapping Template...", command=self.save_mapping_template
        )
        file_menu.add_command(
            label="Load Mapping Template...", command=self.load_mapping_template
        )
        file_menu.add_separator()
        file_menu.add_command(
            label="Export Results", command=self.export_results, accelerator="Ctrl+E"
        )
        file_menu.add_separator()
        file_menu.add_command(label="Exit", command=self._on_close)
        menubar.add_cascade(label="File", menu=file_menu)
        self.file_menu = file_menu

        # View menu
        view_menu = tk.Menu(menubar, tearoff=0)
        view_menu.add_command(
            label="Show Blocking Strategy", command=self.show_blocking_dialog
        )
        menubar.add_cascade(label="View", menu=view_menu)

        # Analysis menu
        analysis_menu = tk.Menu(menubar, tearoff=0)
        analysis_menu.add_command(
            label="Show Last Blocking Analysis",
            command=self.show_blocking_analysis,
            accelerator="Ctrl+B",
        )
        analysis_menu.add_command(
            label="Show Algorithm Details", command=self._show_algorithm_details
        )
        menubar.add_cascade(label="Analysis", menu=analysis_menu)

        # Settings menu (NEW)
        settings_menu = tk.Menu(menubar, tearoff=0)

        # Integrations submenu
        integrations_menu = tk.Menu(settings_menu, tearoff=0)
        integrations_menu.add_command(
            label="Salesforce...", command=self.show_salesforce_settings
        )
        integrations_menu.add_command(
            label="HubSpot...", command=self.show_hubspot_settings
        )
        settings_menu.add_cascade(label="Integrations", menu=integrations_menu)

        settings_menu.add_separator()
        settings_menu.add_command(
            label="Preprocessing...", command=self.show_preprocessing_settings
        )
        settings_menu.add_command(
            label="Custom Match Rules...", command=self.show_custom_rules_settings
        )

        menubar.add_cascade(label="Settings", menu=settings_menu)

        # Help menu
        help_menu = tk.Menu(menubar, tearoff=0)
        help_menu.add_command(label="About", command=self.show_about)
        menubar.add_cascade(label="Help", menu=help_menu)

        # Keyboard shortcuts
        self.bind_all("<Control-n>", lambda e: self.reset_session())
        self.bind_all("<Control-s>", lambda e: self.save_session())
        self.bind_all("<Control-e>", lambda e: self.export_results())
        self.bind_all("<Control-b>", lambda e: self.show_blocking_analysis())

    def _build_mapping_section(self, parent) -> ttk.LabelFrame:
        """Build the field mapping section."""
        frame = ttk.LabelFrame(parent, text=" Field Mapping ")
        frame.columnconfigure(0, weight=1)
        frame.columnconfigure(1, weight=0)
        frame.columnconfigure(2, weight=1)
        frame.rowconfigure(0, weight=1)

        # Source columns tree
        src_frame = ttk.Frame(frame)
        src_frame.grid(row=0, column=0, sticky="nsew", padx=5, pady=5)
        src_frame.rowconfigure(1, weight=1)
        src_frame.columnconfigure(0, weight=1)

        ttk.Label(src_frame, text="Source Columns").grid(row=0, column=0, sticky="w")

        columns = ("suggestion", "col_name", "fill_rate", "distinctness", "type")
        self.tree_src = ttk.Treeview(
            src_frame, columns=columns, show="headings", selectmode="browse", height=6
        )

        # Configure columns
        self.tree_src.heading("suggestion", text="💡")
        self.tree_src.heading("col_name", text="Column Name")
        self.tree_src.heading("fill_rate", text="Fill %")
        self.tree_src.heading("distinctness", text="Distinct")
        self.tree_src.heading("type", text="Type")

        self.tree_src.column("suggestion", width=30, stretch=False)
        self.tree_src.column("col_name", width=180)
        self.tree_src.column("fill_rate", width=70, anchor="e")
        self.tree_src.column("distinctness", width=120, anchor="e")
        self.tree_src.column("type", width=90)

        self.tree_src.grid(row=1, column=0, sticky="nsew")

        # Scrollbar
        vsb = ttk.Scrollbar(src_frame, orient="vertical", command=self.tree_src.yview)
        vsb.grid(row=1, column=1, sticky="ns")
        self.tree_src.configure(yscrollcommand=vsb.set)

        # Controls
        ctrl_frame = ttk.Frame(frame)
        ctrl_frame.grid(row=0, column=1, sticky="ns", padx=10, pady=5)

        ttk.Label(ctrl_frame, text="Weight").pack(pady=(20, 5))
        self.spin_weight = ttk.Spinbox(
            ctrl_frame, from_=CONFIG["MIN_WEIGHT"], to=CONFIG["MAX_WEIGHT"], width=5
        )
        self.spin_weight.set(CONFIG["DEFAULT_WEIGHT"])
        self.spin_weight.pack()

        ttk.Button(
            ctrl_frame, text="Auto-Map Fields", command=self.on_auto_map_fields_clicked
        ).pack(pady=(20, 5), fill="x")

        self.btn_add_mapping = ttk.Button(
            ctrl_frame, text="Add ->", command=self.add_mapping
        )
        self.btn_add_mapping.pack(pady=5, fill="x")

        self.btn_remove_mapping = ttk.Button(
            ctrl_frame, text="Remove", command=self.remove_mapping
        )
        self.btn_remove_mapping.pack(pady=5, fill="x")

        self.btn_clear_mappings = ttk.Button(
            ctrl_frame, text="Clear All", command=self.clear_mappings
        )
        self.btn_clear_mappings.pack(pady=5, fill="x")

        # Reference columns tree
        ref_frame = ttk.Frame(frame)
        ref_frame.grid(row=0, column=2, sticky="nsew", padx=5, pady=5)
        ref_frame.rowconfigure(1, weight=1)
        ref_frame.columnconfigure(0, weight=1)

        self.lbl_tree_ref = ttk.Label(ref_frame, text="Reference Columns")
        self.lbl_tree_ref.grid(row=0, column=0, sticky="w")

        self.tree_ref = ttk.Treeview(
            ref_frame, columns=columns, show="headings", selectmode="browse", height=6
        )

        # Configure columns (same as source)
        self.tree_ref.heading("suggestion", text="💡")
        self.tree_ref.heading("col_name", text="Column Name")
        self.tree_ref.heading("fill_rate", text="Fill %")
        self.tree_ref.heading("distinctness", text="Distinct")
        self.tree_ref.heading("type", text="Type")

        self.tree_ref.column("suggestion", width=30, stretch=False)
        self.tree_ref.column("col_name", width=180)
        self.tree_ref.column("fill_rate", width=70, anchor="e")
        self.tree_ref.column("distinctness", width=120, anchor="e")
        self.tree_ref.column("type", width=90)

        self.tree_ref.grid(row=1, column=0, sticky="nsew")

        # Scrollbar
        vsb2 = ttk.Scrollbar(ref_frame, orient="vertical", command=self.tree_ref.yview)
        vsb2.grid(row=1, column=1, sticky="ns")
        self.tree_ref.configure(yscrollcommand=vsb2.set)

        return frame

    def _build_config_notebook(self, parent) -> ttk.Notebook:
        """Build the configuration notebook."""
        notebook = ttk.Notebook(parent)

        # NEW: Matching Strategy tab (FIRST)
        strategy_tab = ttk.Frame(notebook)
        notebook.add(strategy_tab, text="Matching Strategy")
        self._build_strategy_tab(strategy_tab)

        # Webhook tab (now second)
        webhook_tab = ttk.Frame(notebook)
        notebook.add(webhook_tab, text="Webhook")
        self._build_webhook_tab(webhook_tab)

        # Advanced tab
        adv_tab = ttk.Frame(notebook)
        notebook.add(adv_tab, text="Advanced")
        self._build_advanced_tab(adv_tab)

        return notebook

    # In the FuzzyMatcherApp class, ADD these three new methods

    def _build_strategy_tab(self, parent) -> None:
        """Build the matching strategy tab with ensemble scoring prominent."""
        parent.columnconfigure(0, weight=1)

        # Main frame with padding
        main_frame = ttk.Frame(parent, padding="10")
        main_frame.pack(fill=tk.BOTH, expand=True)

        # Algorithm selection frame
        algo_frame = ttk.LabelFrame(main_frame, text="Scoring Method", padding="15")
        algo_frame.pack(fill="x", pady=(0, 20))

        # Ensemble Score option (default)
        ensemble_radio = ttk.Radiobutton(
            algo_frame,
            text="🎯 Ensemble Score (Recommended)",
            variable=self.ensemble_var,
            value=True,
            command=self._on_scoring_method_change,
        )
        ensemble_radio.pack(anchor="w", pady=(0, 5))

        ensemble_desc = ttk.Label(
            algo_frame,
            text="Automatically selects the best algorithm for each field based on its data characteristics.\n"
            "Uses TokenSetRatio for company names, Exact for emails, JaroWinkler for names, etc.",
            font=("TkDefaultFont", 9),
            foreground="gray",
            wraplength=700,
        )
        ensemble_desc.pack(anchor="w", padx=(20, 0), pady=(0, 15))

        # Single algorithm option
        single_radio = ttk.Radiobutton(
            algo_frame,
            text="Single Algorithm",
            variable=self.ensemble_var,
            value=False,
            command=self._on_scoring_method_change,
        )
        single_radio.pack(anchor="w", pady=(0, 5))

        # Single algorithm selector (disabled when ensemble is selected)
        self.single_algo_frame = ttk.Frame(algo_frame)
        self.single_algo_frame.pack(anchor="w", padx=(20, 0), pady=(0, 10))

        ttk.Label(self.single_algo_frame, text="Algorithm:").pack(
            side="left", padx=(0, 10)
        )
        self.single_algo_combo = ttk.Combobox(
            self.single_algo_frame,
            textvariable=self.algorithm_var,
            values=list(ALGORITHMS.keys()),
            state="disabled" if self.ensemble_var.get() else "readonly",
            width=20,
        )
        self.single_algo_combo.pack(side="left")

        # Initialize the view
        self._on_scoring_method_change()

    def _on_scoring_method_change(self):
        """Handle scoring method change."""
        is_ensemble = self.ensemble_var.get()

        self.enable_multi_algo.set(is_ensemble)
        self.single_algo_combo.config(state="disabled" if is_ensemble else "readonly")

        if not is_ensemble:
            # Ensure a valid single algorithm is selected
            if self.algorithm_var.get() not in ALGORITHMS:
                self.algorithm_var.set("WRatio")  # Default to WRatio

        self.refresh_ui()

    def _build_webhook_tab(self, parent) -> None:
        """Build webhook tab with collapsible content."""
        parent.columnconfigure(0, weight=1)

        # Collapsible header
        header_frame = ttk.Frame(parent, cursor="hand2")
        header_frame.grid(row=0, column=0, sticky="ew", padx=10, pady=5)

        self.webhook_expanded = tk.BooleanVar(value=False)

        # Chevron and title
        self.webhook_chevron = ttk.Label(
            header_frame, text="▶", font=("TkDefaultFont", 10)
        )
        self.webhook_chevron.pack(side=tk.LEFT, padx=(0, 5))

        ttk.Label(
            header_frame,
            text="Webhook Configuration",
            font=("TkDefaultFont", 11, "bold"),
        ).pack(side=tk.LEFT)

        ttk.Label(
            header_frame,
            text="(click to expand)",
            font=("TkDefaultFont", 9),
            foreground="gray",
        ).pack(side=tk.LEFT, padx=10)

        # Content frame (hidden initially)
        self.webhook_content = ttk.Frame(parent)

        # Webhook frame inside content
        webhook_frame = ttk.LabelFrame(
            self.webhook_content, text="Webhook Settings", padding=10
        )
        webhook_frame.pack(fill="both", expand=True, padx=10, pady=5)
        webhook_frame.columnconfigure(1, weight=1)

        # URL input
        ttk.Label(webhook_frame, text="Webhook URL:").grid(
            row=0, column=0, sticky="w", padx=5, pady=5
        )

        self.webhook_entry = ttk.Entry(
            webhook_frame, textvariable=self.webhook_var, width=60
        )
        self.webhook_entry.grid(row=0, column=1, sticky="ew", padx=5, pady=5)
        ToolTip(
            self.webhook_entry, "POST JSON with job status when processing completes"
        )

        # Instructions
        instructions = ttk.Label(
            webhook_frame,
            text="Enter a webhook URL to receive match results. The results will be POSTed as JSON.",
            wraplength=500,
        )
        instructions.grid(
            row=1, column=0, columnspan=2, sticky="w", padx=5, pady=(10, 0)
        )

        # Test button
        ttk.Button(webhook_frame, text="Test Webhook", command=self.test_webhook).grid(
            row=2, column=1, sticky="e", padx=5, pady=(10, 0)
        )

        # Toggle function
        def toggle_webhook(event=None):
            expanded = self.webhook_expanded.get()
            self.webhook_expanded.set(not expanded)
            if not expanded:
                self.webhook_chevron.config(text="▼")
                self.webhook_content.grid(row=1, column=0, sticky="ew", padx=10, pady=5)
            else:
                self.webhook_chevron.config(text="▶")
                self.webhook_content.grid_forget()

        # Bind click events
        header_frame.bind("<Button-1>", toggle_webhook)
        for child in header_frame.winfo_children():
            child.bind("<Button-1>", toggle_webhook)

    def _build_advanced_tab(self, parent) -> None:
        """Build advanced tab with collapsible content."""
        parent.columnconfigure(0, weight=1)

        # Collapsible header
        header_frame = ttk.Frame(parent, cursor="hand2")
        header_frame.grid(row=0, column=0, sticky="ew", padx=10, pady=5)

        self.advanced_expanded = tk.BooleanVar(value=True)  # Start expanded

        # Chevron and title
        self.advanced_chevron = ttk.Label(
            header_frame, text="▼", font=("TkDefaultFont", 10)
        )
        self.advanced_chevron.pack(side=tk.LEFT, padx=(0, 5))

        ttk.Label(
            header_frame,
            text="Blocking Configuration",
            font=("TkDefaultFont", 11, "bold"),
        ).pack(side=tk.LEFT)

        ttk.Label(
            header_frame,
            text="(click to collapse)",
            font=("TkDefaultFont", 9),
            foreground="gray",
        ).pack(side=tk.LEFT, padx=10)

        # Content frame
        self.advanced_content = ttk.Frame(parent)
        self.advanced_content.grid(row=1, column=0, sticky="ew", padx=10, pady=5)
        self.advanced_content.columnconfigure(1, weight=1)

        # Blocking configuration content
        ttk.Label(
            self.advanced_content,
            text="Blocking groups similar records to speed up processing:",
            style="Status.TLabel",
        ).grid(row=0, column=0, columnspan=2, sticky="w", padx=5, pady=5)

        # Source blocking column - STORE AS INSTANCE ATTRIBUTE
        self.lbl_src_block = ttk.Label(
            self.advanced_content, text="Source Blocking Column:"
        )
        self.lbl_src_block.grid(row=1, column=0, sticky="w", padx=5, pady=5)

        self.cbo_src_block = ttk.Combobox(
            self.advanced_content,
            textvariable=self.source_block_col_var,
            state="readonly",
        )
        self.cbo_src_block.grid(row=1, column=1, sticky="ew", padx=5, pady=5)

        # Reference blocking column - STORE AS INSTANCE ATTRIBUTE
        self.lbl_ref_block = ttk.Label(
            self.advanced_content, text="Reference Blocking Column:"
        )
        self.lbl_ref_block.grid(row=2, column=0, sticky="w", padx=5, pady=5)

        self.cbo_ref_block = ttk.Combobox(
            self.advanced_content, textvariable=self.ref_block_col_var, state="readonly"
        )
        self.cbo_ref_block.grid(row=2, column=1, sticky="ew", padx=5, pady=5)

        # Block key length
        ttk.Label(self.advanced_content, text="Block Key Length:").grid(
            row=3, column=0, sticky="w", padx=5, pady=5
        )

        ttk.Spinbox(
            self.advanced_content,
            textvariable=self.block_key_length_var,
            from_=1,
            to=10,
            width=10,
        ).grid(row=3, column=1, sticky="w", padx=5, pady=5)

        # Auto-select button
        ttk.Button(
            self.advanced_content,
            text="Auto-Select Blocking",
            command=self.on_auto_blocking_clicked,
        ).grid(row=4, column=0, columnspan=2, padx=5, pady=10)

        # Toggle function
        def toggle_advanced(event=None):
            expanded = self.advanced_expanded.get()
            self.advanced_expanded.set(not expanded)
            if not expanded:
                self.advanced_chevron.config(text="▼")
                self.advanced_content.grid(
                    row=1, column=0, sticky="ew", padx=10, pady=5
                )
                header_frame.winfo_children()[2].config(text="(click to collapse)")
            else:
                self.advanced_chevron.config(text="▶")
                self.advanced_content.grid_forget()
                header_frame.winfo_children()[2].config(text="(click to expand)")

        # Bind click events
        header_frame.bind("<Button-1>", toggle_advanced)
        for child in header_frame.winfo_children():
            child.bind("<Button-1>", toggle_advanced)

    def _build_rules_tab(self, parent) -> None:
        """Build custom rules tab content."""
        parent.columnconfigure(0, weight=1)
        parent.rowconfigure(1, weight=1)

        # Instructions
        instructions = (
            "Define custom matching rules to fine-tune the matching process.\n"
            "Rules are evaluated in order and scores are combined."
        )
        ttk.Label(parent, text=instructions, wraplength=600).grid(
            row=0, column=0, sticky="w", padx=5, pady=5
        )

        # Rules container
        container_frame = ttk.Frame(parent)
        container_frame.grid(row=1, column=0, sticky="nsew", padx=5, pady=5)
        container_frame.columnconfigure(0, weight=1)
        container_frame.rowconfigure(0, weight=1)

        # Scrollable frame
        canvas = tk.Canvas(container_frame, height=200)
        scrollbar = ttk.Scrollbar(
            container_frame, orient="vertical", command=canvas.yview
        )
        self.rules_container = ttk.Frame(canvas)

        canvas.configure(yscrollcommand=scrollbar.set)
        canvas_frame = canvas.create_window(
            (0, 0), window=self.rules_container, anchor="nw"
        )

        canvas.grid(row=0, column=0, sticky="nsew")
        scrollbar.grid(row=0, column=1, sticky="ns")

        # Configure canvas scrolling
        def _configure_canvas(event=None):
            canvas.configure(scrollregion=canvas.bbox("all"))
            canvas.itemconfig(canvas_frame, width=canvas.winfo_width())

        self.rules_container.bind("<Configure>", _configure_canvas)
        canvas.bind("<Configure>", _configure_canvas)

        # Add rule button
        btn_frame = ttk.Frame(parent)
        btn_frame.grid(row=2, column=0, sticky="ew", padx=5, pady=5)

        ttk.Button(btn_frame, text="+ Add Rule", command=self._add_rule_row).pack(
            side="left"
        )

    def test_webhook(self):
        """Test the configured webhook in background thread."""
        webhook_url = self.webhook_var.get()
        if not webhook_url:
            messagebox.showwarning("No URL", "Please enter a webhook URL to test.")
            return

        # Ensure thread pool is active
        self._ensure_pool_active()

        # Run the test in a background thread to avoid blocking UI
        def test_async():
            try:
                import requests
                from time import time

                test_payload = {
                    "event": "test",
                    "message": "This is a test notification from FoundryMatch.",
                    "timestamp": time.time(),
                }
                response = requests.post(webhook_url, json=test_payload, timeout=10)
                response.raise_for_status()
                # Update UI on main thread
                self.after(
                    0,
                    lambda: messagebox.showinfo(
                        "Test Successful",
                        f"Successfully sent test notification to:\n{webhook_url}\n\n"
                        f"Status Code: {response.status_code}",
                    ),
                )
            except requests.exceptions.RequestException as exc:
                # Update UI on main thread
                self.after(
                    0,
                    lambda err=exc: messagebox.showerror(
                        "Test Failed", f"Failed to send test notification:\n{err}"
                    ),
                )

        # Start the background thread
        threading.Thread(target=test_async, daemon=True).start()
        self.status_var.set("Testing webhook...")

    def _build_controls_toolbar(self, parent) -> ttk.Frame:
        """Build the controls toolbar."""
        frame = ttk.Frame(parent)
        frame.columnconfigure(6, weight=1)

        # Algorithm selection
        alg_frame = ttk.Frame(frame)
        alg_frame.grid(row=0, column=0, sticky="w", padx=5)

        ttk.Label(alg_frame, text="Algorithm:").pack(side="left", padx=(0, 5))

        self.cbo_alg = ttk.OptionMenu(
            alg_frame,
            self.algorithm_var,
            self.algorithm_var.get(),
            *ALGORITHMS.keys(),
            command=self.on_algorithm_change,
        )
        self.cbo_alg.pack(side="left")

        # Create tooltip that will be updated
        self.algorithm_menu_tooltip = ToolTip(self.cbo_alg, "Select matching algorithm")
        self.update_algorithm_tooltip()

        # Ruleset selection
        ruleset_frame = ttk.Frame(frame)
        ruleset_frame.grid(row=0, column=1, sticky="w", padx=5)

        ttk.Label(ruleset_frame, text="Ruleset:").pack(side="left", padx=(0, 5))

        # Get available rulesets
        ruleset_names = (
            list(AVAILABLE_RULESETS.keys()) if AVAILABLE_RULESETS else ["default"]
        )
        self.cbo_ruleset = ttk.Combobox(
            ruleset_frame,
            textvariable=self.rules_var,
            values=ruleset_names,
            state="readonly",
            width=15,
        )
        self.cbo_ruleset.pack(side="left")
        if "default" in ruleset_names:
            self.rules_var.set("default")
        ToolTip(self.cbo_ruleset, "Select preprocessing ruleset")

        # Threshold
        thresh_frame = ttk.Frame(frame)
        thresh_frame.grid(row=0, column=2, sticky="w", padx=5)

        ttk.Label(thresh_frame, text="Threshold:").pack(side="left", padx=(0, 5))

        self.spin_thresh = ttk.Spinbox(
            thresh_frame, textvariable=self.threshold, from_=0, to=100, width=5
        )
        self.spin_thresh.pack(side="left")

        ttk.Label(thresh_frame, text="%").pack(side="left", padx=(2, 0))

        # Block limit (duplicate mode)
        self.limit_frame = ttk.Frame(frame)
        self.limit_frame.grid(row=0, column=2, sticky="w", padx=20)

        self.lbl_block_limit = ttk.Label(
            self.limit_frame, text="Block Limit:", width=12
        )  # Add width
        self.lbl_block_limit.pack(side="left", padx=(0, 5))

        self.spin_limit = ttk.Spinbox(
            self.limit_frame, textvariable=self.block_limit, from_=10, to=1000, width=6
        )
        self.spin_limit.pack(side="left")

        # Results mode selector
        results_mode_frame = ttk.Frame(frame)
        results_mode_frame.grid(row=0, column=3, sticky="w", padx=20)

        ttk.Label(results_mode_frame, text="Results:").pack(side="left", padx=(0, 5))

        self.cbo_results_mode = ttk.Combobox(
            results_mode_frame,
            textvariable=self.results_mode,
            values=["All Matches", "Best Match Only"],
            state="readonly",
            width=15,
        )
        self.cbo_results_mode.pack(side="left")

        # Map display names to internal values when the selection changes
        def on_results_mode_change(event):
            display_val = self.cbo_results_mode.get()
            internal_val = "best" if display_val == "Best Match Only" else "all"
            self.results_mode.set(internal_val)

        self.cbo_results_mode.bind("<<ComboboxSelected>>", on_results_mode_change)

        # Set initial display value from the variable
        current_mode = self.results_mode.get()
        initial_display = "Best Match Only" if current_mode == "best" else "All Matches"
        self.cbo_results_mode.set(initial_display)

        # Spacer
        ttk.Frame(frame).grid(row=0, column=4, sticky="ew")

        # Buttons
        btn_frame = ttk.Frame(frame)
        btn_frame.grid(row=0, column=5, sticky="e", padx=5)

        self.btn_start = ttk.Button(
            btn_frame,
            text="Start Processing",
            command=self.start_processing,
            style="Primary.TButton",
        )
        self.btn_start.pack(side="left", padx=2)

        self.btn_cancel = ttk.Button(
            btn_frame, text="Cancel", command=self.cancel_processing, state="disabled"
        )
        self.btn_cancel.pack(side="left", padx=2)

        self.btn_export = ttk.Button(
            btn_frame,
            text="Export Results",
            command=self.export_results,
            state="disabled",
        )
        self.btn_export.pack(side="left", padx=2)

        add_salesforce_results_button(self, btn_frame)

        return frame

    def _build_status_bar(self, parent) -> ttk.Frame:
        """Build an enhanced, consolidated status bar with dedicated areas for all info."""
        frame = ttk.Frame(parent)
        # Configure the main status label's column to expand and fill space
        frame.columnconfigure(2, weight=1)

        # Left-aligned widgets
        # --------------------
        # Progress bar
        self.bar = ttk.Progressbar(frame, mode="determinate", length=250)
        self.bar.grid(row=0, column=0, sticky="w", padx=(5, 2), pady=2)

        # Progress percentage
        self.lbl_progress = ttk.Label(
            frame, text="0.0%", width=6, style="Status.TLabel"
        )
        self.lbl_progress.grid(row=0, column=1, sticky="w", padx=(0, 10), pady=2)

        # Center-expanding widget
        # -----------------------
        # Main status message
        self.lbl_status = ttk.Label(frame, text="Ready", style="Status.TLabel")
        self.lbl_status.grid(row=0, column=2, sticky="ew", padx=5, pady=2)

        # Right-aligned widgets
        # ---------------------
        # B2C Domain Filter stats
        self.lbl_b2c_stats = ttk.Label(
            frame, text="", style="Status.TLabel", foreground=CONFIG["COLORS"]["info"]
        )
        self.lbl_b2c_stats.grid(row=0, column=3, sticky="e", padx=(0, 10), pady=2)

        # Comparisons Per Second display
        self.lbl_cps = ttk.Label(frame, text="", width=20, style="Status.TLabel")
        self.lbl_cps.grid(row=0, column=4, sticky="e", padx=(0, 10), pady=2)

        # Estimated Time Remaining
        self.lbl_eta = ttk.Label(frame, text="", width=15, style="Status.TLabel")
        self.lbl_eta.grid(row=0, column=5, sticky="e", padx=(0, 10), pady=2)

        # V2 Progress System Phase indicator
        self.lbl_phase = ttk.Label(frame, text="", width=10, style="Status.TLabel")
        self.lbl_phase.grid(row=0, column=6, sticky="e", padx=(0, 10), pady=2)

        # Processing Mode indicator
        self.lbl_mode = ttk.Label(frame, text="Mode -> Scale", style="Status.TLabel")
        self.lbl_mode.grid(row=0, column=7, sticky="e", padx=(0, 10), pady=2)

        # Modern Engine badge
        badge = ttk.Label(
            frame,
            text="⚡ Modern Engine",
            style="Status.TLabel",
            foreground=CONFIG["COLORS"]["success"],
        )
        badge.grid(row=0, column=8, sticky="e", padx=(0, 5), pady=2)

        return frame

    # ─────────────────────────────────────────────────────────────────────────
    # UI Helper Methods
    # ─────────────────────────────────────────────────────────────────────────

    def threadsafe_update_status(self, message: str, is_error: bool = False) -> None:
        """Thread-safe status bar update."""

        def update():
            if hasattr(self, "lbl_status") and self.lbl_status:
                self.lbl_status.config(text=message)
                if is_error:
                    self.lbl_status.config(foreground=CONFIG["COLORS"]["danger"])
                else:
                    self.lbl_status.config(foreground="black")

        # Schedule the update on the main thread
        self.after_idle(update)

    def refresh_ui(self) -> None:
        """Debounces UI refresh calls to prevent flicker from rapid state changes."""
        if hasattr(self, "_refresh_after_id"):
            self.after_cancel(self._refresh_after_id)
        # Schedule the actual refresh to run after 50ms of inactivity
        self._refresh_after_id = self._after(50, self._do_refresh_ui)

    def _do_refresh_ui(self) -> None:
        """The actual UI refresh implementation."""
        if getattr(self, "_refreshing", False):
            return
        self._refreshing = True
        try:
            # Check if window still exists
            if not self.winfo_exists():
                return
            # Update mode-specific visibility
            is_match_mode = self.mode.get() == "match"
            is_dedupe_mode = self.mode.get() == "dedupe"

            # === DEDUPE MODE UI CHANGES START HERE ===

            # Hide reference slot in Data Input section (from SpaceOptimizedUI)
            if hasattr(self, "ref_slot") and self.ref_slot:
                if is_dedupe_mode:
                    self.ref_slot.grid_forget()
                else:
                    self.ref_slot.grid(row=1, column=1, sticky="ew", padx=(4, 0))

            # Handle Field Mapping PanedWindow properly
            if (
                hasattr(self, "field_pane")
                and hasattr(self, "tree_src")
                and hasattr(self, "tree_ref")
                and self.tree_src is not None
                and self.tree_ref is not None
            ):
                # Get the parent frames of the trees
                src_frame = self.tree_src.master
                ref_frame = self.tree_ref.master

                # Get current panes
                panes = self.field_pane.panes()

                if is_dedupe_mode:
                    # Remove reference pane if it's there
                    if len(panes) > 1:
                        self.field_pane.forget(ref_frame)
                    # Update source label
                    if src_frame:
                        src_label = src_frame.winfo_children()[0]
                        if isinstance(src_label, ttk.Label):
                            src_label.config(
                                text="Select Columns for Duplicate Detection"
                            )
                else:  # Match mode
                    # Ensure both panes are present
                    if len(panes) == 1:
                        # Re-add reference pane if it was removed
                        self.field_pane.add(ref_frame, weight=1)
                    # Update source label back to normal
                    if src_frame:
                        src_label = src_frame.winfo_children()[0]
                        if isinstance(src_label, ttk.Label):
                            src_label.config(text="Source Columns")
                    # Update reference label
                    if ref_frame:
                        ref_label = ref_frame.winfo_children()[0]
                        if isinstance(ref_label, ttk.Label):
                            ref_label.config(text="Reference Columns")

            # === DEDUPE MODE UI CHANGES END HERE ===

            # Advanced tab (blocking) visibility
            if hasattr(self, "lbl_src_block"):
                if is_match_mode:
                    self.lbl_src_block.config(text="Source Blocking Column:")
                    if hasattr(self, "lbl_ref_block"):
                        self.lbl_ref_block.grid(
                            row=2, column=0, sticky="w", padx=5, pady=5
                        )
                    if hasattr(self, "cbo_ref_block"):
                        self.cbo_ref_block.grid(
                            row=2, column=1, sticky="ew", padx=5, pady=5
                        )
                else:  # Dedupe mode
                    self.lbl_src_block.config(text="Blocking Column:")
                    if hasattr(self, "lbl_ref_block"):
                        self.lbl_ref_block.grid_forget()
                    if hasattr(self, "cbo_ref_block"):
                        self.cbo_ref_block.grid_forget()

            # Reference controls from SpaceOptimizedUI
            if hasattr(self, "reference_widgets"):
                ref_widgets = self.reference_widgets
                if is_match_mode:
                    ref_widgets["action"].config(state="normal")
                    ref_widgets["id_combo"].config(
                        state="readonly" if self.ref_df is not None else "disabled"
                    )
                else:
                    ref_widgets["action"].config(state="disabled")
                    ref_widgets["id_combo"].config(state="disabled")
            if hasattr(self, "tree_ref") and isinstance(self.tree_ref, ttk.Treeview):
                self.tree_ref.config(selectmode="browse" if is_match_mode else "none")

            # Block limit visibility
            if self.limit_frame:
                if is_match_mode:
                    self.limit_frame.grid_remove()
                else:
                    self.limit_frame.grid()

            # Update ID column dropdowns using SpaceOptimizedUI widgets
            if hasattr(self, "source_widgets"):
                src_widgets = self.source_widgets
                if self.source_df is not None:
                    cols = list(self.source_df.columns)
                    src_widgets["id_combo"].config(values=cols, state="readonly")
                else:
                    src_widgets["id_combo"].config(state="disabled")

            if hasattr(self, "reference_widgets"):
                ref_widgets = self.reference_widgets
                if is_match_mode and self.ref_df is not None:
                    cols = list(self.ref_df.columns)
                    ref_widgets["id_combo"].config(values=cols, state="readonly")
                else:
                    ref_widgets["id_combo"].config(state="disabled")

            # Update blocking column dropdowns
            if self.cbo_src_block and self.source_df is not None:
                self.cbo_src_block.config(values=list(self.source_df.columns))

            if self.cbo_ref_block:
                if is_match_mode and self.ref_df is not None:
                    self.cbo_ref_block.config(values=list(self.ref_df.columns))
                    self.cbo_ref_block.config(state="readonly")
                else:
                    self.cbo_ref_block.config(state="disabled")

            # Button states with defensive checks
            can_process = (
                not self.processing
                and self.mappings_confirmed
                and self.source_df is not None
                and (not is_match_mode or self.ref_df is not None)
            )

            if (
                hasattr(self, "btn_start")
                and self.btn_start
                and hasattr(self.btn_start, "winfo_exists")
                and self.btn_start.winfo_exists()
            ):
                try:
                    self.btn_start.config(state="normal" if can_process else "disabled")
                except tk.TclError:
                    pass

            if (
                hasattr(self, "btn_cancel")
                and self.btn_cancel
                and hasattr(self.btn_cancel, "winfo_exists")
                and self.btn_cancel.winfo_exists()
            ):
                try:
                    self.btn_cancel.config(
                        state="normal" if self.processing else "disabled"
                    )
                except tk.TclError:
                    pass

            if (
                hasattr(self, "btn_export")
                and self.btn_export
                and hasattr(self.btn_export, "winfo_exists")
                and self.btn_export.winfo_exists()
            ):
                try:
                    self.btn_export.config(
                        state=(
                            "normal"
                            if self.results_df is not None and not self.results_df.empty
                            else "disabled"
                        )
                    )
                except tk.TclError:
                    pass

            # Mapping confirmation button
            if (
                hasattr(self, "btn_confirm_mappings")
                and self.btn_confirm_mappings
                and hasattr(self.btn_confirm_mappings, "winfo_exists")
                and self.btn_confirm_mappings.winfo_exists()
            ):
                try:
                    # Check if Salesforce is loaded
                    is_salesforce = str(getattr(self, "src_path", "")).startswith(
                        "Salesforce:"
                    ) or str(getattr(self, "ref_path", "")).startswith("Salesforce:")

                    if self.mappings_confirmed:
                        self.btn_confirm_mappings.config(
                            text="Unlock Mappings",
                            command=self.on_unlock_mappings_clicked,
                            state="normal" if not is_salesforce else "disabled",
                        )
                    else:
                        # For Salesforce, change text to indicate limitation
                        button_text = (
                            "Confirm Mappings (No Analysis)"
                            if is_salesforce
                            else "Confirm & Analyze Mappings"
                        )
                        self.btn_confirm_mappings.config(
                            text=button_text,
                            command=self.on_confirm_mappings_clicked,
                            state="normal",  # Keep enabled but with different text
                        )
                except tk.TclError:
                    pass

            # Update file menu
            if self.file_menu:
                self.file_menu.entryconfig(
                    "Export Results",
                    state=(
                        "normal"
                        if self.results_df is not None and not self.results_df.empty
                        else "disabled"
                    ),
                )

            # Custom rules refresh
            for rule_row in self.rule_rows:
                rule_row.refresh_state()

            # === NEW SPACE OPTIMIZATION CODE STARTS HERE ===

            # Show mapping trees if data loaded (replaces placeholders with actual trees)
            if hasattr(self, "_show_mapping_trees"):
                self._show_mapping_trees()

            # Show/hide active mappings section dynamically based on whether mappings exist
            has_mappings = hasattr(self, "mappings") and len(self.mappings) > 0

            if hasattr(self, "active_frame") and hasattr(self, "active_frame_visible"):
                if has_mappings and not self.active_frame_visible:
                    self.update_idletasks()  # Smooth transition
                    self.active_frame.grid(
                        row=2, column=0, sticky="ew", padx=10, pady=5
                    )
                    self.active_frame_visible = True
                elif not has_mappings and self.active_frame_visible:
                    self.active_frame.grid_forget()
                    self.active_frame_visible = False

            # Update mapping control button states
            if hasattr(self, "btn_clear_mappings"):
                self.btn_clear_mappings.config(
                    state=tk.NORMAL if has_mappings else tk.DISABLED
                )

            if hasattr(self, "btn_remove_mapping"):
                # Remove button state handled by selection
                selection = (
                    self.map_list.curselection() if hasattr(self, "map_list") else []
                )
                self.btn_remove_mapping.config(
                    state=tk.NORMAL if selection else tk.DISABLED
                )

            # === END OF NEW SPACE OPTIMIZATION CODE ===

            # Show source info if loaded
            if hasattr(self, "source_df") and self.source_df is not None:
                if hasattr(self, "lbl_src") and self.lbl_src:
                    self.lbl_src.grid()
                if hasattr(self, "src_id_frame") and self.src_id_frame:
                    self.src_id_frame.grid()
            else:
                if hasattr(self, "lbl_src") and self.lbl_src:
                    self.lbl_src.grid_remove()
                if hasattr(self, "src_id_frame") and self.src_id_frame:
                    self.src_id_frame.grid_remove()

            # Show reference info if loaded (but hide in dedupe mode)
            if (
                hasattr(self, "ref_df")
                and self.ref_df is not None
                and not is_dedupe_mode
            ):
                if hasattr(self, "lbl_ref") and self.lbl_ref:
                    self.lbl_ref.grid()
                if hasattr(self, "ref_id_frame") and self.ref_id_frame:
                    self.ref_id_frame.grid()
            else:
                if hasattr(self, "lbl_ref") and self.lbl_ref:
                    self.lbl_ref.grid_remove()
                if hasattr(self, "ref_id_frame") and self.ref_id_frame:
                    self.ref_id_frame.grid_remove()

            # Update Salesforce status in data picker
            if hasattr(self, "data_source_picker"):
                self.data_source_picker.update_salesforce_status()

            # Update data loader displays
            if hasattr(self, "data_loader"):
                self.data_loader.update_displays()
                self.data_loader.update_mode(self.mode.get())

            if hasattr(self, "_update_loader_banner"):
                self._update_loader_banner()

            # IMPORTANT: Do NOT call _populate_column_trees here
            # It's already called by data-changing methods like _load_data_file

            modify_refresh_ui_for_salesforce(self)
        finally:
            self._refreshing = False

    # gui.py - Add tooltip functionality
    def _setup_tree_tooltips(self, tree):
        """Setup hover tooltips for tree items"""
        self.tooltip = None

        def on_motion(event):
            # Clean up old tooltip
            if self.tooltip:
                self.tooltip.destroy()
                self.tooltip = None

            # Get item under cursor
            item = tree.identify("item", event.x, event.y)
            if (
                item
                and hasattr(self, "column_tooltips")
                and item in self.column_tooltips
            ):
                # Create tooltip
                x = tree.winfo_rootx() + event.x + 20
                y = tree.winfo_rooty() + event.y

                self.tooltip = tk.Toplevel(self)
                self.tooltip.wm_overrideredirect(True)
                self.tooltip.wm_geometry(f"+{x}+{y}")

                label = tk.Label(
                    self.tooltip,
                    text=self.column_tooltips[item],
                    background="#ffffdd",
                    relief="solid",
                    borderwidth=1,
                    font=("TkDefaultFont", 9),
                )
                label.pack()

        def on_leave(event):
            if self.tooltip:
                self.tooltip.destroy()
                self.tooltip = None

        tree.bind("<Motion>", on_motion)
        tree.bind("<Leave>", on_leave)

    def _setup_tree_bindings(self, tree):
        """Setup all tree event bindings"""
        # Existing bindings...

        # Right-click for icon override
        tree.bind("<Button-3>", lambda e: self._show_icon_override_menu(e, tree))

    def _show_icon_override_menu(self, event, tree):
        """Show context menu for icon overrides"""
        # Get clicked item
        item = tree.identify("item", event.x, event.y)
        if not item or item not in self.column_hints:
            return

        # Create menu
        menu = tk.Menu(self, tearoff=0)
        menu.configure(bg="white")

        current_hints = self.column_hints[item]
        current_icons = current_hints["hints"]["icons"]

        # Header
        menu.add_command(
            label=f"Override icons for: {current_hints['column_name']}",
            state="disabled",
            font=("TkDefaultFont", 10, "bold"),
        )
        menu.add_separator()

        # Icon options
        icon_options = [
            ("🔑", "Unique Identifier"),
            ("🧲", "Good for Matching"),
            ("🏢", "Company/Organization"),
            ("👤", "Person Name"),
            ("📧", "Email Address"),
            ("📱", "Phone Number"),
            ("📍", "Address/Location"),
            ("📅", "Date/Time"),
            ("🌐", "Website/URL"),
            ("⭐", "High Quality"),
            ("⚠️", "Quality Warning"),
            ("📊", "Data Column"),
            ("", "Remove All Icons"),
        ]

        for icon, label in icon_options:
            # Show checkmark for current icons
            if icon and icon in current_icons:
                display_label = f"✓ {icon} {label}"
            else:
                display_label = f"  {icon} {label}" if icon else f"  {label}"

            menu.add_command(
                label=display_label,
                command=lambda i=icon, it=item: self._override_icon(tree, it, i),
            )

        # Show menu
        menu.post(event.x_root, event.y_root)

    def _override_icon(self, tree, item, new_icon):
        """Handle icon override with telemetry"""
        if item not in self.column_hints:
            return

        hints_data = self.column_hints[item]
        original_icons = hints_data["hints"]["icons"].copy()

        # Determine new icons
        if new_icon == "":
            # Remove all icons
            new_icons = []
            action = "removed_all"
        elif new_icon in original_icons:
            # Toggle off
            new_icons = [i for i in original_icons if i != new_icon]
            action = "removed"
        else:
            # Add icon (max 2)
            new_icons = original_icons + [new_icon]
            new_icons = new_icons[-2:]  # Keep last 2
            action = "added"

        # Update display
        icon_str = " ".join(new_icons)
        tree.set(item, "Icon", icon_str)

        # Update stored data
        hints_data["hints"]["icons"] = new_icons
        hints_data["overridden"] = True

        # Record telemetry
        if self.telemetry and self.telemetry.enabled:
            self.telemetry.record_icon_override(
                dataset_shape=self.current_dataset_shape,
                original_icons=original_icons,
                user_action=action,
                new_icons=new_icons,
                column_properties=hints_data["stats"],
            )

        # Show feedback
        self.status_bar.config(text=f"Updated icons for '{hints_data['column_name']}'")

    def _add_rule_row(self) -> None:
        """Add a new custom rule row."""
        idx = len(self.rule_rows)
        rule_row = RuleRow(self.rules_container, self, idx, self._remove_rule_row)
        rule_row.grid(row=idx, column=0, sticky="ew", padx=5, pady=2)
        self.rule_rows.append(rule_row)

    def _remove_rule_row(self, rule_row: RuleRow) -> None:
        """Remove a custom rule row."""
        self.rule_rows.remove(rule_row)
        rule_row.destroy()

        # Re-grid remaining rows
        for i, row in enumerate(self.rule_rows):
            row.idx = i
            row.grid(row=i, column=0, sticky="ew", padx=5, pady=2)

    def _update_id_combos(self) -> None:
        """Update ID column dropdowns."""
        # Source ID
        if self.cbo_src_id and self.source_df is not None:
            self.cbo_src_id.config(
                values=list(self.source_df.columns), state="readonly"
            )
        elif self.cbo_src_id:
            self.cbo_src_id.config(values=[], state="disabled")

        # Reference ID
        if self.cbo_ref_id and self.mode.get() == "match" and self.ref_df is not None:
            self.cbo_ref_id.config(values=list(self.ref_df.columns), state="readonly")
        elif self.cbo_ref_id:
            self.cbo_ref_id.config(values=[], state="disabled")

    def _update_block_combos(self) -> None:
        """Update blocking column dropdowns."""
        # Source blocking
        if self.cbo_src_block and self.source_df is not None:
            self.cbo_src_block.config(
                values=[""] + list(self.source_df.columns), state="readonly"
            )
        elif self.cbo_src_block:
            self.cbo_src_block.config(values=[], state="disabled")

        # Reference blocking
        if (
            self.cbo_ref_block
            and self.mode.get() == "match"
            and self.ref_df is not None
        ):
            self.cbo_ref_block.config(
                values=[""] + list(self.ref_df.columns), state="readonly"
            )
        elif self.cbo_ref_block:
            self.cbo_ref_block.config(values=[], state="disabled")

    def on_algorithm_change(self, selected: str) -> None:
        """Handle algorithm selection change."""
        log.info(f"Algorithm changed to: {selected}")
        self.update_algorithm_tooltip()

    def update_algorithm_tooltip(self) -> None:
        """Update the algorithm selection tooltip."""
        if hasattr(self, "algorithm_menu_tooltip"):
            # Use the ALGORITHMS dictionary defined at the top of the file
            algo_info = ALGORITHMS.get(self.algorithm_var.get(), {})
            self.algorithm_menu_tooltip.text = algo_info.get(
                "tooltip", "Select matching algorithm"
            )

    def update_processing_mode_auto(self) -> None:
        """Auto-detect processing mode based on data size."""
        if self.source_df is None:
            return

        ref_df = self.ref_df if self.mode.get() == "match" else None
        auto_mode = decide_processing_mode(self.source_df, ref_df)

        self.current_processing_mode.set(auto_mode)

        if self.lbl_mode:
            self.lbl_mode.config(text=f"Mode -> {auto_mode.capitalize()}")

    def on_auto_blocking_clicked(self) -> None:
        """Handle auto-blocking button click."""
        if self.source_df is None or self.source_df.empty:
            messagebox.showwarning("No Data", "Please load source data first")
            return

        if self.mode.get() == "match" and not self.ref_df:
            messagebox.showwarning("No Data", "Please load reference data first")
            return

        # This is a simplified version - in production, you'd run
        # a more sophisticated analysis
        log.info("Auto-selecting blocking columns")

        # For now, just trigger the locking analysis
        if self.mappings_confirmed:
            self.initiate_locking_analysis()
        else:
            messagebox.showinfo(
                "Confirm Mappings First",
                "Please confirm field mappings before analyzing blocking strategy",
            )

    def auto_select_blocking(self) -> None:
        """
        Pick a sensible blocking column/key-length based on the current
        data and field mappings, then refresh the UI.
        Public helper method used by session-loader & menu.
        """
        # This delegates to the existing private helper if it exists
        if hasattr(self, "_auto_select_blocking"):
            self._auto_select_blocking()
        else:
            # Simple fallback implementation
            if self.source_df is not None and not self.source_df.empty:
                # Try to find a good blocking column
                for col in self.source_df.columns:
                    col_lower = col.lower()
                    if any(kw in col_lower for kw in ["id", "key", "code"]):
                        self.source_block_col_var.set(col)
                        if self.mode.get() == "match" and self.ref_df is not None:
                            # Also try to find matching column in reference
                            for ref_col in self.ref_df.columns:
                                if ref_col.lower() == col_lower:
                                    self.ref_block_col_var.set(ref_col)
                                    break
                            else:
                                # If no exact match, use the same name
                                self.ref_block_col_var.set(col)
                        break
                else:
                    # No ID-like column found, use first column as fallback
                    self.source_block_col_var.set(self.source_df.columns[0])
                    if self.mode.get() == "match" and self.ref_df is not None:
                        self.ref_block_col_var.set(self.ref_df.columns[0])

                # Set a reasonable block key length
                self.block_key_length_var.set(3)

                log.info(
                    f"Auto-selected blocking column: {self.source_block_col_var.get()}"
                )

    def show_blocking_dialog(self) -> None:
        """Show current blocking strategy in a dialog."""
        if not self.blocking_config:
            messagebox.showinfo(
                "No Blocking Strategy",
                "No blocking strategy has been determined yet.\n\n"
                "Confirm mappings to analyze the optimal strategy.",
            )
            return

        # Create dialog
        dialog = tk.Toplevel(self)
        dialog.title("Blocking Strategy")
        dialog.geometry("600x400")

        # Create text widget
        text = tk.Text(dialog, wrap=tk.WORD, padx=10, pady=10)
        text.pack(fill=tk.BOTH, expand=True)

        # Add content
        content = f"""Current Blocking Strategy
========================

Source Column: {self.blocking_config.original_src_col or 'None'}
Reference Column: {self.blocking_config.original_ref_col or 'None'}
Block Key Length: {self.blocking_config.block_key_length or 'N/A'}

B2C Domain Filtering: {'Enabled' if self.b2c_enabled_var.get() else 'Disabled'}

Performance Metrics:
- Entropy: {self.blocking_config.entropy:.2f}
- Estimated Reduction: {self.blocking_config.estimated_reduction_pct:.1f}%
- Quality Score: {self.blocking_config.quality_score:.2f}

This strategy will group records by the first {self.blocking_config.block_key_length}
characters of the selected columns, reducing the number of comparisons needed.
"""

        text.insert("1.0", content)
        text.config(state="disabled")

        # Close button
        ttk.Button(dialog, text="Close", command=dialog.destroy).pack(pady=10)

        dialog.transient(self)
        dialog.grab_set()

    # Keep this robust version of the function
    # In gui.py, replace your existing show_blocking_analysis method with this:

    def _get_algorithm_details(self) -> Dict:
        """Get detailed algorithm usage information."""
        details = {"enabled": self.enable_multi_algo.get(), "mappings": []}
        for mapping in self.mappings:
            details["mappings"].append(
                {
                    "source": mapping["source"],
                    "ref": mapping.get("ref", mapping["source"]),
                    "algorithm": mapping.get(
                        "preferred_algo", self.algorithm_var.get()
                    ),
                    "confidence": mapping.get("algo_confidence", 0.0),
                    "weight": mapping.get("weight", 1.0),
                }
            )
        return details

    def show_blocking_analysis(self):
        """Show the last blocking analysis window with algorithm details if enabled."""
        if (
            not hasattr(self, "auto_determined_locking_strategy")
            or not self.auto_determined_locking_strategy
        ):
            messagebox.showinfo(
                "No Analysis",
                "No blocking analysis has been run yet. Please confirm mappings to generate one.",
            )
            return

        try:
            # Create the enhanced transparency window instance with all required arguments
            transparency_window = BlockingTransparencyWindow(
                parent=self,
                blocking_result=self.auto_determined_locking_strategy,
                source_df=self.source_df,
                ref_df=self.ref_df if self.mode.get() == "match" else None,
                mode=self.mode.get(),
            )

            # NEW: Add algorithm details to the window instance if multi-algo is enabled
            if self.enable_multi_algo.get():
                # This line was causing a NameError before assigning the window to a variable
                transparency_window.algorithm_details = self._get_algorithm_details()

        except Exception as e:
            log.error(f"Failed to show blocking analysis window: {e}")
            messagebox.showerror("Error", f"Could not display analysis window: {e}")

    def _show_algorithm_details(self):
        """Show dialog with algorithm usage details."""
        if not self.mappings:
            messagebox.showinfo("No Mappings", "No mappings to analyze.")
            return

        # Create dialog
        dialog = tk.Toplevel(self)
        dialog.title("Algorithm Usage Details")
        dialog.geometry("600x400")

        # Create text widget with scrollbar
        text_frame = ttk.Frame(dialog)
        text_frame.pack(fill="both", expand=True, padx=10, pady=10)

        text = tk.Text(text_frame, wrap="word", width=70, height=20)
        scrollbar = ttk.Scrollbar(text_frame, orient="vertical", command=text.yview)
        text.configure(yscrollcommand=scrollbar.set)

        text.pack(side="left", fill="both", expand=True)
        scrollbar.pack(side="right", fill="y")

        # Analyze algorithm usage
        algo_counts = {}
        for mapping in self.mappings:
            algo = mapping.get("preferred_algo", self.algorithm_var.get())
            algo_counts[algo] = algo_counts.get(algo, 0) + 1

        # Build report
        report = "Algorithm Usage Report\n"
        report += "=" * 50 + "\n\n"

        report += "Summary:\n"
        for algo, count in sorted(
            algo_counts.items(), key=lambda x: x[1], reverse=True
        ):
            report += f"  {algo}: {count} columns\n"

        report += "\n\nDetailed Mappings:\n"
        report += "-" * 50 + "\n"

        for mapping in self.mappings:
            algo = mapping.get("preferred_algo", self.algorithm_var.get())
            confidence = mapping.get("algo_confidence", 0.0)
            weight = mapping.get("weight", 1.0)

            report += f"\n{mapping['source']}"
            if self.mode.get() == "match":
                report += f" -> {mapping['ref']}"
            report += f"\n  Algorithm: {algo}"
            if mapping.get("preferred_algo"):
                report += f" (confidence: {confidence:.0%})"
            report += f"\n  Weight: {weight:.1f}\n"

        # Insert report
        text.insert("1.0", report)
        text.configure(state="disabled")

        # Close button
        ttk.Button(dialog, text="Close", command=dialog.destroy).pack(pady=(0, 10))

    def show_salesforce_settings(self):
        """Show Salesforce configuration dialog."""
        try:
            # Try to show the compact dialog
            from .gui_compact_sf import CompactSalesforceDialog

            # Check if connected
            if not hasattr(self, "sf_integration") or not self.sf_integration:
                # Show connection dialog first
                if hasattr(self, "_connect_to_salesforce"):
                    self._connect_to_salesforce()
                    return

            # Show the Salesforce data selection dialog
            dialog = CompactSalesforceDialog(self, self.sf_integration, is_source=True)

        except ImportError:
            # Fallback if compact dialog not available
            messagebox.showinfo(
                "Salesforce Integration",
                "Salesforce integration is not fully configured.\n"
                "Please ensure all Salesforce modules are installed.",
            )

    def show_hubspot_settings(self):
        """Show HubSpot configuration dialog."""
        messagebox.showinfo(
            "Coming Soon", "HubSpot integration will be available in a future release."
        )

    def show_preprocessing_settings(self):
        """Show preprocessing configuration dialog."""
        dialog = tk.Toplevel(self)
        dialog.title("Preprocessing Settings")
        dialog.geometry("700x500")
        dialog.transient(self)
        dialog.grab_set()

        # Create notebook for preprocessing options
        notebook = ttk.Notebook(dialog, padding="10")
        notebook.pack(fill=tk.BOTH, expand=True, padx=10, pady=10)

        # Normalization rules tab
        norm_frame = ttk.Frame(notebook)
        notebook.add(norm_frame, text="Normalization Rules")

        # Create checkboxes for each normalization rule
        norm_label = ttk.Label(
            norm_frame, text="Select text normalization rules to apply:"
        )
        norm_label.pack(anchor=tk.W, padx=10, pady=(10, 5))

        rules_frame = ttk.Frame(norm_frame)
        rules_frame.pack(fill=tk.BOTH, expand=True, padx=20, pady=10)

        for i, (name, var) in enumerate(sorted(self.norm_rule_vars.items())):
            cb = ttk.Checkbutton(
                rules_frame, text=name.replace("_", " ").title(), variable=var
            )
            cb.grid(row=i // 2, column=i % 2, sticky="w", padx=10, pady=2)

        # B2C Domain Filtering tab
        b2c_frame = ttk.Frame(notebook)
        notebook.add(b2c_frame, text="Domain Filtering")

        # Add B2C filter controls
        from .b2c_gui_integration import add_b2c_filter_controls

        add_b2c_filter_controls(b2c_frame, self)

        # Button frame
        button_frame = ttk.Frame(dialog)
        button_frame.pack(fill=tk.X, pady=(0, 10), padx=10)

        def apply_settings():
            # Settings are already bound to variables, just close
            dialog.destroy()
            self.refresh_ui()

        ttk.Button(button_frame, text="Apply", command=apply_settings).pack(
            side=tk.RIGHT, padx=5
        )
        ttk.Button(button_frame, text="Cancel", command=dialog.destroy).pack(
            side=tk.RIGHT
        )

        # Center the dialog
        dialog.update_idletasks()
        x = (dialog.winfo_screenwidth() // 2) - (dialog.winfo_width() // 2)
        y = (dialog.winfo_screenheight() // 2) - (dialog.winfo_height() // 2)
        dialog.geometry(f"+{x}+{y}")

    def show_custom_rules_settings(self):
        """Show custom match rules configuration dialog."""
        dialog = tk.Toplevel(self)
        dialog.title("Custom Match Rules")
        dialog.geometry("800x600")
        dialog.transient(self)
        dialog.grab_set()

        # Main frame
        main_frame = ttk.Frame(dialog, padding="10")
        main_frame.pack(fill=tk.BOTH, expand=True)
        main_frame.columnconfigure(0, weight=1)
        main_frame.rowconfigure(1, weight=1)

        # Instructions
        instructions = ttk.Label(
            main_frame,
            text="Define custom matching rules for specific fields. Rules are applied in order.",
            wraplength=750,
        )
        instructions.grid(row=0, column=0, sticky="w", pady=(0, 10))

        # Rules listbox with scrollbar
        list_frame = ttk.Frame(main_frame)
        list_frame.grid(row=1, column=0, sticky="nsew")
        list_frame.columnconfigure(0, weight=1)
        list_frame.rowconfigure(0, weight=1)

        # Create listbox for rules
        rules_listbox = tk.Listbox(list_frame, height=15)
        rules_listbox.grid(row=0, column=0, sticky="nsew")

        scrollbar = ttk.Scrollbar(
            list_frame, orient="vertical", command=rules_listbox.yview
        )
        scrollbar.grid(row=0, column=1, sticky="ns")
        rules_listbox.configure(yscrollcommand=scrollbar.set)

        # Load existing rules if any
        # TODO: Populate from self.custom_rules or similar

        # Control buttons
        control_frame = ttk.Frame(main_frame)
        control_frame.grid(row=2, column=0, sticky="ew", pady=(10, 0))

        ttk.Button(
            control_frame,
            text="Add Rule...",
            command=lambda: self._add_custom_rule(rules_listbox),
        ).pack(side=tk.LEFT, padx=5)
        ttk.Button(
            control_frame,
            text="Edit Rule...",
            command=lambda: self._edit_custom_rule(rules_listbox),
        ).pack(side=tk.LEFT, padx=5)
        ttk.Button(
            control_frame,
            text="Delete Rule",
            command=lambda: self._delete_custom_rule(rules_listbox),
        ).pack(side=tk.LEFT, padx=5)

        ttk.Button(
            control_frame,
            text="Move Up",
            command=lambda: self._move_rule(rules_listbox, -1),
        ).pack(side=tk.LEFT, padx=20)
        ttk.Button(
            control_frame,
            text="Move Down",
            command=lambda: self._move_rule(rules_listbox, 1),
        ).pack(side=tk.LEFT, padx=5)

        # Dialog buttons
        button_frame = ttk.Frame(dialog)
        button_frame.pack(fill=tk.X, pady=(10, 0))

        ttk.Button(button_frame, text="OK", command=dialog.destroy).pack(
            side=tk.RIGHT, padx=5
        )
        ttk.Button(button_frame, text="Cancel", command=dialog.destroy).pack(
            side=tk.RIGHT
        )

        # Center the dialog
        dialog.update_idletasks()
        x = (dialog.winfo_screenwidth() // 2) - (dialog.winfo_width() // 2)
        y = (dialog.winfo_screenheight() // 2) - (dialog.winfo_height() // 2)
        dialog.geometry(f"+{x}+{y}")

    def show_about(self):
        """Show about dialog."""
        about_text = f"""FoundryOps Fuzzy Matcher
Version {CONFIG['APP_VERSION']}

Advanced entity resolution with:
- Modern preprocessing engine
- B2C domain filtering
- Intelligent blocking strategies
- Salesforce integration

© 2025 FoundryOps
"""
        messagebox.showinfo("About", about_text)

    # Helper methods for custom rules dialog
    def _add_custom_rule(self, listbox):
        """Add a new custom rule."""
        # TODO: Open a dialog to create a new rule
        messagebox.showinfo("Add Rule", "Rule creation dialog coming soon")

    def _edit_custom_rule(self, listbox):
        """Edit selected custom rule."""
        selection = listbox.curselection()
        if not selection:
            messagebox.showwarning("No Selection", "Please select a rule to edit")
            return
        # TODO: Open edit dialog
        messagebox.showinfo("Edit Rule", "Rule editing dialog coming soon")

    def _delete_custom_rule(self, listbox):
        """Delete selected custom rule."""
        selection = listbox.curselection()
        if not selection:
            messagebox.showwarning("No Selection", "Please select a rule to delete")
            return
        if messagebox.askyesno("Confirm Delete", "Delete the selected rule?"):
            listbox.delete(selection[0])

    def _move_rule(self, listbox, direction):
        """Move selected rule up or down."""
        selection = listbox.curselection()
        if not selection:
            return

        index = selection[0]
        if direction == -1 and index == 0:
            return  # Can't move first item up
        if direction == 1 and index == listbox.size() - 1:
            return  # Can't move last item down

        # Get the item text
        item = listbox.get(index)
        # Delete from current position
        listbox.delete(index)
        # Insert at new position
        listbox.insert(index + direction, item)
        # Select the moved item
        listbox.selection_set(index + direction)

    # ─────────────────────────────────────────────────────────────────────────
    # Mapping Template Methods
    # ─────────────────────────────────────────────────────────────────────────

    def save_mapping_template(self) -> None:
        """Save current mappings as a template."""
        if not self.mappings:
            messagebox.showwarning("No Mappings", "No mappings to save")
            return

        file_path = filedialog.asksaveasfilename(
            title="Save Mapping Template",
            defaultextension=".yaml",
            filetypes=[("YAML files", "*.yaml"), ("All files", "*.*")],
        )

        if not file_path:
            return

        try:
            template = {
                "version": "1.0",
                "mappings": self.mappings,
                "mode": self.mode.get(),
                "algorithm": self.algorithm_var.get(),
                "threshold": self.threshold.get(),
            }

            with open(file_path, "w") as f:
                yaml.dump(template, f, default_flow_style=False)

            log.info(f"Saved mapping template to {file_path}")
            messagebox.showinfo("Success", "Mapping template saved")

        except Exception as e:
            log.error(f"Error saving mapping template: {e}", exc_info=True)
            messagebox.showerror("Save Error", f"Failed to save template: {e}")

    def load_mapping_template(self) -> None:
        """Load mappings from a template file."""
        file_path = filedialog.askopenfilename(
            title="Load Mapping Template",
            filetypes=[("YAML files", "*.yaml"), ("All files", "*.*")],
        )

        if not file_path:
            return

        try:
            with open(file_path, "r") as f:
                template = yaml.safe_load(f)

            # Validate template
            if not isinstance(template, dict) or "mappings" not in template:
                raise ValueError("Invalid template format")

            # Apply mappings
            self.mappings = template["mappings"]
            self._refresh_mappings_list()

            # Apply other settings if present
            if "mode" in template:
                self.mode.set(template["mode"])
            if "algorithm" in template:
                self.algorithm_var.set(template["algorithm"])
            if "threshold" in template:
                self.threshold.set(template["threshold"])

            self.refresh_ui()
            log.info(f"Loaded mapping template from {file_path}")

        except Exception as e:
            log.error(f"Error loading mapping template: {e}", exc_info=True)
            messagebox.showerror("Load Error", f"Failed to load template: {e}")

    def _check_auto_load_mapping_template(self, source_path: str) -> None:
        """Check if there's a mapping template for this source file."""
        source_name = Path(source_path).stem
        template_path = MAPPING_DIR / f"{source_name}.yaml"

        if template_path.exists():
            if messagebox.askyesno(
                "Load Template?",
                f"Found mapping template for this file.\nLoad mappings from {template_path.name}?",
            ):
                try:
                    with open(template_path, "r") as f:
                        template = yaml.safe_load(f)
                    self.mappings = template.get("mappings", [])
                    self._refresh_mappings_list()
                    log.info(f"Auto-loaded mapping template from {template_path}")
                except Exception as e:
                    log.error(f"Error auto-loading template: {e}", exc_info=True)

    # ─────────────────────────────────────────────────────────────────────────
    # Utility Methods
    # ─────────────────────────────────────────────────────────────────────────

    def _format_number(self, value: float, decimals: int = 0) -> str:
        """Format number with thousands separators."""
        if decimals == 0:
            return f"{int(value):,}"
        else:
            return f"{value:,.{decimals}f}"

    def _format_rate(self, rate: float, unit: str) -> str:
        """Format rate with appropriate scale."""
        if rate >= 1_000_000:
            return f"{self._format_number(rate / 1_000_000, 1)}M {unit}"
        elif rate >= 1_000:
            return f"{self._format_number(rate / 1_000, 0)}K {unit}"
        else:
            return f"{self._format_number(rate, 0)} {unit}"

    def _format_time_duration(self, seconds: float) -> str:
        """Format time duration in seconds to human-readable format."""
        if seconds < 0:
            return "Calculating..."

        if seconds < 60:
            return f"{int(seconds)}s"
        elif seconds < 3600:
            minutes = int(seconds / 60)
            secs = int(seconds % 60)
            return f"{minutes}m {secs}s"
        else:
            hours = int(seconds / 3600)
            minutes = int((seconds % 3600) / 60)
            return f"{hours}h {minutes}m"

    def _after(self, ms: int, callback):
        """Tracked version of after() that we can cancel all at once"""
        token = None  # Initialize first

        def wrapped_callback():
            """Defensive wrapper with error handling and cleanup"""
            try:
                return callback()
            except Exception:
                log.exception("Unhandled exception in after() callback")
            finally:
                # Always clean up the token
                if token:
                    self._after_tokens.discard(token)

        token = self.after(ms, wrapped_callback)
        self._after_tokens.add(token)
        return token

    def _cancel_all_after(self):
        """Cancel all tracked after() timers"""
        for token in list(self._after_tokens):
            try:
                self.after_cancel(token)
            except tk.TclError:
                pass  # Already fired or cancelled
        self._after_tokens.clear()

    def reset_run_state(self, *, preserve_locking: bool = True):
        """Complete state reset between runs - call at START of each operation

        Args:
            preserve_locking: If True, preserves auto_determined_locking_strategy
        """
        log.info(
            f"RESET: Pre-state: queue={self.progress_queue.qsize() if hasattr(self.progress_queue, 'qsize') else 'unknown'}, "
            f"results={len(self.results_df) if self.results_df is not None else 0}, "
            f"timers={len(getattr(self, '_after_tokens', []))}, "
            f"preserve_locking={preserve_locking}"
        )

        # 1. Cancel ALL timers first
        self._cancel_all_after()

        # 2. Stop any polling
        if hasattr(self, "_poll_id") and self._poll_id:
            self.after_cancel(self._poll_id)
            self._poll_id = None

        # 3. Clear operation tracking
        self.current_operation_id = None
        self._is_processing_cancelled = False
        self._is_analyzing = False
        self.processing = False

        # 4. Clear progress state
        self.cumulative_items = 0.0
        self.cumulative_comparisons = 0.0
        self.smoothed_eta_s = None
        self.smoothed_cps = None
        self._final_shown = False
        self._last_total_items = 0
        self._last_cumulative_items = 0
        self.current_phase = None
        self._final_phase = False
        self.last_event_received_ts = 0.0

        # 5. Clear completion tracking (per-run only!)
        if hasattr(self, "_completed_operations"):
            self._completed_operations.clear()

        # 6. Clear results and clean up previous spill files
        self.results_df = None

        # Clean up previous results before starting new run
        if hasattr(self, "results_path") and self.results_path:
            try:
                # Get file size before deletion to subtract from total
                file_path = Path(self.results_path)
                if file_path.exists():
                    file_size = file_path.stat().st_size
                    file_path.unlink()
                    self.total_disk_usage = max(0, self.total_disk_usage - file_size)
                    log.info(
                        f"Cleaned up previous results: {self.results_path} ({file_size / 1_000_000:.1f}MB)"
                    )
                    log.info(
                        f"Total temp disk usage: {self.total_disk_usage / 1_000_000:.1f}MB"
                    )
            except Exception as e:
                log.warning(f"Could not delete previous results: {e}")
            finally:
                self.results_path = None
                self.results_row_count = 0

        # Only clear strategy when explicitly asked
        if not preserve_locking:
            if hasattr(self, "auto_determined_locking_strategy"):
                self.auto_determined_locking_strategy = None

        # 7. Drain queue
        while not self.progress_queue.empty():
            try:
                self.progress_queue.get_nowait()
            except Exception:
                break

        # 8. Reset UI
        self._set_progress_determinate(value=0)
        if hasattr(self, "lbl_progress") and self.lbl_progress:
            self.lbl_progress.config(text="0%")
        if hasattr(self, "lbl_eta") and self.lbl_eta:
            self.lbl_eta.config(text="")
        if hasattr(self, "lbl_cps") and self.lbl_cps:
            self.lbl_cps.config(text="")
        if hasattr(self, "lbl_status") and self.lbl_status:
            self.lbl_status.config(text="Ready")

        # 9. Close any open results popups
        if hasattr(self, "results_popup") and self.results_popup:
            try:
                self.results_popup.destroy()
            except Exception:
                pass
            self.results_popup = None

        # 10. Force garbage collection
        gc.collect()

        log.info("RESET: Complete")

    def _drain_progress_queue(self) -> None:
        """Drain any stale events from the progress queue."""
        q = getattr(self, "progress_queue", None)
        if not q:
            return

        from queue import Empty

        drained = 0
        while True:
            try:
                q.get_nowait()
                drained += 1
            except Empty:
                break

        if drained > 0:
            log.debug(f"Drained {drained} stale events from progress queue")

    def _ensure_pool_active(self) -> None:
        """Ensure thread pool is active for background tasks."""
        if self.pool is None or self.pool._shutdown:
            self.pool = cf.ThreadPoolExecutor(
                max_workers=CONFIG["MAX_WORKERS"],
                thread_name_prefix="FuzzyMatcherWorker",
            )
            log.info(f"UI thread pool created (max_workers={CONFIG['MAX_WORKERS']})")

    def _ensure_engine_process_pool_active(self) -> None:
        """
        Make sure we have a live process pool; recreate it if it was shut down.
        Compatible with Python 3.10-3.12+.
        """
        if self.engine_process_pool is None:  # not created yet
            self.engine_process_pool = cf.ProcessPoolExecutor(
                max_workers=CONFIG["MAX_ENGINE_WORKERS"]
            )
            log.info(
                f"Created engine process pool with {CONFIG['MAX_ENGINE_WORKERS']} workers"
            )
            return

        # Check if the pool is dead in a version-agnostic way
        is_dead = False
        if hasattr(self.engine_process_pool, "_shutdown"):
            is_dead = self.engine_process_pool._shutdown
        elif (
            getattr(self.engine_process_pool, "_state", None) == 2
        ):  # _STATE_FINISHED == 2
            is_dead = True

        if is_dead:
            log.info("Engine process pool was shut down. Recreating.")
            self.engine_process_pool = cf.ProcessPoolExecutor(
                max_workers=CONFIG["MAX_ENGINE_WORKERS"]
            )
            log.debug(
                f"Re-created engine process pool with {CONFIG['MAX_ENGINE_WORKERS']} workers"
            )

    def _send_webhook_notification(
        self, op_id: str, error: Optional[str] = None
    ) -> None:
        """Send webhook notification about job completion."""
        webhook_url = self.webhook_var.get()
        if not webhook_url:
            return

        try:
            payload = {
                "job_id": op_id,
                "status": "failed" if error else "completed",
                "mode": self.mode.get(),
                "timestamp": time.time(),
                "results_count": (
                    len(self.results_df) if self.results_df is not None else 0
                ),
            }

            if error:
                payload["error"] = error

            response = requests.post(
                webhook_url,
                json=payload,
                timeout=10,
                headers={"Content-Type": "application/json"},
            )
            response.raise_for_status()
            log.info(f"Webhook notification sent to {webhook_url}")

        except Exception as e:
            log.error(f"Failed to send webhook notification: {e}", exc_info=True)

    def _save_rules_to_yaml(self) -> Optional[str]:
        """Save custom rules to a temporary YAML file."""
        if not self.rule_rows:
            return None

        try:
            rules = [row.to_dict() for row in self.rule_rows]

            # Create temp file
            fd, path = tempfile.mkstemp(suffix=".yaml", prefix="custom_rules_")

            with os.fdopen(fd, "w") as f:
                yaml.dump({"rules": rules}, f)

            return path

        except Exception as e:
            log.error(f"Error saving rules: {e}")
            return None

    def _load_custom_semantic_types(self) -> None:
        """Load custom semantic types from configuration."""
        try:
            types_file = CUSTOM_SEMANTIC_TYPES_DIR / "custom_types.yaml"
            if types_file.exists():
                with open(types_file, "r") as f:
                    self.custom_semantic_types = yaml.safe_load(f) or []
            else:
                self.custom_semantic_types = []
        except Exception as e:
            log.error(f"Error loading custom semantic types: {e}", exc_info=True)
            self.custom_semantic_types = []

    def _load_last_session_deferred(self) -> None:
        """Load last session with deferred execution."""
        try:
            self.load_last_session()
        except Exception as e:
            log.error(f"Failed to load last session: {e}", exc_info=True)

    def _check_for_updates(self) -> None:
        """Check for application updates in background."""
        try:
            # Placeholder for update check logic
            log.debug("Update check completed")
        except Exception as e:
            log.error(f"Update check failed: {e}", exc_info=True)

    def _on_close(self) -> None:
        """Handle application close."""
        if self.processing:
            if not messagebox.askyesno(
                "Processing Active", "Processing is currently active. Exit anyway?"
            ):
                return

            self._is_processing_cancelled = True

        # Save session
        try:
            self.save_session()
        except Exception as e:
            log.error(f"Error saving session on close: {e}", exc_info=True)

        # Shutdown thread pools gracefully
        try:
            # Cancel all pending after() callbacks
            if hasattr(self, "_after_tokens"):
                for token in self._after_tokens:
                    try:
                        self.after_cancel(token)
                    except Exception:
                        pass
                self._after_tokens.clear()

            # Stop progress polling
            if hasattr(self, "_poll_id") and self._poll_id:
                try:
                    self.after_cancel(self._poll_id)
                except Exception:
                    pass
                self._poll_id = None

            if hasattr(self, "pool") and self.pool:
                log.info("Shutting down UI thread pool...")
                try:
                    # Try graceful shutdown with timeout
                    self.pool.shutdown(wait=True, timeout=3)
                except (TimeoutError, Exception) as e:
                    log.warning(
                        f"UI pool graceful shutdown failed: {e}, forcing shutdown"
                    )
                    # Force shutdown if graceful fails
                    self.pool.shutdown(wait=False)
                finally:
                    self.pool = None

            if hasattr(self, "engine_process_pool") and self.engine_process_pool:
                log.info("Shutting down engine process pool...")
                try:
                    # Try graceful shutdown with timeout
                    self.engine_process_pool.shutdown(wait=True, timeout=3)
                except (TimeoutError, Exception) as e:
                    log.warning(
                        f"Engine pool graceful shutdown failed: {e}, forcing shutdown"
                    )
                    # Force shutdown if graceful fails
                    self.engine_process_pool.shutdown(wait=False)
                finally:
                    self.engine_process_pool = None

            # Clear completed operations to free memory
            if hasattr(self, "_completed_operations"):
                self._completed_operations.clear()

            # Final garbage collection before exit
            gc.collect()
        except Exception as e:
            log.error(f"Error shutting down pools: {e}", exc_info=True)

        # Already shutdown in the try block above, remove duplicate

        # Close window
        self.quit()


# ─────────────────────────────────────────────────────────────────────────────
# Utility Functions
# ─────────────────────────────────────────────────────────────────────────────


def _log_thread_exceptions(func):
    """Decorator to log exceptions from thread pool functions."""

    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except Exception as e:
            log.exception(f"Exception in thread pool function {func.__name__}: {e}")
            raise

    return wrapper


# ─────────────────────────────────────────────────────────────────────────────
# Entry Point
# ─────────────────────────────────────────────────────────────────────────────


def main():
    """Main entry point for the application."""
    # Force spawn method for multiprocessing before any pools are created
    import multiprocessing as mp

    try:
        if mp.get_start_method(allow_none=True) != "spawn":
            mp.set_start_method("spawn", force=True)
            log.info("Set multiprocessing start method to 'spawn'")
    except RuntimeError:
        pass  # Already set, that's fine

    # Configure logging
    configure_logging(CONFIG["LOG_FILE"], level=logging.INFO)

    log.info("=" * 60)
    log.info(f"Starting FoundryOps Fuzzy Matcher GUI v{CONFIG['APP_VERSION']} (Async)")
    log.info("=" * 60)

    # Create and run application
    app = FuzzyMatcherApp()

    try:
        app.mainloop()
    except KeyboardInterrupt:
        log.info("Application interrupted by user")
    except Exception as e:
        log.critical(f"Fatal error in main loop: {e}", exc_info=True)
        messagebox.showerror(
            "Fatal Error",
            f"A critical error occurred:\n\n{e}\n\nPlease check the log file for details.",
        )
    finally:
        log.info("Application shutting down")
        logging.shutdown()  # Flush all logs before exit


if __name__ == "__main__":
    import multiprocessing as mp

    mp.freeze_support()  # Required for Windows packaging
    main()
