import math
import os
import re
import time
from collections import defaultdict
from datetime import datetime
from typing import Any, Callable, Dict, List, Optional, Tuple

from clickzetta.zettapark.session import Session
from loguru import logger

from semantic_model_generator.clickzetta_utils.clickzetta_connector import (
    AUTOGEN_TOKEN,
    DIMENSION_DATATYPES,
    MEASURE_DATATYPES,
    OBJECT_DATATYPES,
    TIME_MEASURE_DATATYPES,
    get_table_representation,
    get_valid_schemas_tables_columns_df,
)
from semantic_model_generator.clickzetta_utils.utils import (
    create_fqn_table,
    join_quoted_identifiers,
    normalize_identifier,
    quote_identifier,
)
from semantic_model_generator.data_processing import data_types, proto_utils
from semantic_model_generator.llm import (
    DashscopeClient,
    DashscopeSettings,
    enrich_semantic_model,
    get_dashscope_settings,
)
from semantic_model_generator.llm.progress_tracker import (
    EnrichmentProgressTracker,
    EnrichmentStage,
)
from semantic_model_generator.protos import semantic_model_pb2
from semantic_model_generator.validate.context_length import validate_context_length
from semantic_model_generator.validate.keywords import CZ_RESERVED_WORDS

_PLACEHOLDER_COMMENT = "  "
_FILL_OUT_TOKEN = " # <FILL-OUT>"
# TODO add _AUTO_GEN_TOKEN to the end of the auto generated descriptions.
_AUTOGEN_COMMENT_TOKEN = (
    " # <AUTO-GENERATED DESCRIPTION, PLEASE MODIFY AND REMOVE THE __ AT THE END>"
)
_DEFAULT_N_SAMPLE_VALUES_PER_COL = 10
_AUTOGEN_COMMENT_WARNING = f"# NOTE: This file was auto-generated by the semantic model generator. Please fill out placeholders marked with {_FILL_OUT_TOKEN} (or remove if not relevant) and verify autogenerated comments.\n"
_GENERIC_IDENTIFIER_TOKENS = {
    "ID",
    "NAME",
    "CODE",
    "KEY",
    "VALUE",
    "NUMBER",
}


def _singularize(token: str) -> str:
    if token.endswith("IES") and len(token) > 3:
        return token[:-3] + "Y"
    if token.endswith(("SES", "XES", "ZES", "CHES", "SHES")) and len(token) > 4:
        return token[:-2]
    if token.endswith("S") and len(token) > 3:
        return token[:-1]
    return token


def _clean_column_type(raw: Any) -> str:
    if raw is None:
        return ""
    text = str(raw).strip()
    if not text:
        return ""
    text = re.sub(r"\s+", " ", text.upper())
    for suffix in (" NOT NULL", " NULL"):
        if text.endswith(suffix):
            text = text[: -len(suffix)].strip()
    return text


def _base_type_from_type(column_type: str) -> str:
    cleaned = _clean_column_type(column_type)
    token = cleaned.split(" ")[0] if cleaned else ""
    return token.split("(")[0]


def _identifier_tokens(
    name: str, prefixes_to_drop: Optional[set[str]] = None
) -> List[str]:
    name = name.replace("-", "_")
    raw_tokens = re.split(r"[^0-9A-Za-z]+", name)
    tokens: List[str] = []
    for token in raw_tokens:
        if not token:
            continue
        split = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", token).split()
        for part in split:
            tokens.append(part.upper())
    tokens = [token for token in tokens if token]
    if prefixes_to_drop and len(tokens) >= 2 and tokens[0] in prefixes_to_drop:
        tokens = tokens[1:]
    return tokens


def _is_generic_identifier(name: str) -> bool:
    tokens = [token for token in _identifier_tokens(name) if token]
    if not tokens:
        return True
    normalized_tokens = {token.upper() for token in tokens}
    return normalized_tokens.issubset(_GENERIC_IDENTIFIER_TOKENS)


def _sanitize_identifier_name(
    name: str, prefixes_to_drop: Optional[set[str]] = None
) -> str:
    if not name:
        return ""

    cleaned = name.replace("-", "_")
    cleaned = re.sub(r"\s+", "_", cleaned)
    parts = [part for part in cleaned.split("_") if part]
    if not parts:
        return ""

    if len(parts) >= 2 and len(parts[0]) == 1:
        parts = parts[1:]

    if prefixes_to_drop:
        drop_tokens = {token.upper() for token in prefixes_to_drop}
        while parts and parts[0].upper() in drop_tokens:
            parts = parts[1:]

    if not parts:
        return ""

    rebuilt = "_".join(parts)
    tokens = _identifier_tokens(rebuilt)
    if not tokens:
        return ""

    if len(parts) == 1:
        return "".join(tokens)
    return "_".join(tokens)


_INVALID_ID_CHARS = re.compile(r"[^0-9A-Za-z_$]")


def _is_valid_identifier(name: str) -> bool:
    if not name:
        return False
    candidate = name.strip()
    if not candidate:
        return False
    return candidate.replace("_", "").replace("$", "").isalnum()


def _normalize_identifier(name: str) -> str:
    if not name:
        return ""
    text = re.sub(r"\s+", "_", name.strip())
    text = text.replace("-", "_")
    text = _INVALID_ID_CHARS.sub("", text)
    text = re.sub(r"_+", "_", text)
    return text.strip("_")


def _safe_semantic_identifier(
    raw_name: str,
    used_names: set[str],
    fallback_prefix: str,
    prefixes_to_drop: Optional[set[str]] = None,
) -> str:
    """
    Produce a YAML-safe identifier that avoids reserved keywords and duplicates.
    """

    variants: List[str] = []
    normalized = _normalize_identifier(raw_name)
    if normalized:
        variants.append(normalized)
    sanitized = _sanitize_identifier_name(raw_name, prefixes_to_drop=prefixes_to_drop)
    if sanitized:
        variants.append(sanitized.lower())
    variants.append(f"{fallback_prefix}_field")

    for variant in variants:
        candidate = _normalize_identifier(variant)
        if not candidate:
            continue
        if candidate[0].isdigit():
            candidate = f"{fallback_prefix}_{candidate}"
        candidate = candidate.lower()
        base = candidate
        suffix = 2
        while (
            base in used_names
            or base.upper() in CZ_RESERVED_WORDS
            or not _is_valid_identifier(base)
        ):
            base = f"{candidate}_{suffix}"
            suffix += 1
        if base.upper() in CZ_RESERVED_WORDS or not _is_valid_identifier(base):
            continue
        used_names.add(base)
        return base

    suffix = 1
    while True:
        candidate = f"{fallback_prefix}_{suffix}"
        if (
            candidate not in used_names
            and candidate.upper() not in CZ_RESERVED_WORDS
            and _is_valid_identifier(candidate)
        ):
            used_names.add(candidate)
            return candidate
        suffix += 1


def _could_be_identifier_column(column_name: str, base_type: str, table_name: str = "") -> bool:
    """
    Check if a column could potentially be a primary key based on naming and type.
    More strict than _is_identifier_like - used for sample data PK inference.

    Strategy:
    1. Exclude non-key types (DECIMAL, DATE, TEXT)
    2. Exclude obvious non-key columns (NAME, COMMENT, BALANCE, etc.)
    3. If table_name provided, check if column references THIS table (PK) vs OTHER tables (FK)
    4. Otherwise, only accept simple key patterns (ID, KEY, NUM, etc.)

    Args:
        column_name: The column name to check
        base_type: The column's base data type
        table_name: Optional table name to help distinguish PK from FK
    """
    # Type must be appropriate for a key (integer or short string, not decimal/date/text)
    if base_type not in {"NUMBER", "STRING", "BIGINT", "INTEGER", "INT", "VARCHAR"}:
        return False

    col_upper = column_name.upper()
    tokens = _identifier_tokens(column_name)

    # Exclude obvious non-key columns first
    exclude_indicators = {"NAME", "COMMENT", "DESC", "DESCRIPTION", "ADDRESS",
                         "PHONE", "EMAIL", "BALANCE", "AMOUNT", "PRICE", "COST",
                         "DATE", "TIME", "TIMESTAMP", "QTY", "QUANTITY", "TEXT"}
    for token in tokens:
        if token in exclude_indicators:
            return False
        if any(token.endswith(suffix) for suffix in ["BAL", "AMT", "AVAIL"]):
            return False

    # If we have table name, use it to distinguish PK from FK
    if table_name:
        table_tokens = {t.upper() for t in _identifier_tokens(table_name)}

        # Check each token in column name
        for token in tokens:
            # Token ends with KEY or ID
            if token.endswith("KEY") or token.endswith("ID"):
                # Extract the prefix (what comes before KEY/ID)
                if token.endswith("KEY"):
                    prefix = token[:-3]  # Remove "KEY"
                else:
                    prefix = token[:-2]  # Remove "ID"

                # Check if this prefix matches table name
                # e.g., ORDERKEY in ORDERS table -> ORDER matches ORDER(S)
                # But CUSTKEY in ORDERS table -> CUST doesn't match ORDER
                for table_token in table_tokens:
                    # Direct match or prefix match
                    if prefix == table_token or table_token.startswith(prefix) or prefix.startswith(table_token):
                        return True  # This is a PK candidate (references this table)

                # If prefix doesn't match table name, it's likely a FK
                # e.g., O_CUSTKEY in ORDERS -> CUST doesn't match ORDER -> FK
                # But also need to check for common FK entity names
                common_fk_entities = {
                    "CUST", "CUSTOMER", "NATION", "REGION", "SUPP", "SUPPLIER",
                    "PART", "PRODUCT", "USER", "ACCOUNT", "CATEGORY",
                    "COMPANY", "DEPARTMENT", "EMPLOYEE", "PARENT", "OWNER"
                }
                if any(prefix.startswith(entity) for entity in common_fk_entities):
                    return False  # This is a FK, not a PK

        # Also check for simple patterns like LINENUMBER, LINENUM
        for token in tokens:
            if token in {"NUM", "NO", "NUMBER"} or token.endswith("NUM") or token.endswith("NO") or token.endswith("NUMBER"):
                # But not if it looks like a quantity field
                if not any(excl in token for excl in ["QTY", "CNT", "COUNT", "AVAIL"]):
                    return True

    # Without table name, only accept very simple patterns
    simple_pk_indicators = {"ID", "KEY", "PK", "ROWID", "PRIMARY", "NUM", "NO", "CODE"}
    if any(token in simple_pk_indicators for token in tokens):
        return True

    return False


def _is_identifier_like(column_name: str, base_type: str) -> bool:
    """
    Heuristic to detect identifier-style numeric columns that should stay as dimensions.
    """

    tokens = _identifier_tokens(column_name)
    if not tokens:
        return False

    tail = tokens[-1]
    if tail in {"ID", "KEY", "ROWID", "ROW_ID", "PK", "PRIMARY", "PRIMARYKEY"}:
        return True
    if tail.endswith("ID") and len(tail) > 2:
        return True
    if tail.endswith("KEY") and len(tail) > 3:
        return True

    return False


def _should_exclude_from_relationship_matching(column_name: str, base_type: str = None) -> bool:
    """
    Check if a column should be excluded from FK-PK relationship matching.

    Excludes columns that are clearly not suitable for relationships:
    - Timestamp fields (created_at, updated_at, deleted_at, _created_at, etc.)
    - Content/text fields (description, content, comment, notes, etc.)
    - Measurement fields (amount, price, quantity without ID suffix)
    - System fields with underscore prefix (_created_at, _updated_at, etc.)

    This prevents false positive matches like:
    - comments.created_at = posts.created_at
    - comments._created_at = posts._created_at (system field)
    - comments.content = posts.content

    Returns True if column should be EXCLUDED (not used for matching)
    """
    if not column_name:
        return True

    col_upper = column_name.upper()
    tokens = _identifier_tokens(column_name)

    # Check for underscore-prefixed system fields
    # These are typically system-managed fields, not business keys
    # Examples: _created_at, _updated_at, _version, _deleted_at
    # BUT: _id, _key might be valid business keys in some schemas (e.g., MongoDB style)
    if col_upper.startswith("_"):
        # Strip the leading underscore and check if it's a system field pattern
        stripped = col_upper[1:]

        # Special case: bare _ID or _KEY might be business keys, don't exclude
        # But _CREATED_AT, _UPDATED_AT, etc. are clearly system fields
        if stripped in {"ID", "KEY", "NUM", "CODE", "NO"}:
            # These might be business keys, don't exclude
            return False

        # Common system field patterns with underscore prefix
        system_patterns = {
            "CREATED_AT", "UPDATED_AT", "DELETED_AT", "MODIFIED_AT",
            "CREATED_TIME", "UPDATED_TIME", "DELETED_TIME", "MODIFIED_TIME",
            "CREATE_TIME", "UPDATE_TIME", "DELETE_TIME", "MODIFY_TIME",
            "CREATED_DATE", "UPDATED_DATE", "DELETED_DATE", "MODIFIED_DATE",
            "CREATE_DATE", "UPDATE_DATE", "DELETE_DATE", "MODIFY_DATE",
            "CREATEDAT", "UPDATEDAT", "DELETEDAT", "MODIFIEDAT",
            "TIMESTAMP", "DATETIME", "DATE_TIME",
            "VERSION", "REVISION", "REV",
            "ROWVERSION", "ROW_VERSION",
            "ETAG", "E_TAG",
            "CREATED_BY", "UPDATED_BY", "DELETED_BY", "MODIFIED_BY",
            "CREATOR", "UPDATER", "MODIFIER",
            "DESCRIPTION", "CONTENT", "COMMENT", "NOTE", "TEXT",  # System content fields
            "AMOUNT", "PRICE", "COST", "QUANTITY", "BALANCE"  # System measurement fields
        }

        if stripped in system_patterns:
            return True

        # Also check if it ends with system patterns
        for pattern in ["_AT", "_TIME", "_DATE", "_BY", "_VERSION"]:
            if stripped.endswith(pattern):
                return True

    # Exclude timestamp/date columns
    timestamp_patterns = {
        "CREATED_AT", "UPDATED_AT", "DELETED_AT", "MODIFIED_AT",
        "CREATED_TIME", "UPDATED_TIME", "DELETED_TIME", "MODIFIED_TIME",
        "CREATE_TIME", "UPDATE_TIME", "DELETE_TIME", "MODIFY_TIME",
        "CREATEDAT", "UPDATEDAT", "DELETEDAT", "MODIFIEDAT",
        "CREATED_DATE", "UPDATED_DATE", "DELETED_DATE", "MODIFIED_DATE",
        "CREATE_DATE", "UPDATE_DATE", "DELETE_DATE", "MODIFY_DATE",
        "TIMESTAMP", "DATETIME", "DATE_TIME"
    }

    if col_upper in timestamp_patterns:
        return True

    # Check for *_AT, *_DATE, *_TIME patterns
    # But be careful: date_key, date_id are OK (they're identifiers, not timestamps)
    for token in tokens:
        if token.endswith("_AT") or token.endswith("AT"):
            # Check if it looks like a timestamp (not like STATUS_AT which might be valid)
            prev_tokens = [t for t in tokens if t != token]
            if prev_tokens and prev_tokens[-1] in {"CREATED", "UPDATED", "DELETED", "MODIFIED", "CREATE", "UPDATE", "DELETE", "MODIFY"}:
                return True
        # Exclude *_DATE and *_TIME UNLESS it's followed by KEY/ID (like date_key, time_id)
        if token.endswith("_DATE") or token.endswith("_TIME"):
            # Check if next token is KEY or ID
            token_idx = tokens.index(token)
            if token_idx < len(tokens) - 1:
                next_token = tokens[token_idx + 1]
                if next_token in {"KEY", "ID"} or next_token.endswith("KEY") or next_token.endswith("ID"):
                    continue  # date_key, time_id are OK
            return True
        # Exclude bare DATE, TIME, TIMESTAMP columns UNLESS they have KEY/ID suffix
        if token in {"DATE", "TIME", "TIMESTAMP", "DATETIME"}:
            # Check if there's a KEY or ID token
            has_key_id = any(t in {"KEY", "ID"} or t.endswith("KEY") or t.endswith("ID") for t in tokens)
            if not has_key_id:
                return True

    # Exclude content/text fields
    content_patterns = {
        "DESCRIPTION", "CONTENT", "COMMENT", "COMMENTS", "NOTE", "NOTES",
        "TEXT", "BODY", "MESSAGE", "SUMMARY", "ABSTRACT",
        "DESC", "DETAILS", "REMARKS", "MEMO"
    }

    if col_upper in content_patterns:
        return True

    for token in tokens:
        if token in content_patterns:
            # But allow if it's part of an ID pattern (rare but possible)
            if not (token.endswith("_ID") or token.endswith("_KEY")):
                return True

    # Exclude measurement fields without ID/KEY suffix
    # (amount_id or price_id would be OK, but amount or price alone are not keys)
    measurement_patterns = {"AMOUNT", "PRICE", "COST", "TOTAL", "SUBTOTAL",
                           "QUANTITY", "QTY", "COUNT", "SUM", "BALANCE"}

    for token in tokens:
        if token in measurement_patterns:
            # Check if it has ID/KEY suffix
            has_id_suffix = any(t in {"ID", "KEY"} or t.endswith("ID") or t.endswith("KEY")
                               for t in tokens)
            if not has_id_suffix:
                return True

    return False


def _table_variants(table_name: str) -> set[str]:
    table_upper = table_name.upper()
    tokens = _identifier_tokens(table_name)
    variants = {table_upper}
    variants.update(tokens)

    # Add semantic mapping for common table patterns
    semantic_mappings = {
        "ENTITY_ALPHA": {"ALPHA", "ENTITY"},
        "ENTITY_BETA": {"BETA", "ENTITY"},
        "TBL_MASTER": {"MASTER", "TBL"},
        "TBL_DETAIL": {"DETAIL", "TBL"},
        "FOO": {"FOO"},
        "BAR": {"BAR"},
        "PARENT": {"PARENT"},
        "CHILD": {"CHILD"},
    }

    # Check if table name matches known patterns
    if table_upper in semantic_mappings:
        variants.update(semantic_mappings[table_upper])

    # Extract meaningful parts from compound names
    if "_" in table_upper:
        parts = table_upper.split("_")
        variants.update(parts)
        # Add combinations
        if len(parts) >= 2:
            variants.add(parts[-1])  # Last part (e.g., "MASTER" from "TBL_MASTER")
            variants.add(parts[0])   # First part (e.g., "TBL" from "TBL_MASTER")

    for token in list(variants):
        variants.add(_singularize(token))
        if len(token) > 3:
            variants.add(token[:4])
        if len(token) > 3:
            variants.add(token[:3])
        if len(token) > 3:
            variants.add(token[-4:])
        if len(token) > 2:
            variants.add(token[-3:])

    return {variant for variant in variants if variant}




def _normalized_table_tokens(table_name: str) -> set[str]:
    """Return normalized tokens representing a table name."""
    variants = _table_variants(table_name)
    normalized: set[str] = set()
    for variant in variants:
        singular = _singularize(variant)
        if singular and singular not in _GENERIC_IDENTIFIER_TOKENS:
            normalized.add(singular.upper())
    return normalized

def _normalized_column_tokens(column_name: str) -> set[str]:
    tokens = set()
    for token in _identifier_tokens(column_name):
        if not token:
            continue
        singular = _singularize(token)
        upper = singular.upper()
        if not upper:
            continue
        if upper not in _GENERIC_IDENTIFIER_TOKENS:
            tokens.add(upper)
        # Add prefixes without common suffixes like ID/KEY
        if upper.endswith("ID") and len(upper) > 2:
            prefix = upper[:-2]
            if prefix and prefix not in _GENERIC_IDENTIFIER_TOKENS:
                tokens.add(prefix)
        if upper.endswith("KEY") and len(upper) > 3:
            prefix = upper[:-3]
            if prefix and prefix not in _GENERIC_IDENTIFIER_TOKENS:
                tokens.add(prefix)
    return tokens

def _column_mentions_table(column_name: str, table_name: str) -> bool:
    column_tokens = _normalized_column_tokens(column_name)
    if not column_tokens:
        return False
    table_tokens = _normalized_table_tokens(table_name)
    return bool(column_tokens & table_tokens)

_GENERIC_PREFIXES = {
    "DIM",
    "FACT",
    "FCT",
    "BRIDGE",
    "BRG",
    "STG",
    "ODS",
    "OD",
    "DW",
    "VW",
    "VIEW",
    "HUB",
    "SAT",
    "LNK",
    "TMP",
    "TMPV",
}


def _table_prefixes(table_name: str) -> set[str]:
    prefixes: set[str] = set()
    tokens = _identifier_tokens(table_name)
    if not tokens:
        tokens = [table_name.upper()]

    first_token = tokens[0].upper()
    if first_token in _GENERIC_PREFIXES:
        for length in range(1, min(len(first_token), 4) + 1):
            prefixes.add(first_token[:length])
        prefixes.add(first_token)
    return prefixes


def _looks_like_primary_key(table_name: str, column_name: str) -> bool:
    """
    Pure column-pattern-based primary key detection.

    CRITICAL FIX: Only identify columns as PK if they match the table name pattern.
    This prevents foreign keys like S_NATIONKEY from being misidentified as PKs.

    Args:
        table_name: Table name (used for matching)
        column_name: Column name to evaluate

    Returns:
        True if column pattern suggests it's a primary key
    """
    upper_name = column_name.strip().upper()
    upper_table = table_name.strip().upper()

    # Direct primary key indicators
    direct_pk_patterns = {
        "ID", "PK", "PRIMARY_KEY", "PKEY"
    }

    if upper_name in direct_pk_patterns:
        return True

    # Parse column tokens for structured analysis
    tokens = _identifier_tokens(column_name)
    if not tokens:
        return False

    # Parse table name tokens
    table_tokens = _identifier_tokens(table_name)
    table_core = "_".join(table_tokens) if table_tokens else upper_table

    # Pattern 1: {entity}_id or {entity}_key where entity matches table name
    if len(tokens) >= 2:
        last_token = tokens[-1]
        if last_token in {"ID", "KEY"}:
            # Extract the entity prefix
            prefix_tokens = tokens[:-1]
            prefix = "_".join(prefix_tokens)

            # CRITICAL: Only return True if prefix matches table name
            # This prevents foreign keys from being identified as PKs
            # Examples:
            #   S_SUPPKEY in SUPPLIER table: prefix="S_SUPP", matches "SUPPLIER" ✅
            #   S_NATIONKEY in SUPPLIER table: prefix="S_NATION", doesn't match "SUPPLIER" ❌
            if len(prefix) >= 1 and prefix not in _GENERIC_IDENTIFIER_TOKENS:
                # Check if prefix is contained in table name or vice versa
                if prefix in table_core or table_core in prefix:
                    return True
                # Check for abbreviations (first few chars)
                if len(prefix) >= 3 and table_core.startswith(prefix):
                    return True
                if len(table_core) >= 3 and prefix.startswith(table_core):
                    return True

    # Pattern 2: Handle compound tokens that contain KEY/ID (e.g., ORDERKEY)
    if len(tokens) >= 1:
        for token in tokens:
            if token.endswith("ID") and len(token) > 3:
                prefix = token[:-2]
                if len(prefix) >= 2 and prefix not in _GENERIC_IDENTIFIER_TOKENS:
                    # Check if prefix matches table name
                    if prefix in table_core or table_core in prefix:
                        return True
            if token.endswith("KEY") and len(token) > 4:
                prefix = token[:-3]
                if len(prefix) >= 2 and prefix not in _GENERIC_IDENTIFIER_TOKENS:
                    # Check if prefix matches table name
                    if prefix in table_core or table_core in prefix:
                        return True

    # Pattern 3: Common meaningful PK column patterns
    meaningful_pk_patterns = {
        "RECORD_ID", "ROW_ID", "ENTITY_ID", "UNIQUE_ID",
        "SEQUENCE_ID", "AUTO_ID", "SERIAL_ID"
    }

    if upper_name in meaningful_pk_patterns:
        return True

    return False


_TIME_NAME_HINTS = ("DATE", "TIME", "TIMESTAMP", "DT", "DAY", "HOUR")
_ISO_DATE_REGEX = re.compile(r"^\d{4}[-/]\d{2}[-/]\d{2}")


def _is_time_like_column(column: data_types.Column) -> bool:
    base_type = _base_type_from_type(column.column_type)
    if base_type in TIME_MEASURE_DATATYPES:
        return True
    if base_type not in {"STRING", "VARCHAR", "TEXT", "CHAR", "CHARACTER", "NVARCHAR"}:
        return False

    column_name = column.column_name.upper()
    if any(hint in column_name for hint in _TIME_NAME_HINTS):
        return True

    for value in (column.values or [])[:5]:
        val = str(value).strip()
        if not val:
            continue
        candidate = val
        if candidate.endswith("Z"):
            candidate = candidate[:-1]
        candidate = candidate.replace(" ", "T", 1)
        try:
            datetime.fromisoformat(candidate)
            return True
        except ValueError:
            pass
        if _ISO_DATE_REGEX.match(val):
            return True
    return False


def _format_literal(value: str, base_type: str) -> str:
    text = "" if value is None else str(value)
    if base_type in MEASURE_DATATYPES:
        try:
            if "." in text:
                return str(float(text))
            return str(int(text))
        except ValueError:
            pass
    lowered = text.lower()
    if lowered in {"true", "false"}:
        return lowered.upper()
    escaped = text.replace("'", "''")
    return f"'{escaped}'"


def _format_sql_identifier(name: str) -> str:
    """
    Formats an identifier for SQL by wrapping it in backticks.
    """
    return quote_identifier(name)


def _qualified_table_name(fqn: data_types.FQNParts) -> str:
    """
    Builds a fully qualified, backtick-quoted table name.
    """
    parts = [normalize_identifier(part) for part in (fqn.database, fqn.schema_name, fqn.table)]
    return join_quoted_identifiers(*(part for part in parts if part))


def _levenshtein_distance(s1: str, s2: str) -> int:
    """
    Calculate Levenshtein distance between two strings.
    Used for fuzzy column name matching.
    """
    if len(s1) < len(s2):
        return _levenshtein_distance(s2, s1)
    if len(s2) == 0:
        return len(s1)

    previous_row = range(len(s2) + 1)
    for i, c1 in enumerate(s1):
        current_row = [i + 1]
        for j, c2 in enumerate(s2):
            insertions = previous_row[j + 1] + 1
            deletions = current_row[j] + 1
            substitutions = previous_row[j] + (c1 != c2)
            current_row.append(min(insertions, deletions, substitutions))
        previous_row = current_row

    return previous_row[-1]


def _name_similarity(name1: str, name2: str) -> float:
    """
    Calculate column name similarity using universal patterns.
    Returns a score between 0.0 (completely different) and 1.0 (identical).
    """
    if not name1 or not name2:
        return 0.0

    name1_upper = name1.upper()
    name2_upper = name2.upper()

    # Exact match
    if name1_upper == name2_upper:
        return 1.0

    # Extract core entities for semantic comparison
    core1 = _extract_core_entity(name1_upper)
    core2 = _extract_core_entity(name2_upper)

    # Perfect core match (e.g., CUSTKEY vs CUSTKEY)
    if core1 == core2 and core1:
        return 0.95

    # Entity abbreviation match (e.g., CUST vs CUSTOMER)
    if _are_entity_variants(core1, core2):
        return 0.90

    # Standard normalization and Levenshtein
    norm1 = name1_upper.replace("_", "").replace("-", "")
    norm2 = name2_upper.replace("_", "").replace("-", "")

    if norm1 == norm2:
        return 0.95

    # Calculate Levenshtein-based similarity
    max_len = max(len(norm1), len(norm2))
    if max_len == 0:
        return 0.0

    distance = _levenshtein_distance(norm1, norm2)
    similarity = 1.0 - (distance / max_len)

    return max(0.0, similarity)


def _extract_core_entity(column_name: str) -> str:
    """
    Extract core entity from column name for semantic matching.

    Examples:
        c_nationkey → nation
        n_nationkey → nation
        s_nationkey → nation
        n_regionkey → region
        r_regionkey → region
        customer_id → customer
        order_id → order
    """
    name_lower = column_name.lower()

    # Remove common table prefixes (single or double letter followed by underscore)
    # Matches: c_, n_, s_, o_, l_, ps_, p_, r_, etc.
    if "_" in name_lower:
        parts = name_lower.split("_", 1)
        if len(parts) == 2 and len(parts[0]) <= 2:
            # Remove prefix like "c_", "ps_"
            core_part = parts[1]
        else:
            core_part = name_lower
    else:
        core_part = name_lower

    # Remove common suffixes (key, id, num, no, etc.)
    for suffix in ["key", "_key", "id", "_id", "num", "_num", "no", "_no"]:
        if core_part.endswith(suffix):
            core_part = core_part[:-len(suffix)]
            break

    # Clean up any trailing underscores
    core_part = core_part.strip("_")

    return core_part


def _are_entity_variants(entity1: str, entity2: str) -> bool:
    """
    Check if two entities are variants of the same concept.

    Universal approach:
    - Checks for common abbreviations (cust/customer, supp/supplier, etc.)
    - Checks for plural/singular variations
    - Uses string similarity for fuzzy matching
    """
    if not entity1 or not entity2:
        return False

    e1 = entity1.lower()
    e2 = entity2.lower()

    # Exact match
    if e1 == e2:
        return True

    # Common abbreviation patterns (universal, not schema-specific)
    abbrev_patterns = [
        ("cust", "customer"),
        ("supp", "supplier"),
        ("prod", "product"),
        ("cat", "category"),
        ("addr", "address"),
        ("qty", "quantity"),
        ("amt", "amount"),
        ("num", "number"),
        ("desc", "description"),
        ("ref", "reference"),
    ]

    for abbrev, full in abbrev_patterns:
        if (e1 == abbrev and e2.startswith(full)) or (e2 == abbrev and e1.startswith(full)):
            return True

    # Plural/singular variants
    if e1.rstrip("s") == e2.rstrip("s"):
        return True

    # Check string similarity for close matches using Levenshtein ratio
    max_len = max(len(e1), len(e2))
    if max_len:
        distance = _levenshtein_distance(e1.upper(), e2.upper())
        similarity = 1.0 - (distance / max_len)
        if similarity > 0.85:  # Very high similarity threshold
            return True

    return False


def _follows_fk_naming_pattern(fk_column: str, pk_column: str, pk_table: str) -> bool:
    """Check if foreign key follows standard naming patterns."""
    fk_upper = fk_column.upper()
    pk_upper = pk_column.upper()
    pk_table_upper = pk_table.upper()

    # Universal pattern: {table_prefix}_{entity}KEY (common across many schemas)
    if "_" in fk_upper and "_" in pk_upper:
        fk_parts = fk_upper.split("_", 1)
        pk_parts = pk_upper.split("_", 1)

        if len(fk_parts) == 2 and len(pk_parts) == 2:
            # Same entity part (e.g., CUSTKEY, NATIONKEY)
            if fk_parts[1] == pk_parts[1]:
                return True

    # Standard pattern: table_column
    if fk_upper.endswith(f"_{pk_upper}"):
        return True

    # Contains pattern
    if pk_upper in fk_upper and len(pk_upper) >= 4:
        return True

    return False


def _validate_composite_key_consistency(column_pairs: List[tuple[str, str]]) -> bool:
    """
    Validate that composite key relationships are consistent.

    Prevents errors like: s_nationkey = n_nationkey AND s_nationkey = n_regionkey
    where the same source column is mapped to multiple target columns.
    """
    source_to_target = {}

    for left_col, right_col in column_pairs:
        left_lower = left_col.lower()
        right_lower = right_col.lower()

        if left_lower in source_to_target:
            # Same source column mapped to different targets - invalid!
            if source_to_target[left_lower] != right_lower:
                return False
        else:
            source_to_target[left_lower] = right_lower

    return True


def _generate_optimized_relationship_candidates(metadata, _record_pair, status_dict, _timed_out, min_similarity_threshold=0.6):
    """
    Universal relationship candidate generation using FK-PK pattern matching.

    This function uses intelligent FK-PK matching with semantic validation,
    applicable to any database schema.
    """
    domain_patterns_cache = _get_domain_knowledge_patterns()
    # Build a lookup of all primary key candidates
    pk_lookup = {}
    for table_name, table_meta in metadata.items():
        for pk_norm, pk_cols in table_meta["pk_candidates"].items():
            pk_meta = table_meta["columns"].get(pk_norm)
            if pk_meta:
                pk_lookup[(table_name, pk_norm)] = {
                    "meta": pk_meta,
                    "columns": pk_cols,
                    "table": table_name
                }

    # For each table, find potential foreign key candidates
    for fk_table_name, fk_table_meta in metadata.items():
        if status_dict["limited_by_timeout"]:
            break
        if _timed_out():
            status_dict["limited_by_timeout"] = True
            break

        for fk_norm, fk_meta in fk_table_meta["columns"].items():
            if status_dict["limited_by_timeout"]:
                break

            # CRITICAL: Exclude columns that should never be used for relationships
            # This prevents false positives like created_at, content, description matches
            if _should_exclude_from_relationship_matching(fk_meta["names"][0], fk_meta["base_type"]):
                continue

            # Skip primary key columns (except for composite keys)
            if fk_norm in fk_table_meta["pk_candidates"]:
                pk_count = len(fk_table_meta["pk_candidates"])
                if pk_count <= 1:
                    table_convention = _identify_naming_convention(
                        fk_table_name, domain_patterns_cache
                    )
                    if table_convention not in {"dimension", "bridge"}:
                        continue
                # Composite keys with more than one PK are allowed to participate

            # CRITICAL FIX: Find ALL matching primary keys, not just the best one
            # A foreign key column can reference multiple tables' primary keys in different relationships
            # (e.g., LINEITEM.L_PARTKEY can reference both PART.P_PARTKEY and participate in PARTSUPP relationship)
            all_matches = []

            for (pk_table_name, pk_norm), pk_info in pk_lookup.items():
                # Skip same table
                if pk_table_name == fk_table_name:
                    continue

                # CRITICAL: Also exclude PK columns that shouldn't be used for relationships
                if _should_exclude_from_relationship_matching(pk_info["meta"]["names"][0], pk_info["meta"]["base_type"]):
                    continue

                # Type compatibility check
                if fk_meta["base_type"] != pk_info["meta"]["base_type"]:
                    continue

                # Calculate enhanced name similarity
                similarity = _name_similarity(fk_meta["names"][0], pk_info["meta"]["names"][0])

                # Universal enhancement: Core entity matching
                fk_core = _extract_core_entity(fk_meta["names"][0])
                pk_core = _extract_core_entity(pk_info["meta"]["names"][0])

                # Semantic validation - reject obviously wrong matches but allow hierarchical containment
                # SKIP semantic validation if entity names are too short (unreliable)
                # This allows sample data-based matching to work for poorly named columns
                if (fk_core and pk_core and len(fk_core) >= 2 and len(pk_core) >= 2
                    and fk_core != pk_core):
                    cores_related = (
                        fk_core.endswith(pk_core)
                        or pk_core.endswith(fk_core)
                        or fk_core.startswith(pk_core)
                        or pk_core.startswith(fk_core)
                        or _are_entity_variants(fk_core, pk_core)
                    )
                    if not cores_related:
                        continue

                # Entity alignment bonus
                entity_bonus = 0.0
                if fk_core and pk_core:
                    if fk_core == pk_core:
                        # Perfect core entity match - this is STRONG evidence!
                        entity_bonus = 0.50  # Increased from 0.20
                    elif _are_entity_variants(fk_core, pk_core):
                        entity_bonus = 0.30  # Increased from 0.15

                # Total score (universal - no business-specific bonuses)
                total_score = similarity + entity_bonus

                # FK naming pattern bonus
                if _follows_fk_naming_pattern(fk_meta["names"][0], pk_info["meta"]["names"][0], pk_table_name):
                    total_score += 0.10

                # ENHANCEMENT: Sample data-based FK inference
                # Use sample data to validate and strengthen relationship inference
                sample_data_confidence = 0.0
                fk_values = fk_meta.get("values") or []
                pk_values = pk_info["meta"].get("values") or []
                if fk_values and pk_values:
                    sample_data_confidence = _infer_fk_from_sample_data(
                        fk_values, pk_values, min_sample_size=20
                    )
                    # Add sample data evidence to total score
                    # Scale it appropriately (0.0-0.6 bonus for strong evidence)
                    sample_bonus = sample_data_confidence * 0.6
                    total_score += sample_bonus
                    # DEBUG logging (commented out for production)
                    # if sample_data_confidence > 0.1:
                    #     print(f"  📊 Sample data: {fk_table_name}.{fk_meta['names'][0]} → {pk_table_name}.{pk_info['meta']['names'][0]} "
                    #           f"conf={sample_data_confidence:.2f}, bonus={sample_bonus:.2f}, total={total_score:.2f}")

                # Lower threshold if core entities match perfectly
                effective_threshold = min_similarity_threshold
                if fk_core and pk_core and fk_core == pk_core:
                    effective_threshold = 0.3  # Much lower threshold for perfect entity match
                elif sample_data_confidence > 0.6:
                    # Strong sample data evidence allows lower threshold
                    effective_threshold = 0.3

                # Record ALL matches that meet the threshold, not just the best
                if total_score >= effective_threshold:
                    all_matches.append({
                        "pk_table": pk_table_name,
                        "pk_column": pk_info["meta"]["names"][0],
                        "score": total_score,
                        "similarity": similarity,
                        "entity_bonus": entity_bonus,
                        "sample_data_confidence": sample_data_confidence
                    })

            # Sort matches by score (highest first)
            all_matches.sort(key=lambda x: x["score"], reverse=True)

            # CRITICAL: Record all TIED matches (same top score) to avoid missing valid relationships
            # But avoid weak matches that create inconsistent composite keys
            if all_matches:
                # DEBUG: Log for LINEITEM FK columns
                if fk_table_name.upper() == "LINEITEM" and fk_meta["names"][0].upper() in ["L_PARTKEY", "L_SUPPKEY"]:
                    print(f"\n  🔍 DEBUG: Found {len(all_matches)} match(es) for {fk_table_name}.{fk_meta['names'][0]}")
                    for i, match in enumerate(all_matches, 1):
                        print(f"     {i}. {match['pk_table']}.{match['pk_column']} - Score: {match['score']:.3f}")

                # Get the best score
                best_score = all_matches[0]["score"]

                # Record ALL matches with the best score (handle ties)
                # This ensures we don't miss relationships when multiple PKs have equal match quality
                # Example: LINEITEM.l_partkey matches both PART.p_partkey and PARTSUPP.ps_partkey with same score
                for match in all_matches:
                    if match["score"] >= best_score:
                        _record_pair(
                            fk_table_name,
                            match["pk_table"],
                            fk_meta["names"][0],
                            match["pk_column"]
                        )
                    else:
                        # Stop when we hit a lower score (since list is sorted by score descending)
                        break

                # Also record additional matches if they have very high sample data confidence
                # (indicates strong data-driven evidence, even if score is slightly lower)
                if len(all_matches) > 1:
                    for match in all_matches[1:]:
                        if match["score"] < best_score and match["sample_data_confidence"] > 0.8:
                            _record_pair(
                                fk_table_name,
                                match["pk_table"],
                                fk_meta["names"][0],
                                match["pk_column"]
                            )
            else:
                # DEBUG: Log columns that didn't find a match (potential missing relationships)
                fk_core = _extract_core_entity(fk_meta["names"][0])
                if fk_core and len(fk_core) >= 3:  # Only log meaningful entity names
                    # Check if there are any PK candidates with the same core entity
                    potential_matches = []
                    for (pk_table_name, pk_norm), pk_info in pk_lookup.items():
                        if pk_table_name == fk_table_name:
                            continue
                        pk_core = _extract_core_entity(pk_info["meta"]["names"][0])
                        if fk_core == pk_core or _are_entity_variants(fk_core, pk_core):
                            potential_matches.append(f"{pk_table_name}.{pk_info['meta']['names'][0]}")

                    if potential_matches:
                        print(f"  ⚠️  No match found for {fk_table_name}.{fk_meta['names'][0]} (core: {fk_core})")
                        print(f"      Potential matches with same core: {', '.join(potential_matches)}")
                        print(f"      Hint: Check similarity threshold or type compatibility")


def _analyze_composite_key_patterns(
    table_meta: Dict[str, Any],
    column_pairs: List[tuple[str, str]],
    *,
    column_index: int,
) -> Dict[str, Any]:
    """
    Analyze composite key patterns for multi-column relationships.

    Args:
        table_meta: Metadata about the table
        column_pairs: List of column pairs in the relationship
        column_index: Which element of each pair belongs to this table (0 for left, 1 for right)

    Returns:
        Dict with composite key analysis results
    """
    pk_candidates = table_meta.get("pk_candidates", {})
    pk_keys_lower = {key.lower() for key in pk_candidates.keys()}

    # Check if all relationship columns form a composite key
    relationship_cols = []
    for pair in column_pairs:
        if isinstance(pair, tuple):
            if 0 <= column_index < len(pair):
                relationship_cols.append(pair[column_index])
        else:
            relationship_cols.append(pair)

    # Normalize column names for comparison
    global_prefixes = set()  # This should come from context but we'll handle it locally
    table_prefixes = _table_prefixes(
        list(table_meta.get("columns", {}).keys())[0]
        if table_meta.get("columns")
        else ""
    )

    normalized_rel_cols = [
        _sanitize_identifier_name(
            col, prefixes_to_drop=global_prefixes | table_prefixes
        )
        for col in relationship_cols
    ]

    # Check if these columns together might form a composite PK
    pk_col_count = sum(
        1 for col in normalized_rel_cols if col.lower() in pk_keys_lower
    )
    total_pk_candidates = len(pk_candidates)

    analysis = {
        "is_composite_pk": pk_col_count > 1 and pk_col_count == total_pk_candidates,
        "partial_pk": pk_col_count > 0 and pk_col_count < total_pk_candidates,
        "pk_coverage_ratio": (
            pk_col_count / total_pk_candidates if total_pk_candidates > 0 else 0
        ),
        "relationship_column_count": len(relationship_cols),
        "pk_column_count": pk_col_count,
    }

    # Additional analysis: check for sequential ID patterns (common in composite keys)
    if len(relationship_cols) > 1:
        sequential_patterns = []
        for col in relationship_cols:
            if any(
                pattern in col.upper()
                for pattern in ["_ID", "ID", "_KEY", "KEY", "_NUM", "NUM"]
            ):
                sequential_patterns.append(col)

        analysis["sequential_id_pattern"] = len(sequential_patterns) >= 2

    return analysis


def _infer_composite_cardinality(
    left_meta: Dict[str, Any],
    right_meta: Dict[str, Any],
    column_pairs: List[tuple[str, str]],
    left_values_all: List[List[str]],
    right_values_all: List[List[str]],
) -> tuple[str, str]:
    """
    Enhanced cardinality inference that considers composite keys.

    Args:
        left_meta: Left table metadata
        right_meta: Right table metadata
        column_pairs: All column pairs in the relationship
        left_values_all: Sample values for all left columns
        right_values_all: Sample values for all right columns

    Returns:
        tuple: (left_cardinality, right_cardinality)
    """
    # Analyze composite key patterns
    left_analysis = _analyze_composite_key_patterns(
        left_meta, column_pairs, column_index=0
    )
    right_analysis = _analyze_composite_key_patterns(
        right_meta, column_pairs, column_index=1
    )

    # Rule 1: If one side is a complete composite PK, it's likely "1"
    if right_analysis["is_composite_pk"]:
        return ("*", "1")
    elif left_analysis["is_composite_pk"]:
        return ("1", "*")

    # Rule 2: High PK coverage suggests uniqueness
    if right_analysis["pk_coverage_ratio"] >= 0.8:
        return ("*", "1")
    elif left_analysis["pk_coverage_ratio"] >= 0.8:
        return ("1", "*")

    # Rule 3: Composite key uniqueness analysis (if we have sufficient samples)
    MIN_SAMPLE_SIZE = 50  # P1 Fix: Much higher threshold to reduce composite key over-generation

    if (
        left_values_all
        and right_values_all
        and len(left_values_all) >= MIN_SAMPLE_SIZE
        and len(right_values_all) >= MIN_SAMPLE_SIZE
    ):

        # Create composite keys by concatenating values
        left_composite_keys = []
        right_composite_keys = []

        sample_size = min(len(left_values_all), len(right_values_all))

        for i in range(sample_size):
            left_key = "|".join(
                str(vals[i]) if i < len(vals) else "" for vals in left_values_all
            )
            right_key = "|".join(
                str(vals[i]) if i < len(vals) else "" for vals in right_values_all
            )

            if left_key and not _is_nullish(left_key):
                left_composite_keys.append(left_key)
            if right_key and not _is_nullish(right_key):
                right_composite_keys.append(right_key)

        if left_composite_keys and right_composite_keys:
            left_unique_ratio = len(set(left_composite_keys)) / len(left_composite_keys)
            right_unique_ratio = len(set(right_composite_keys)) / len(
                right_composite_keys
            )

            # Lower threshold for composite key uniqueness
            if right_unique_ratio > 0.9:
                return ("*", "1")
            elif left_unique_ratio > 0.9:
                return ("1", "*")

    # Rule 4: Fall back to single-column analysis with adaptive thresholds
    if column_pairs:
        first_left_vals = left_values_all[0] if left_values_all else []
        first_right_vals = right_values_all[0] if right_values_all else []

        # Calculate adaptive thresholds for composite key analysis
        all_values = []
        if left_values_all:
            all_values.extend(left_values_all)
        if right_values_all:
            all_values.extend(right_values_all)

        adaptive_thresholds = _calculate_adaptive_thresholds(
            all_values,
            table_count=2,  # For pairwise comparison
            base_sample_size=len(first_left_vals) if first_left_vals else 10,
        )

        return _infer_cardinality(
            first_left_vals,
            first_right_vals,
            left_analysis.get("pk_column_count", 0) > 0,
            right_analysis.get("pk_column_count", 0) > 0,
            left_table="",  # Table names not available in composite cardinality
            right_table="",
            left_column="",  # Column names not available in composite cardinality
            right_column="",
            adaptive_thresholds=adaptive_thresholds,
        )


def _detect_bridge_table_pattern(
    table_meta: Dict[str, Any],
    all_tables_meta: Dict[str, Dict[str, Any]],
) -> Dict[str, Any]:
    """
    Detect if a table follows bridge/junction table patterns for many-to-many relationships.

    Args:
        table_meta: Metadata for the table being analyzed
        all_tables_meta: Metadata for all tables in the schema

    Returns:
        Dict with bridge table analysis results
    """
    table_name = ""
    for table, meta in all_tables_meta.items():
        if meta == table_meta:
            table_name = table
            break

    if not table_name:
        return {"is_bridge": False, "confidence": 0.0, "connected_tables": []}

    columns_meta = table_meta.get("columns", {})
    pk_candidates = table_meta.get("pk_candidates", {})

    # Rule 1: Bridge table characteristics
    total_columns = len(columns_meta)
    if total_columns == 0:
        return {"is_bridge": False, "confidence": 0.0, "connected_tables": []}

    # Count foreign key-like columns
    fk_like_columns = []
    id_columns = []
    descriptive_columns = 0

    for col_name, col_info in columns_meta.items():
        names = col_info.get("names", [])
        if not names:
            continue

        original_name = names[0]
        base_type = col_info.get("base_type", "")

        # Check if column looks like an ID/foreign key
        if any(
            pattern in original_name.upper() for pattern in ["_ID", "ID", "_KEY", "KEY"]
        ):
            id_columns.append(original_name)

            # Check if this could be a foreign key to another table
            for other_table_name, other_meta in all_tables_meta.items():
                if other_table_name == table_name:
                    continue

                if _looks_like_foreign_key(table_name, other_table_name, original_name):
                    fk_like_columns.append(
                        {
                            "column": original_name,
                            "references_table": other_table_name,
                            "confidence": 0.8,
                        }
                    )
                    break

                # Check if column name contains the other table name
                other_variants = _table_variants(other_table_name)
                col_tokens = _identifier_tokens(original_name)

                for variant in other_variants:
                    if variant in col_tokens:
                        fk_like_columns.append(
                            {
                                "column": original_name,
                                "references_table": other_table_name,
                                "confidence": 0.6,
                            }
                        )
                        break
        else:
            # Count descriptive/non-ID columns
            if base_type in DIMENSION_DATATYPES:
                descriptive_columns += 1

    # Rule 2: Bridge table scoring
    fk_ratio = len(fk_like_columns) / total_columns if total_columns > 0 else 0
    id_ratio = len(id_columns) / total_columns if total_columns > 0 else 0

    # Bridge table criteria:
    # - High ratio of foreign key columns (>= 50%)
    # - Small number of total columns (typically 2-6)
    # - Low number of descriptive columns
    # - No clear single primary key (composite key pattern)

    confidence = 0.0

    # High FK ratio is a strong indicator
    if fk_ratio >= 0.7:
        confidence += 0.4
    elif fk_ratio >= 0.5:
        confidence += 0.3
    elif fk_ratio >= 0.3:
        confidence += 0.2

    # Size factor - bridge tables are typically small
    if total_columns <= 4:
        confidence += 0.3
    elif total_columns <= 6:
        confidence += 0.2
    elif total_columns <= 8:
        confidence += 0.1

    # Few descriptive columns
    if descriptive_columns == 0:
        confidence += 0.2
    elif descriptive_columns <= 1:
        confidence += 0.1

    # Composite key pattern (multiple PKs or IDs)
    if len(pk_candidates) >= 2 or len(id_columns) >= 2:
        confidence += 0.1

    # Name-based heuristics
    table_upper = table_name.upper()
    bridge_keywords = {
        "BRIDGE",
        "JUNCTION",
        "LINK",
        "ASSOC",
        "ASSOCIATION",
        "REL",
        "RELATIONSHIP",
        "MAP",
        "MAPPING",
        "XREF",
        "CROSS_REF",
        "CONNECTOR",
    }

    for keyword in bridge_keywords:
        if keyword in table_upper:
            confidence += 0.2
            break

    # Multi-word table names that combine other table names (e.g., CUSTOMER_ORDER)
    table_tokens = _identifier_tokens(table_name)
    if len(table_tokens) >= 2:
        matching_tables = []
        for token in table_tokens:
            for other_table_name in all_tables_meta.keys():
                if other_table_name != table_name:
                    other_variants = _table_variants(other_table_name)
                    if token in other_variants:
                        matching_tables.append(other_table_name)

        if len(set(matching_tables)) >= 2:
            confidence += 0.15

    # Promote classic two-key bridge tables
    if total_columns <= 3 and len(fk_like_columns) >= 2:
        confidence = max(confidence, 0.85)

    # Normalize confidence
    confidence = min(confidence, 1.0)

    is_bridge = confidence >= 0.6  # Slightly lower threshold to capture standard two-key bridges

    # Get unique connected tables with sufficient confidence
    connected_tables_set = {
        fk["references_table"] for fk in fk_like_columns if fk["confidence"] >= 0.65
    }
    connected_tables = list(connected_tables_set)

    return {
        "is_bridge": is_bridge,
        "confidence": confidence,
        "connected_tables": connected_tables,
        "fk_like_columns": fk_like_columns,
        "fk_ratio": fk_ratio,
        "id_ratio": id_ratio,
        "total_columns": total_columns,
        "descriptive_columns": descriptive_columns,
    }


def _detect_many_to_many_relationships(
    raw_tables: List[tuple[data_types.FQNParts, data_types.Table]],
    metadata: Dict[str, Dict[str, Any]],
    existing_relationships: List[semantic_model_pb2.Relationship],
) -> List[semantic_model_pb2.Relationship]:
    """
    Detect many-to-many relationships through bridge table analysis.

    Args:
        raw_tables: List of raw table metadata
        metadata: Processed table metadata
        existing_relationships: Already detected relationships

    Returns:
        List of additional many-to-many relationships
    """
    many_to_many_relationships = []

    # Analyze each table for bridge patterns
    bridge_tables = {}

    for table_name, table_meta in metadata.items():
        bridge_analysis = _detect_bridge_table_pattern(table_meta, metadata)

        if (
            bridge_analysis["is_bridge"]
            and len(bridge_analysis["connected_tables"]) >= 2
        ):
            bridge_tables[table_name] = bridge_analysis

            logger.debug(
                f"Detected bridge table: {table_name} "
                f"(confidence: {bridge_analysis['confidence']:.2f}, "
                f"connects: {bridge_analysis['connected_tables']})"
            )

    # Create many-to-many relationships through bridge tables
    for bridge_table, bridge_info in bridge_tables.items():
        raw_connected = bridge_info["connected_tables"]
        connected_tables = []
        for table in raw_connected:
            if table not in connected_tables:
                connected_tables.append(table)

        if len(connected_tables) < 2:
            continue

        bridge_meta = metadata.get(bridge_table, {})
        bridge_pk_names = {
            name.upper()
            for pk_list in bridge_meta.get("pk_candidates", {}).values()
            for name in pk_list
        }

        existing_pair_names = {
            (rel.left_table, rel.right_table) for rel in existing_relationships
        }

        # Ensure bridge table retains direct FK relationships to connected tables
        for target_table in connected_tables:
            if (bridge_table, target_table) not in existing_pair_names:
                fk_cols = [
                    fk["column"]
                    for fk in bridge_info["fk_like_columns"]
                    if fk["references_table"] == target_table and fk["confidence"] >= 0.5
                ]
                if not fk_cols:
                    continue
                if bridge_pk_names and not any(col.upper() in bridge_pk_names for col in fk_cols):
                    continue
                target_meta = metadata.get(target_table)
                if not target_meta:
                    continue
                pk_candidates = target_meta.get("pk_candidates", {})
                if not pk_candidates:
                    continue
                # Pick the first declared PK column
                pk_column = next(iter(pk_candidates.values()))[0]

                direct_rel = semantic_model_pb2.Relationship(
                    name=f"{bridge_table}_to_{target_table}",
                    left_table=bridge_table,
                    right_table=target_table,
                    join_type=semantic_model_pb2.JoinType.inner,
                    relationship_type=semantic_model_pb2.RelationshipType.many_to_one,
                )
                direct_rel.relationship_columns.append(
                    semantic_model_pb2.RelationKey(
                        left_column=fk_cols[0].upper(),
                        right_column=pk_column.upper(),
                    )
                )
                many_to_many_relationships.append(direct_rel)
                existing_pair_names.add((bridge_table, target_table))

        # Create relationships for each pair of connected tables
        for i in range(len(connected_tables)):
            for j in range(i + 1, len(connected_tables)):
                left_table = connected_tables[i]
                right_table = connected_tables[j]

                if left_table == right_table:
                    continue

                # Normalize orientation for consistency (fact -> dimension)
                domain_patterns = _get_domain_knowledge_patterns()
                left_conv = _identify_naming_convention(left_table, domain_patterns)
                right_conv = _identify_naming_convention(right_table, domain_patterns)
                if left_conv == "dimension" and right_conv == "fact":
                    left_table, right_table = right_table, left_table
                elif left_conv == right_conv and left_table > right_table:
                    # Deterministic ordering when conventions match
                    left_table, right_table = right_table, left_table

                # Skip if direct relationship already exists
                existing_pair_names = {
                    (rel.left_table, rel.right_table) for rel in existing_relationships
                } | {
                    (rel.right_table, rel.left_table) for rel in existing_relationships
                }
                existing_pair_names |= {
                    (rel.left_table, rel.right_table) for rel in many_to_many_relationships
                } | {
                    (rel.right_table, rel.left_table) for rel in many_to_many_relationships
                }

                if (left_table, right_table) in existing_pair_names:
                    continue

                # Find the foreign key columns in the bridge table
                left_fk_cols = []
                right_fk_cols = []

                for fk_info in bridge_info["fk_like_columns"]:
                    if (
                        fk_info["references_table"] == left_table
                        and fk_info["confidence"] >= 0.5
                    ):
                        left_fk_cols.append(fk_info["column"])
                    elif (
                        fk_info["references_table"] == right_table
                        and fk_info["confidence"] >= 0.5
                    ):
                        right_fk_cols.append(fk_info["column"])

                if not left_fk_cols or not right_fk_cols:
                    continue

                # Create many-to-many relationship
                relationship = semantic_model_pb2.Relationship(
                    name=f"{left_table}_to_{right_table}_via_{bridge_table}",
                    left_table=left_table,
                    right_table=right_table,
                    join_type=semantic_model_pb2.JoinType.inner,  # Bridge tables typically require INNER JOINs
                    relationship_type=semantic_model_pb2.RelationshipType.many_to_many,
                )

                # For now, we'll document the bridge table in the relationship name
                # In a more advanced implementation, we could add bridge table support to the protobuf schema

                # Try to infer the actual FK relationships
                # This is a simplified approach - in practice, you'd need more sophisticated FK detection
                if left_fk_cols and right_fk_cols:
                    # Use the first detected FK columns as a representative
                    relationship.relationship_columns.append(
                        semantic_model_pb2.RelationKey(
                            left_column=left_fk_cols[0].upper(),
                            right_column=right_fk_cols[0].upper(),
                        )
                    )

                many_to_many_relationships.append(relationship)

                logger.debug(
                    f"Created many-to-many relationship: {left_table} <-> {right_table} via {bridge_table}"
                )

    return many_to_many_relationships


def _calculate_relationship_confidence(
    left_table: str,
    right_table: str,
    column_pairs: List[tuple[str, str]],
    left_meta: Dict[str, Any],
    right_meta: Dict[str, Any],
    left_has_pk: bool,
    right_has_pk: bool,
    left_values: List[str],
    right_values: List[str],
    cardinality_result: tuple[str, str],
    join_type: int,
    adaptive_thresholds: Optional[Dict[str, float]] = None,
) -> Dict[str, Any]:
    """
    Calculate confidence score and reasoning for relationship inference.

    Args:
        left_table: Name of the left table
        right_table: Name of the right table
        column_pairs: Column pairs in the relationship
        left_meta: Left table metadata
        right_meta: Right table metadata
        left_has_pk: Whether left side has primary key
        right_has_pk: Whether right side has primary key
        left_values: Sample values from left column
        right_values: Sample values from right column
        cardinality_result: Inferred cardinality tuple
        join_type: Inferred join type
        adaptive_thresholds: Adaptive thresholds used

    Returns:
        Dict with confidence score and detailed reasoning
    """
    confidence_score = 0.0
    reasoning_factors = []
    evidence_details = {}

    # Factor 1: Primary key metadata evidence (universal for all schemas)
    if left_has_pk or right_has_pk:
        # Reduced from 0.4 to 0.3 to decrease metadata dependency
        pk_confidence = 0.3
        confidence_score += pk_confidence
        if left_has_pk and right_has_pk:
            reasoning_factors.append(
                "Both sides have primary key metadata (very strong evidence)"
            )
            evidence_details["pk_evidence"] = "both_pk"
        elif right_has_pk:
            reasoning_factors.append(
                "Right side has primary key metadata (strong evidence)"
            )
            evidence_details["pk_evidence"] = "right_pk"
        elif left_has_pk:
            reasoning_factors.append(
                "Left side has primary key metadata (strong evidence)"
            )
            evidence_details["pk_evidence"] = "left_pk"

    # Factor 2: Name similarity and pattern matching
    if column_pairs:
        total_name_similarity = 0.0
        misaligned_fk_columns: List[str] = []
        aligned_fk_columns = 0
        for left_col, right_col in column_pairs:
            similarity = _name_similarity(left_col, right_col)
            total_name_similarity += similarity

            if _column_mentions_table(left_col, right_table):
                aligned_fk_columns += 1
            else:
                misaligned_fk_columns.append(left_col)

        avg_name_similarity = total_name_similarity / len(column_pairs)
        evidence_details["avg_name_similarity"] = avg_name_similarity

        # Universal name similarity scoring (no schema-specific bonuses)
        if avg_name_similarity >= 0.9:
            name_confidence = 0.35
            reasoning_factors.append(
                f"Very high column name similarity ({avg_name_similarity:.2f})"
            )
        elif avg_name_similarity >= 0.7:
            name_confidence = 0.25
            reasoning_factors.append(
                f"High column name similarity ({avg_name_similarity:.2f})"
            )
        elif avg_name_similarity >= 0.5:
            name_confidence = 0.2   # Enhanced: 0.15 → 0.2
            reasoning_factors.append(
                f"Moderate column name similarity ({avg_name_similarity:.2f})"
            )
        elif avg_name_similarity >= 0.3:
            name_confidence = 0.1
            reasoning_factors.append(
                f"Low column name similarity ({avg_name_similarity:.2f})"
            )
        else:
            name_confidence = 0.05
            reasoning_factors.append(
                f"Very low column name similarity ({avg_name_similarity:.2f})"
            )

        confidence_score += name_confidence

        generic_pair_count = sum(
            1
            for left_col, right_col in column_pairs
            if _is_generic_identifier(left_col)
            and _is_generic_identifier(right_col)
        )
        if generic_pair_count:
            penalty = min(0.15 * generic_pair_count, 0.3)
            confidence_score = max(confidence_score - penalty, 0.0)
            reasoning_factors.append(
                f"Generic identifier names detected on both sides (-{penalty:.2f} confidence)"
            )

        # Check for foreign key naming patterns
        fk_pattern_confidence = 0.0
        for left_col, right_col in column_pairs:
            if _looks_like_foreign_key(left_table, right_table, left_col):
                fk_pattern_confidence += 0.1
                reasoning_factors.append(
                    f"Column '{left_col}' follows FK naming pattern"
                )

        confidence_score += min(fk_pattern_confidence, 0.2)

        if misaligned_fk_columns:
            mismatched = ", ".join(sorted({col.upper() for col in misaligned_fk_columns}))
            reasoning_factors.append(
                f"Foreign key column(s) {mismatched} do not reference table '{right_table.upper()}'"
            )
            confidence_score = min(confidence_score, 0.15)
        elif aligned_fk_columns:
            alignment_bonus = min(0.15, 0.05 * aligned_fk_columns)
            confidence_score += alignment_bonus
            reasoning_factors.append(
                f"Foreign key column naming aligns with target table '{right_table.upper()}'"
            )

    # Factor 3: Sample data uniqueness consistency
    if left_values and right_values:
        sample_size = min(len(left_values), len(right_values))
        evidence_details["sample_size"] = sample_size

        if sample_size >= 20:  # Sufficient sample size
            left_non_null = [v for v in left_values if not _is_nullish(v)]
            right_non_null = [v for v in right_values if not _is_nullish(v)]

            if left_non_null and right_non_null:
                left_unique_ratio = len(set(left_non_null)) / len(left_non_null)
                right_unique_ratio = len(set(right_non_null)) / len(right_non_null)

                evidence_details["left_unique_ratio"] = left_unique_ratio
                evidence_details["right_unique_ratio"] = right_unique_ratio

                # Check if uniqueness pattern matches inferred cardinality
                left_card, right_card = cardinality_result
                uniqueness_threshold = (
                    adaptive_thresholds.get("uniqueness_threshold", 0.95)
                    if adaptive_thresholds
                    else 0.95
                )

                cardinality_consistency = False
                if left_card == "1" and left_unique_ratio > uniqueness_threshold:
                    cardinality_consistency = True
                elif (
                    left_card in ("*", "+")
                    and left_unique_ratio <= uniqueness_threshold
                ):
                    cardinality_consistency = True

                if right_card == "1" and right_unique_ratio > uniqueness_threshold:
                    cardinality_consistency = cardinality_consistency and True
                elif (
                    right_card in ("*", "+")
                    and right_unique_ratio <= uniqueness_threshold
                ):
                    cardinality_consistency = cardinality_consistency and True

                if cardinality_consistency:
                    uniqueness_confidence = 0.2
                    reasoning_factors.append(
                        "Sample uniqueness patterns support inferred cardinality"
                    )
                else:
                    uniqueness_confidence = 0.1
                    reasoning_factors.append(
                        "Sample uniqueness patterns partially support cardinality"
                    )

                confidence_score += uniqueness_confidence
        else:
            reasoning_factors.append(
                f"Limited sample size ({sample_size}) reduces confidence"
            )

    # Factor 4: Data type compatibility
    if column_pairs and left_meta and right_meta:
        type_compatible_count = 0
        for left_col, right_col in column_pairs:
            # Get column types
            left_col_key = _sanitize_identifier_name(left_col)
            right_col_key = _sanitize_identifier_name(right_col)

            left_col_meta = left_meta.get("columns", {}).get(left_col_key, {})
            right_col_meta = right_meta.get("columns", {}).get(right_col_key, {})

            left_type = left_col_meta.get("base_type", "")
            right_type = right_col_meta.get("base_type", "")

            if left_type == right_type:
                type_compatible_count += 1

        type_compatibility_ratio = type_compatible_count / len(column_pairs)
        evidence_details["type_compatibility_ratio"] = type_compatibility_ratio

        if type_compatibility_ratio == 1.0:
            type_confidence = 0.1
            reasoning_factors.append("All column types are compatible")
        elif type_compatibility_ratio >= 0.8:
            type_confidence = 0.08
            reasoning_factors.append("Most column types are compatible")
        elif type_compatibility_ratio >= 0.5:
            type_confidence = 0.05
            reasoning_factors.append("Some column types are compatible")
        else:
            type_confidence = 0.0
            reasoning_factors.append("Column type compatibility is questionable")

        confidence_score += type_confidence

    # Bonus: identical column identifiers provide strong evidence
    if any(
        left_col.upper() == right_col.upper() for left_col, right_col in column_pairs
    ):
        confidence_score += 0.1
        reasoning_factors.append(
            "Foreign key and primary key share identical identifier"
        )

    # Factor 5: Table role consistency (business logic)
    left_role = _detect_table_role(left_table, left_meta)
    right_role = _detect_table_role(right_table, right_meta)
    evidence_details["left_table_role"] = left_role
    evidence_details["right_table_role"] = right_role

    relationship_context = _get_business_relationship_context(
        left_table, right_table, left_role, right_role
    )
    evidence_details["relationship_context"] = relationship_context

    if relationship_context in ["fact_to_dimension", "dimension_to_fact"]:
        role_confidence = 0.15
        reasoning_factors.append(
            f"Strong business relationship pattern: {relationship_context}"
        )
    elif relationship_context in ["dimension_hierarchy", "bridge_relationship"]:
        role_confidence = 0.1
        reasoning_factors.append(
            f"Valid business relationship pattern: {relationship_context}"
        )
    elif relationship_context == "fact_to_fact":
        role_confidence = 0.05
        reasoning_factors.append("Unusual but possible fact-to-fact relationship")
    else:
        role_confidence = 0.0
        reasoning_factors.append("Unclear business relationship pattern")

    confidence_score += role_confidence

    # Factor 6: Multiple column relationships (composite keys)
    if len(column_pairs) > 1:
        composite_confidence = 0.05
        reasoning_factors.append(
            f"Multi-column relationship ({len(column_pairs)} columns) increases confidence"
        )
        confidence_score += composite_confidence

    # Universal penalty for inconsistent composite keys (works for ANY schema)
    if len(column_pairs) > 1:
        if not _validate_composite_key_consistency(column_pairs):
            # This is a logical error: same source column cannot map to multiple different targets
            confidence_score = max(confidence_score - 0.5, 0.1)  # Heavy penalty
            reasoning_factors.append("Inconsistent composite key detected (same source column mapped to multiple targets)")
            evidence_details["inconsistent_composite"] = True

    # Normalize confidence score to 0-1 range
    confidence_score = min(confidence_score, 1.0)

    # Determine confidence level category
    if confidence_score >= 0.8:
        confidence_level = "very_high"
        confidence_description = "Very High Confidence"
    elif confidence_score >= 0.6:
        confidence_level = "high"
        confidence_description = "High Confidence"
    elif confidence_score >= 0.4:
        confidence_level = "medium"
        confidence_description = "Medium Confidence"
    elif confidence_score >= 0.2:
        confidence_level = "low"
        confidence_description = "Low Confidence"
    else:
        confidence_level = "very_low"
        confidence_description = "Very Low Confidence"

    return {
        "confidence_score": confidence_score,
        "confidence_level": confidence_level,
        "confidence_description": confidence_description,
        "reasoning_factors": reasoning_factors,
        "evidence_details": evidence_details,
        "inferred_cardinality": f"{cardinality_result[0]}:{cardinality_result[1]}",
        "join_type": (
            "INNER" if join_type == semantic_model_pb2.JoinType.inner else "LEFT_OUTER"
        ),
        "column_count": len(column_pairs),
    }


def _get_domain_knowledge_patterns() -> Dict[str, Any]:
    """
    Define common data warehouse patterns and domain knowledge.

    Returns:
        Dict containing predefined patterns and business rules
    """
    return {
        # Common business entity patterns
        "business_entities": {
            "customer": {
                "table_patterns": [
                    "CUSTOMER",
                    "CUST",
                    "CLIENT",
                    "ACCOUNT_HOLDER",
                    "USER",
                    "MEMBER",
                ],
                "pk_patterns": [
                    "CUSTOMER_ID",
                    "CUST_ID",
                    "CLIENT_ID",
                    "USER_ID",
                    "MEMBER_ID",
                ],
                "typical_attributes": [
                    "NAME",
                    "EMAIL",
                    "PHONE",
                    "ADDRESS",
                    "STATUS",
                    "TYPE",
                    "SEGMENT",
                ],
                "role": "dimension",
            },
            "product": {
                "table_patterns": ["PRODUCT", "ITEM", "SKU", "INVENTORY", "CATALOG"],
                "pk_patterns": ["PRODUCT_ID", "ITEM_ID", "SKU", "PRODUCT_KEY"],
                "typical_attributes": [
                    "NAME",
                    "DESCRIPTION",
                    "CATEGORY",
                    "PRICE",
                    "BRAND",
                    "STATUS",
                ],
                "role": "dimension",
            },
            "order": {
                "table_patterns": ["ORDER", "TRANSACTION", "SALE", "PURCHASE"],
                "pk_patterns": [
                    "ORDER_ID",
                    "TRANSACTION_ID",
                    "SALE_ID",
                    "ORDER_NUMBER",
                ],
                "typical_attributes": ["DATE", "AMOUNT", "STATUS", "QUANTITY", "TOTAL"],
                "role": "fact",
            },
            "date": {
                "table_patterns": ["DATE", "TIME", "CALENDAR", "DIM_DATE"],
                "pk_patterns": ["DATE_ID", "DATE_KEY", "TIME_ID"],
                "typical_attributes": [
                    "YEAR",
                    "MONTH",
                    "DAY",
                    "QUARTER",
                    "WEEK",
                    "WEEKDAY",
                ],
                "role": "dimension",
            },
            "location": {
                "table_patterns": [
                    "LOCATION",
                    "GEOGRAPHY",
                    "ADDRESS",
                    "REGION",
                    "TERRITORY",
                ],
                "pk_patterns": ["LOCATION_ID", "GEO_ID", "ADDRESS_ID", "REGION_ID"],
                "typical_attributes": [
                    "COUNTRY",
                    "STATE",
                    "CITY",
                    "ZIP",
                    "LATITUDE",
                    "LONGITUDE",
                ],
                "role": "dimension",
            },
            "employee": {
                "table_patterns": ["EMPLOYEE", "STAFF", "WORKER", "PERSONNEL"],
                "pk_patterns": ["EMPLOYEE_ID", "STAFF_ID", "EMP_ID"],
                "typical_attributes": [
                    "NAME",
                    "DEPARTMENT",
                    "TITLE",
                    "MANAGER",
                    "HIRE_DATE",
                ],
                "role": "dimension",
            },
        },
        # Common relationship patterns in data warehouses
        "relationship_patterns": {
            "star_schema": {
                "pattern": "fact_to_dimension",
                "confidence_boost": 0.2,
                "description": "Standard star schema fact-to-dimension relationship",
            },
            "snowflake_schema": {
                "pattern": "dimension_hierarchy",
                "confidence_boost": 0.15,
                "description": "Snowflake schema dimension hierarchy",
            },
            "bridge_table": {
                "pattern": "many_to_many_via_bridge",
                "confidence_boost": 0.1,
                "description": "Many-to-many relationship through bridge table",
            },
            "time_dimension": {
                "pattern": "temporal_relationship",
                "confidence_boost": 0.25,
                "description": "Time-based relationship (very common in warehouses)",
            },
        },
        # Known FK patterns that often appear in real data warehouses
        "common_fk_patterns": {
            "customer_references": [
                "CUSTOMER_ID",
                "CUST_ID",
                "CLIENT_ID",
                "ACCOUNT_ID",
                "USER_ID",
            ],
            "product_references": [
                "PRODUCT_ID",
                "ITEM_ID",
                "SKU",
                "PROD_ID",
                "CATALOG_ID",
            ],
            "date_references": [
                "DATE_ID",
                "ORDER_DATE_ID",
                "SHIP_DATE_ID",
                "CREATE_DATE_ID",
                "TRANSACTION_DATE_ID",
                "DATE_KEY",
            ],
            "location_references": [
                "LOCATION_ID",
                "ADDRESS_ID",
                "SHIP_TO_ID",
                "BILL_TO_ID",
                "WAREHOUSE_ID",
                "STORE_ID",
            ],
        },
        # Table naming conventions that indicate specific patterns
        "naming_conventions": {
            "fact_indicators": [
                "FACT_",
                "FCT_",
                "F_",
                "SALES_",
                "ORDERS_",
                "TRANSACTIONS_",
                "REVENUE_",
                "METRICS_",
                "EVENTS_",
                "ACTIVITY_",
            ],
            "dimension_indicators": [
                "DIM_",
                "D_",
                "REF_",
                "LKP_",
                "LOOKUP_",
                "MASTER_",
            ],
            "bridge_indicators": [
                "BRG_",
                "BRIDGE_",
                "XREF_",
                "MAP_",
                "ASSOC_",
                "LINK_",
            ],
            "staging_indicators": [
                "STG_",
                "STAGING_",
                "TMP_",
                "TEMP_",
                "RAW_",
                "LANDING_",
            ],
        },
    }


def _apply_domain_knowledge(
    left_table: str,
    right_table: str,
    column_pairs: List[tuple[str, str]],
    left_meta: Dict[str, Any],
    right_meta: Dict[str, Any],
    current_confidence: float,
) -> Dict[str, Any]:
    """
    Apply domain knowledge to enhance relationship inference confidence.

    Args:
        left_table: Name of the left table
        right_table: Name of the right table
        column_pairs: Column pairs in the relationship
        left_meta: Left table metadata
        right_meta: Right table metadata
        current_confidence: Current confidence score

    Returns:
        Dict with enhanced confidence and domain knowledge insights
    """
    domain_patterns = _get_domain_knowledge_patterns()
    enhancement_factors = []
    confidence_boost = 0.0

    # Factor 1: Recognize business entity patterns
    left_entity = _identify_business_entity(left_table, left_meta, domain_patterns)
    right_entity = _identify_business_entity(right_table, right_meta, domain_patterns)

    if left_entity and right_entity:
        # Check for common business relationship patterns
        entity_pair = f"{left_entity}-{right_entity}"
        common_pairs = {
            "order-customer": 0.25,
            "order-product": 0.25,
            "order-date": 0.3,
            "customer-location": 0.2,
            "product-location": 0.15,
            "employee-location": 0.15,
            "order-employee": 0.2,
        }

        # Check both directions
        if entity_pair in common_pairs:
            boost = common_pairs[entity_pair]
            confidence_boost += boost
            enhancement_factors.append(
                f"Recognized common business pattern: {entity_pair} (+{boost:.2f})"
            )
        elif f"{right_entity}-{left_entity}" in common_pairs:
            boost = common_pairs[f"{right_entity}-{left_entity}"]
            confidence_boost += boost
            enhancement_factors.append(
                f"Recognized common business pattern: {right_entity}-{left_entity} (+{boost:.2f})"
            )

    # Factor 2: Check for standard FK naming patterns
    for left_col, right_col in column_pairs:
        fk_pattern_match = _check_standard_fk_patterns(
            left_col, right_col, domain_patterns
        )
        if fk_pattern_match:
            confidence_boost += 0.15
            enhancement_factors.append(
                f"Standard FK pattern detected: {fk_pattern_match}"
            )

    # Factor 3: Table naming convention analysis
    left_convention = _identify_naming_convention(left_table, domain_patterns)
    right_convention = _identify_naming_convention(right_table, domain_patterns)

    if left_convention and right_convention:
        # Boost confidence for expected patterns
        if (left_convention == "fact" and right_convention == "dimension") or (
            left_convention == "dimension" and right_convention == "fact"
        ):
            confidence_boost += 0.2
            enhancement_factors.append("Standard fact-dimension naming pattern (+0.20)")
        elif left_convention == "dimension" and right_convention == "dimension":
            hierarchy_boost = 0.2
            confidence_boost += hierarchy_boost
            enhancement_factors.append(
                f"Dimension hierarchy naming pattern (+{hierarchy_boost:.2f})"
            )

    # Additional boost when FK column explicitly references the target table name
    for left_col, right_col in column_pairs:
        if _column_mentions_table(left_col, right_table):
            confidence_boost += 0.1
            enhancement_factors.append(
                f"Foreign key column '{left_col}' references target table tokens (+0.10)"
            )
            break

    # Factor 4: Time dimension special handling
    if _is_time_dimension_pattern(right_table, right_meta, domain_patterns):
        confidence_boost += 0.2
        enhancement_factors.append("Time dimension relationship (very common) (+0.20)")

    # Factor 5: Schema pattern recognition (star vs snowflake)
    schema_pattern = _detect_schema_pattern(
        left_table, right_table, left_meta, right_meta, domain_patterns
    )
    if schema_pattern:
        pattern_boost = domain_patterns["relationship_patterns"][schema_pattern][
            "confidence_boost"
        ]
        confidence_boost += pattern_boost
        pattern_desc = domain_patterns["relationship_patterns"][schema_pattern][
            "description"
        ]
        enhancement_factors.append(
            f"Schema pattern: {pattern_desc} (+{pattern_boost:.2f})"
        )

    # Apply the boost but cap the final confidence at 1.0
    enhanced_confidence = min(current_confidence + confidence_boost, 1.0)

    return {
        "enhanced_confidence": enhanced_confidence,
        "confidence_boost": confidence_boost,
        "domain_factors": enhancement_factors,
        "left_entity": left_entity,
        "right_entity": right_entity,
        "left_convention": left_convention,
        "right_convention": right_convention,
        "schema_pattern": schema_pattern,
    }


def _identify_business_entity(
    table_name: str, table_meta: Dict[str, Any], domain_patterns: Dict[str, Any]
) -> Optional[str]:
    """Identify what business entity a table represents."""
    table_upper = table_name.upper()
    business_entities = domain_patterns["business_entities"]

    for entity_type, entity_info in business_entities.items():
        # Check table name patterns
        for pattern in entity_info["table_patterns"]:
            if pattern in table_upper:
                return entity_type

        # Check primary key patterns
        pk_candidates = table_meta.get("pk_candidates", {})
        for pk_pattern in entity_info["pk_patterns"]:
            for pk_norm in pk_candidates.keys():
                if (
                    pk_pattern.replace("_", "").upper()
                    in pk_norm.replace("_", "").upper()
                ):
                    return entity_type

    return None


def _check_standard_fk_patterns(
    left_col: str, right_col: str, domain_patterns: Dict[str, Any]
) -> Optional[str]:
    """Check if column pair matches standard FK patterns."""
    common_fks = domain_patterns["common_fk_patterns"]

    left_upper = left_col.upper()
    right_upper = right_col.upper()

    for pattern_type, patterns in common_fks.items():
        for pattern in patterns:
            if pattern in left_upper or pattern in right_upper:
                return f"{pattern_type}: {pattern}"

    return None


def _identify_naming_convention(
    table_name: str, domain_patterns: Dict[str, Any]
) -> Optional[str]:
    """Identify the naming convention used for a table."""
    table_upper = table_name.upper()
    naming_conventions = domain_patterns["naming_conventions"]

    for convention_type, indicators in naming_conventions.items():
        for indicator in indicators:
            if table_upper.startswith(indicator) or indicator in table_upper:
                return convention_type.replace("_indicators", "")

    return None


def _is_time_dimension_pattern(
    table_name: str, table_meta: Dict[str, Any], domain_patterns: Dict[str, Any]
) -> bool:
    """Check if table follows time dimension patterns."""
    table_upper = table_name.upper()
    time_patterns = domain_patterns["business_entities"]["date"]["table_patterns"]

    for pattern in time_patterns:
        if pattern in table_upper:
            return True

    # Check for typical time dimension attributes
    time_attributes = ["YEAR", "MONTH", "DAY", "QUARTER", "WEEK", "DATE"]
    columns = table_meta.get("columns", {})

    time_attr_count = 0
    for col_info in columns.values():
        names = col_info.get("names", [])
        if names:
            col_name = names[0].upper()
            for time_attr in time_attributes:
                if time_attr in col_name:
                    time_attr_count += 1
                    break

    # If more than 30% of columns are time-related, likely a time dimension
    total_columns = len(columns)
    if total_columns > 0 and time_attr_count / total_columns > 0.3:
        return True

    return False


def _detect_schema_pattern(
    left_table: str,
    right_table: str,
    left_meta: Dict[str, Any],
    right_meta: Dict[str, Any],
    domain_patterns: Dict[str, Any],
) -> Optional[str]:
    """Detect common schema patterns (star, snowflake, etc.)."""
    left_role = _detect_table_role(left_table, left_meta)
    right_role = _detect_table_role(right_table, right_meta)

    # Star schema pattern: fact table to dimension
    if (left_role == "fact" and right_role == "dimension") or (
        left_role == "dimension" and right_role == "fact"
    ):
        return "star_schema"

    # Snowflake schema pattern: dimension to dimension
    if left_role == "dimension" and right_role == "dimension":
        return "snowflake_schema"

    # Time dimension pattern (very common)
    if _is_time_dimension_pattern(
        right_table, right_meta, domain_patterns
    ) or _is_time_dimension_pattern(left_table, left_meta, domain_patterns):
        return "time_dimension"

    # Bridge table pattern
    if left_role == "bridge" or right_role == "bridge":
        return "bridge_table"

    return None


def _infer_pk_from_sample_data(
    column_values: List[Any],
    min_uniqueness: float = 0.95,
    min_sample_size: int = 20,
) -> bool:
    """
    Infer if a column is likely a primary key based on sample data uniqueness.

    Args:
        column_values: Sample values from the column
        min_uniqueness: Minimum uniqueness ratio to be considered a PK (default 0.95)
        min_sample_size: Minimum number of samples required

    Returns:
        bool: True if column appears to be a primary key based on uniqueness
    """
    if not column_values or len(column_values) < min_sample_size:
        return False

    # Filter out null values
    non_null = [v for v in column_values if not _is_nullish(v)]
    if len(non_null) < min_sample_size:
        return False

    # Calculate uniqueness ratio
    unique_count = len(set(str(v) for v in non_null))
    total_count = len(non_null)
    uniqueness_ratio = unique_count / total_count

    return uniqueness_ratio >= min_uniqueness


def _infer_fk_from_sample_data(
    fk_values: List[Any],
    pk_values: List[Any],
    min_sample_size: int = 20,
) -> float:
    """
    Infer FK-PK relationship likelihood from sample data patterns.

    Analyzes uniqueness and value overlap patterns to determine if fk_values
    could be a foreign key referencing pk_values as primary key.

    Args:
        fk_values: Sample values from potential FK column
        pk_values: Sample values from potential PK column
        min_sample_size: Minimum sample size for reliable inference

    Returns:
        float: Confidence score (0.0 - 1.0), where:
               0.0 = No FK-PK relationship
               0.3 = Weak evidence
               0.6 = Moderate evidence
               0.9 = Strong evidence
    """
    if not fk_values or not pk_values:
        return 0.0

    # Filter out null values
    fk_non_null = [v for v in fk_values if not _is_nullish(v)]
    pk_non_null = [v for v in pk_values if not _is_nullish(v)]

    if len(fk_non_null) < min_sample_size or len(pk_non_null) < min_sample_size:
        return 0.0  # Insufficient sample size

    # Calculate uniqueness ratios
    fk_unique_ratio = len(set(str(v) for v in fk_non_null)) / len(fk_non_null)
    pk_unique_ratio = len(set(str(v) for v in pk_non_null)) / len(pk_non_null)

    # Calculate value overlap (how many FK values exist in PK values)
    fk_set = set(str(v) for v in fk_non_null)
    pk_set = set(str(v) for v in pk_non_null)
    overlap_ratio = len(fk_set & pk_set) / len(fk_set) if fk_set else 0.0

    # FK-PK pattern characteristics:
    # 1. PK should have HIGH uniqueness (close to 1.0)
    # 2. FK can have LOW uniqueness (repeating values OK)
    # 3. FK values should EXIST in PK values (high overlap)

    confidence = 0.0

    # Factor 1: PK uniqueness (0.0 - 0.4 points)
    if pk_unique_ratio > 0.95:
        confidence += 0.4
    elif pk_unique_ratio > 0.85:
        confidence += 0.3
    elif pk_unique_ratio > 0.70:
        confidence += 0.15

    # Factor 2: Value overlap (0.0 - 0.5 points)
    # Strong evidence if most FK values exist in PK
    if overlap_ratio > 0.80:
        confidence += 0.5
    elif overlap_ratio > 0.60:
        confidence += 0.35
    elif overlap_ratio > 0.40:
        confidence += 0.2
    elif overlap_ratio > 0.20:
        confidence += 0.1

    # Factor 3: FK uniqueness pattern (0.0 - 0.1 points)
    # FK having lower uniqueness than PK is expected
    if fk_unique_ratio < pk_unique_ratio:
        uniqueness_diff = pk_unique_ratio - fk_unique_ratio
        if uniqueness_diff > 0.3:
            confidence += 0.1
        elif uniqueness_diff > 0.1:
            confidence += 0.05

    return min(confidence, 1.0)


def _calculate_adaptive_thresholds(
    values_list: List[List[str]],
    table_count: int = 2,
    base_sample_size: int = 10,
) -> Dict[str, float]:
    """
    Calculate adaptive thresholds based on data characteristics.

    Args:
        values_list: List of sample value lists from multiple columns
        table_count: Number of tables in the schema
        base_sample_size: Base sample size from configuration

    Returns:
        Dict with adaptive threshold values
    """
    if not values_list or not any(values_list):
        return {
            "min_sample_size": 50,
            "uniqueness_threshold": 0.95,
            "similarity_threshold": 0.6,
        }

    # Calculate sample statistics
    sample_sizes = [len(vals) for vals in values_list if vals]
    max_sample_size = max(sample_sizes) if sample_sizes else base_sample_size
    avg_sample_size = (
        sum(sample_sizes) / len(sample_sizes) if sample_sizes else base_sample_size
    )

    # Calculate data distribution characteristics
    total_unique_values = 0
    total_values = 0
    skew_ratios = []

    for values in values_list:
        if not values:
            continue

        unique_count = len(set(str(v) for v in values if not _is_nullish(v)))
        total_count = len([v for v in values if not _is_nullish(v)])

        if total_count > 0:
            total_unique_values += unique_count
            total_values += total_count

            # Calculate value distribution skew
            value_counts = {}
            for v in values:
                if not _is_nullish(v):
                    val_str = str(v)
                    value_counts[val_str] = value_counts.get(val_str, 0) + 1

            if len(value_counts) > 1:
                max_freq = max(value_counts.values())
                min_freq = min(value_counts.values())
                skew = max_freq / min_freq if min_freq > 0 else float("inf")
                skew_ratios.append(skew)

    # Calculate overall uniqueness ratio
    overall_uniqueness = total_unique_values / total_values if total_values > 0 else 0
    avg_skew = sum(skew_ratios) / len(skew_ratios) if skew_ratios else 1.0

    # Adaptive threshold calculation
    thresholds = {}

    # 1. Minimum sample size: adjust based on data characteristics
    base_min_size = 50

    # Reduce threshold for highly unique data (likely keys)
    if overall_uniqueness > 0.9:
        min_size_adj = 0.6  # More confident with unique data
    elif overall_uniqueness > 0.7:
        min_size_adj = 0.8
    else:
        min_size_adj = 1.0  # Keep conservative for low uniqueness

    # Adjust for data skew - more skewed data needs larger samples
    if avg_skew > 10:  # Highly skewed
        min_size_adj *= 1.5
    elif avg_skew > 5:  # Moderately skewed
        min_size_adj *= 1.2

    # Adjust for schema complexity - more tables = need more confidence
    if table_count > 10:
        min_size_adj *= 1.3
    elif table_count > 5:
        min_size_adj *= 1.1

    # Scale with base sample size from configuration
    size_scale_factor = (
        min(max_sample_size / base_sample_size, 3.0) if base_sample_size > 0 else 1.0
    )
    min_size_adj *= size_scale_factor

    thresholds["min_sample_size"] = max(int(base_min_size * min_size_adj), 10)

    # 2. Uniqueness threshold: adjust based on data characteristics
    base_uniqueness = 0.95

    # For highly skewed data, be more strict
    if avg_skew > 10:
        uniqueness_adj = 1.02  # Require higher uniqueness
    elif avg_skew > 5:
        uniqueness_adj = 1.01
    else:
        uniqueness_adj = 1.0

    # For small samples, be more conservative
    if avg_sample_size < 20:
        uniqueness_adj *= 1.02

    thresholds["uniqueness_threshold"] = min(base_uniqueness * uniqueness_adj, 0.99)

    # 3. Name similarity threshold: adjust based on naming patterns
    base_similarity = 0.6

    # If we have very structured naming (consistent prefixes/suffixes), be more lenient
    prefix_consistency = _calculate_naming_consistency(values_list)
    if prefix_consistency > 0.8:
        similarity_adj = 0.9  # Lower threshold for consistent naming
    elif prefix_consistency > 0.6:
        similarity_adj = 0.95
    else:
        similarity_adj = 1.0

    thresholds["similarity_threshold"] = max(base_similarity * similarity_adj, 0.4)

    return thresholds


def _calculate_naming_consistency(values_list: List[List[str]]) -> float:
    """
    Calculate consistency in naming patterns across sample values.

    Returns:
        float: Consistency score between 0.0 and 1.0
    """
    if not values_list:
        return 0.0

    # Look for common prefixes and suffixes patterns
    all_values = []
    for values in values_list:
        if values:
            all_values.extend([str(v).upper() for v in values if not _is_nullish(v)])

    if len(all_values) < 2:
        return 0.0

    # Check for common prefixes/suffixes
    prefix_patterns = {}
    suffix_patterns = {}

    for value in all_values:
        # Extract potential prefixes (first 1-3 chars)
        for i in range(1, min(4, len(value))):
            prefix = value[:i]
            prefix_patterns[prefix] = prefix_patterns.get(prefix, 0) + 1

        # Extract potential suffixes (last 1-3 chars)
        for i in range(1, min(4, len(value))):
            suffix = value[-i:]
            suffix_patterns[suffix] = suffix_patterns.get(suffix, 0) + 1

    # Calculate consistency scores
    total_values = len(all_values)
    max_prefix_freq = max(prefix_patterns.values()) if prefix_patterns else 0
    max_suffix_freq = max(suffix_patterns.values()) if suffix_patterns else 0

    prefix_consistency = max_prefix_freq / total_values
    suffix_consistency = max_suffix_freq / total_values

    return max(prefix_consistency, suffix_consistency)


def _infer_cardinality(
    left_values: List[str],
    right_values: List[str],
    left_is_pk: bool,
    right_is_pk: bool,
    left_table: str = "",
    right_table: str = "",
    left_column: str = "",
    right_column: str = "",
    *,
    adaptive_thresholds: Optional[Dict[str, float]] = None,
) -> tuple[str, str]:
    """
    Infer relationship cardinality based on uniqueness ratios with adaptive thresholds.

    Returns:
        tuple: (left_cardinality, right_cardinality) where values can be:
               "*" (many), "1" (one), "?" (zero or one), "+" (one or more)

    Note:
        Uses adaptive thresholds based on data characteristics to improve accuracy.
        Falls back to conservative defaults when data is insufficient or ambiguous.
    """
    # Use adaptive thresholds if provided, otherwise fall back to defaults
    if adaptive_thresholds:
        min_sample_size = int(adaptive_thresholds.get("min_sample_size", 50))
        uniqueness_threshold = adaptive_thresholds.get("uniqueness_threshold", 0.95)
    else:
        min_sample_size = 50
        uniqueness_threshold = 0.95

    # RULE 1: Logical inference based on primary key metadata (highest priority)
    # This is universal and works for ANY schema!

    # Standard FK → PK pattern: many-to-one
    if right_is_pk and not left_is_pk:
        return ("*", "1")

    # Reverse PK → FK pattern: one-to-many
    if left_is_pk and not right_is_pk:
        return ("1", "*")

    # Both are PKs: analyze for potential one-to-one relationship
    # BUT be conservative - 1:1 relationships are rare
    if left_is_pk and right_is_pk:
        # Check if sample data supports 1:1 (need strong evidence)
        if left_values and right_values:
            left_non_null = [v for v in left_values if not _is_nullish(v)]
            right_non_null = [v for v in right_values if not _is_nullish(v)]

            if left_non_null and right_non_null:
                # Check value overlap - should be high for true 1:1
                left_set = set(left_non_null)
                right_set = set(right_non_null)
                overlap = len(left_set.intersection(right_set))

                # High overlap suggests these might be the same entity
                if overlap > max(len(left_set), len(right_set)) * 0.8:
                    return ("1", "1")

        # Default: treat as many-to-one even if both are PKs
        # (safer assumption, prevents false 1:1 relationships)
        return ("*", "1")

    # RULE 2: Use adaptive uniqueness heuristics with sufficient sample size
    if left_values and right_values:
        sample_size = min(len(left_values), len(right_values))

        # Apply adaptive minimum sample size
        if sample_size < min_sample_size:
            # Conservative default: assume many-to-one (most common pattern)
            # This prevents false 1:1 inference from small samples
            return ("*", "1")

        # Large enough sample: trust uniqueness ratios with adaptive threshold
        left_non_null = [v for v in left_values if not _is_nullish(v)]
        right_non_null = [v for v in right_values if not _is_nullish(v)]

        # P2 Fix: Enhanced null and sample quality analysis
        left_null_ratio = 1 - (len(left_non_null) / len(left_values)) if left_values else 0
        right_null_ratio = 1 - (len(right_non_null) / len(right_values)) if right_values else 0

        # If too many nulls, be more conservative
        if left_null_ratio > 0.5 or right_null_ratio > 0.5:
            # High null ratio suggests data quality issues or optional relationships
            return ("*", "1")  # Conservative default

        # Calculate uniqueness with improved handling
        left_unique_ratio = (
            len(set(left_non_null)) / len(left_non_null) if left_non_null else 0
        )
        right_unique_ratio = (
            len(set(right_non_null)) / len(right_non_null) if right_non_null else 0
        )

        # P2 Fix: Dynamic threshold adjustment based on sample size
        # Larger samples need higher uniqueness to be considered "unique"
        adjusted_threshold = uniqueness_threshold
        if sample_size > 100:
            adjusted_threshold = min(0.98, uniqueness_threshold + 0.02)
        elif sample_size > 500:
            adjusted_threshold = min(0.99, uniqueness_threshold + 0.03)

        # Apply adjusted uniqueness threshold
        left_is_unique = left_unique_ratio > adjusted_threshold
        right_is_unique = right_unique_ratio > adjusted_threshold

        # P2 Fix: Additional validation for 1:1 relationships (they are rare)
        if left_is_unique and right_is_unique:
            # Be extra careful with 1:1 - require very high confidence
            if (left_unique_ratio > 0.98 and right_unique_ratio > 0.98 and
                sample_size >= min_sample_size * 2):  # Double the sample size requirement
                return ("1", "1")
            else:
                return ("*", "1")  # Default to safer many-to-one
        elif right_is_unique:
            return ("*", "1")
        elif left_is_unique:
            return ("1", "*")

    # RULE 3: Default to many-to-one (most common case in data warehouses)
    return ("*", "1")


def _is_nullish(value: Any) -> bool:
    """
    Determines whether a sampled value should be treated as NULL/empty.
    Handles common ClickZetta sampling formats where NULL becomes NaN or an empty string.
    """
    if value is None:
        return True
    if isinstance(value, float) and math.isnan(value):
        return True
    text = str(value).strip()
    if not text:
        return True
    normalized = text.upper()
    return normalized in {"NULL", "NONE", "NAN", "NA", "N/A", "NOT AVAILABLE"}


def _column_has_null_via_query(
    session: Session,
    fqn: data_types.FQNParts,
    column_name: str,
    cache: Dict[Tuple[str, str, str, str], bool],
) -> bool:
    """
    Executes a targeted IS NULL query to conclusively detect nullable foreign keys.
    Results are cached per table/column to avoid redundant lookups.
    """
    key = (
        (fqn.database or "").upper(),
        (fqn.schema_name or "").upper(),
        (fqn.table or "").upper(),
        column_name.upper(),
    )
    if key in cache:
        return cache[key]

    if not column_name:
        cache[key] = False
        return False

    qualified_table = _qualified_table_name(fqn)
    column_identifier = _format_sql_identifier(column_name)
    if not column_identifier or not qualified_table:
        cache[key] = False
        return False

    query = (
        f"SELECT {column_identifier} FROM {qualified_table} "
        f"WHERE {column_identifier} IS NULL LIMIT 1"
    )

    try:
        df = session.sql(query).to_pandas()
        has_null = not df.empty
    except Exception as exc:  # pragma: no cover - best-effort diagnostic query
        logger.debug(
            "Strict null check query failed for %s.%s.%s.%s: %s",
            fqn.database,
            fqn.schema_name,
            fqn.table,
            column_name,
            exc,
        )
        has_null = False

    cache[key] = has_null
    return has_null


def _detect_table_role(table_name: str, columns_info: Dict[str, Any]) -> str:
    """
    Detect the semantic role of a table (fact, dimension, bridge, etc.).

    Args:
        table_name: Name of the table
        columns_info: Dictionary containing column metadata

    Returns:
        str: Table role ('fact', 'dimension', 'bridge', 'staging', 'unknown')
    """
    tokens = _identifier_tokens(table_name)

    # Rule 1: Explicit prefixes/suffixes
    fact_indicators = {
        "FACT",
        "FCT",
        "TXN",
        "TRANSACTION",
        "EVENT",
        "LOG",
        "SALES",
        "ORDER",
    }
    dim_indicators = {"DIM", "DIMENSION", "LOOKUP", "REF", "REFERENCE", "MASTER"}
    bridge_indicators = {"BRIDGE", "BRG", "LINK", "JUNCTION", "ASSOC", "ASSOCIATION"}
    staging_indicators = {"STG", "STAGING", "TMP", "TEMP", "WORK", "LANDING", "RAW"}

    # Check for explicit indicators in table name
    for token in tokens:
        if token in fact_indicators:
            return "fact"
        elif token in dim_indicators:
            return "dimension"
        elif token in bridge_indicators:
            return "bridge"
        elif token in staging_indicators:
            return "staging"

    # Rule 2: Analyze column patterns
    pk_count = len(columns_info.get("pk_candidates", {}))
    total_columns = len(columns_info.get("columns", {}))

    if total_columns == 0:
        return "unknown"

    # Count different column types
    measure_like_count = 0
    dimension_like_count = 0
    id_count = 0

    for col_info in columns_info.get("columns", {}).values():
        base_type = col_info.get("base_type", "")
        names = col_info.get("names", [])

        if names:
            col_name = names[0].upper()
            # Count ID/key columns
            if any(suffix in col_name for suffix in ["_ID", "ID", "_KEY", "KEY"]):
                id_count += 1

            # Count measure-like columns (amounts, counts, quantities)
            if any(
                word in col_name
                for word in [
                    "AMOUNT",
                    "QTY",
                    "QUANTITY",
                    "COUNT",
                    "TOTAL",
                    "SUM",
                    "AVG",
                ]
            ):
                measure_like_count += 1
            elif base_type in MEASURE_DATATYPES and not col_info.get(
                "is_identifier", False
            ):
                measure_like_count += 1
            else:
                dimension_like_count += 1

    # Rule 3: Heuristic classification
    id_ratio = id_count / total_columns if total_columns > 0 else 0
    measure_ratio = measure_like_count / total_columns if total_columns > 0 else 0

    # Bridge table: mostly IDs with few other columns
    if id_ratio > 0.6 and total_columns <= 10:
        return "bridge"

    # Fact table: has measures and multiple foreign keys
    if measure_ratio > 0.2 and id_count >= 2:
        return "fact"

    # Dimension table: mostly descriptive attributes, single PK
    if pk_count == 1 and measure_ratio < 0.1:
        return "dimension"

    # Default to unknown if no clear pattern
    return "unknown"


def _get_business_relationship_context(
    left_table: str, right_table: str, left_role: str, right_role: str
) -> str:
    """
    Determine business relationship context between tables based on their roles.

    Returns:
        str: Context hint for join type inference
    """
    # Fact to Dimension relationships
    if left_role == "fact" and right_role == "dimension":
        return "fact_to_dimension"
    elif left_role == "dimension" and right_role == "fact":
        return "dimension_to_fact"

    # Bridge table relationships
    elif left_role == "bridge" or right_role == "bridge":
        return "bridge_relationship"

    # Dimension to Dimension (hierarchy or lookup)
    elif left_role == "dimension" and right_role == "dimension":
        return "dimension_hierarchy"

    # Fact to Fact (rare but possible)
    elif left_role == "fact" and right_role == "fact":
        return "fact_to_fact"

    # Staging relationships
    elif "staging" in [left_role, right_role]:
        return "staging_relationship"

    return "unknown_relationship"


def _infer_join_type(
    left_table: str,
    right_table: str,
    left_card: str,
    right_card: str,
    left_is_pk: bool,
    right_is_pk: bool,
    left_values: List[str],
    right_values: List[str],
    *,
    has_null_fk: bool = False,
    left_table_meta: Optional[Dict[str, Any]] = None,
    right_table_meta: Optional[Dict[str, Any]] = None,
) -> int:
    """
    Enhanced JOIN type inference based on table roles and business semantics.

    Args:
        left_table: Name of the left table
        right_table: Name of the right table
        left_card: Left cardinality ("1", "*", etc.)
        right_card: Right cardinality ("1", "*", etc.)
        left_is_pk: Whether left column is a primary key
        right_is_pk: Whether right column is a primary key
        left_values: Sample values from left column
        right_values: Sample values from right column
        has_null_fk: Whether strict null detection found NULLs
        left_table_meta: Metadata about the left table
        right_table_meta: Metadata about the right table

    Returns:
        semantic_model_pb2.JoinType value (inner or left_outer)

    Enhanced Decision Logic:
        1. Strict null detection (highest priority)
        2. Business semantic rules based on table roles
        3. NULL detection in sample values
        4. Naming pattern heuristics
        5. Conservative INNER JOIN default
    """

    # RULE 1: Default to INNER JOIN (most common and safest)
    default_join = semantic_model_pb2.JoinType.inner

    # RULE 2: Strict mode override - explicit null detection via SQL (highest priority)
    if has_null_fk:
        reason = "strict null probe"
        logger.debug(
            f"Join type inference for {left_table} -> {right_table}: "
            f"LEFT_OUTER ({reason})"
        )
        return semantic_model_pb2.JoinType.left_outer

    # RULE 3: Enhanced business semantics based on table roles
    left_role = "unknown"
    right_role = "unknown"
    relationship_context = "unknown_relationship"

    if left_table_meta and right_table_meta:
        left_role = _detect_table_role(left_table, left_table_meta)
        right_role = _detect_table_role(right_table, right_table_meta)
        relationship_context = _get_business_relationship_context(
            left_table, right_table, left_role, right_role
        )

        # Apply business rules based on relationship context
        if relationship_context == "fact_to_dimension":
            # Fact → Dimension: usually INNER, but check for optional dimensions
            if any(
                keyword in right_table.upper()
                for keyword in [
                    "PROMO",
                    "PROMOTION",
                    "DISCOUNT",
                    "COUPON",
                    "OPTIONAL",
                    "SECONDARY",
                ]
            ):
                logger.debug(
                    f"Join type inference for {left_table} -> {right_table}: "
                    f"LEFT_OUTER (fact to optional dimension: {right_role})"
                )
                return semantic_model_pb2.JoinType.left_outer

        elif relationship_context == "dimension_hierarchy":
            # Dimension → Dimension: often optional (parent-child, category hierarchies)
            if left_card in ("*", "+") and right_card == "1":
                logger.debug(
                    f"Join type inference for {left_table} -> {right_table}: "
                    f"LEFT_OUTER (dimension hierarchy: child to parent)"
                )
                return semantic_model_pb2.JoinType.left_outer

        elif relationship_context == "bridge_relationship":
            # Bridge tables: typically INNER to maintain referential integrity
            logger.debug(
                f"Join type inference for {left_table} -> {right_table}: "
                f"INNER (bridge table relationship)"
            )
            return semantic_model_pb2.JoinType.inner

        elif relationship_context == "staging_relationship":
            # Staging tables: often have incomplete data, prefer LEFT OUTER
            logger.debug(
                f"Join type inference for {left_table} -> {right_table}: "
                f"LEFT_OUTER (staging table relationship)"
            )
            return semantic_model_pb2.JoinType.left_outer

    # RULE 4: Check for NULL values in foreign key (left side) using samples
    if left_values and left_card in ("*", "+"):
        sample_window = left_values[: min(len(left_values), 25)]
        has_nulls = any(_is_nullish(val) for val in sample_window)
        if has_nulls:
            logger.debug(
                f"Join type inference for {left_table} -> {right_table}: "
                f"LEFT_OUTER (detected NULL values in FK column)"
            )
            return semantic_model_pb2.JoinType.left_outer

    # RULE 5: Naming pattern heuristics for optional relationships
    right_upper = right_table.upper()
    optional_keywords = {
        "OPTIONAL",
        "ALTERNATE",
        "SECONDARY",
        "BACKUP",
        "FALLBACK",
        "PROMO",
        "PROMOTION",
        "DISCOUNT",
        "COUPON",
        "TEMP",
        "TMP",
    }

    for keyword in optional_keywords:
        if keyword in right_upper:
            logger.debug(
                f"Join type inference for {left_table} -> {right_table}: "
                f"LEFT_OUTER (optional relationship pattern: {keyword})"
            )
            return semantic_model_pb2.JoinType.left_outer

    # RULE 6: Time dimension special case - often required
    if any(time_hint in right_upper for time_hint in ["DATE", "TIME", "CALENDAR"]):
        logger.debug(
            f"Join type inference for {left_table} -> {right_table}: "
            f"INNER (time dimension - typically required)"
        )
        return semantic_model_pb2.JoinType.inner

    # RULE 7: Default to INNER JOIN
    # Most conservative choice that ensures data integrity
    logger.debug(
        f"Join type inference for {left_table} -> {right_table}: "
        f"INNER (default - roles: {left_role}->{right_role}, context: {relationship_context})"
    )
    return default_join


def _is_valid_shared_column_relationship(fk_column: str, pk_column: str, pk_table: str, fk_table: str) -> bool:
    """
    Pure column-pattern-based validation for shared column relationships.
    Removes table name dependencies to work with ANY table names.

    Args:
        fk_column: Foreign key column name
        pk_column: Primary key column name
        pk_table: Table containing the primary key (used for equality check only)
        fk_table: Table containing the foreign key (used for equality check only)

    Returns:
        True if this is a valid relationship based on column patterns

    P0 Fix: Pure column-name matching without hardcoded table validation
    """
    # For shared column relationships, column names should be identical
    if fk_column.upper() != pk_column.upper():
        return False

    # Don't create relationships between identical table names
    if pk_table.upper() == fk_table.upper():
        return False

    # Special case: NEVER allow generic "id" shared column relationships
    if pk_column.upper() == "ID":
        # "id" is too generic for shared column relationships
        # Require explicit FK patterns like {entity}_id instead
        return False

    # For non-"id" columns with identical names, validate column meaningfulness
    column_upper = pk_column.upper()

    # Reject generic column names that shouldn't form relationships
    generic_columns = _GENERIC_IDENTIFIER_TOKENS | {
        "TYPE", "STATUS", "STATE", "ACTIVE", "ENABLED", "DELETED",
        "CREATED", "UPDATED", "MODIFIED", "VERSION", "LEVEL"
    }

    if column_upper in generic_columns:
        return False

    # For meaningful specific column names, allow the relationship
    # Examples: account_number, product_sku, order_reference, etc.
    return len(column_upper) >= 3  # Require minimum meaningful length


def _are_tables_semantically_related(table_a: str, table_b: str) -> bool:
    """
    Pattern-based semantic relationship detection without hardcoded table names.

    P0 Fix: Removes hardcoded table pairs to work with ANY table names.
    Uses naming patterns to infer relationships instead.
    """
    # Convert to upper for comparison
    table_a_upper = table_a.upper()
    table_b_upper = table_b.upper()

    # Don't relate identical tables
    if table_a_upper == table_b_upper:
        return False

    # Pattern 1: Hierarchical relationships (parent/child naming)
    hierarchical_patterns = [
        ("PARENT", "CHILD"), ("MASTER", "DETAIL"), ("MAIN", "SUB"),
        ("PRIMARY", "SECONDARY"), ("BASE", "DERIVED"), ("HEAD", "LINE")
    ]

    for parent_pattern, child_pattern in hierarchical_patterns:
        if (parent_pattern in table_a_upper and child_pattern in table_b_upper) or \
           (child_pattern in table_a_upper and parent_pattern in table_b_upper):
            return True

    # Pattern 2: Entity-Detail relationships (table_name + table_name_items/details)
    if table_a_upper.endswith("_ITEMS") or table_a_upper.endswith("_DETAILS"):
        base_name = table_a_upper.replace("_ITEMS", "").replace("_DETAILS", "")
        if base_name == table_b_upper:
            return True

    if table_b_upper.endswith("_ITEMS") or table_b_upper.endswith("_DETAILS"):
        base_name = table_b_upper.replace("_ITEMS", "").replace("_DETAILS", "")
        if base_name == table_a_upper:
            return True

    # Pattern 3: Dimensional relationships (dim_ prefix patterns)
    if table_a_upper.startswith("DIM_") and not table_b_upper.startswith("DIM_"):
        return True
    if table_b_upper.startswith("DIM_") and not table_a_upper.startswith("DIM_"):
        return True

    # Pattern 4: Fact-Dimension relationships (fact_ + dim_ patterns)
    if table_a_upper.startswith("FACT_") and table_b_upper.startswith("DIM_"):
        return True
    if table_b_upper.startswith("FACT_") and table_a_upper.startswith("DIM_"):
        return True

    # Default: Consider tables potentially related to allow FK discovery
    # Let column-based matching be the primary filter
    return True


def _is_valid_suffix_match(fk_column: str, pk_column: str, pk_table: str) -> bool:
    """
    Enhanced column-name-based validation that supports multiple naming conventions.

    Args:
        fk_column: Foreign key column name (normalized)
        pk_column: Primary key column name (normalized)
        pk_table: Primary key table name (used for advanced matching)

    Returns:
        True if this appears to be a valid FK->PK relationship based on column patterns

    Universal Rules (applicable to any schema):
    1. For pk_column == "id": Only allow well-structured FK columns with meaningful prefixes
    2. For other pk_columns: Apply pattern-based validation
    3. Universal support: Handle prefix-based naming (table_entity_key patterns)
    4. Focus on column naming patterns while leveraging table context when helpful
    """
    # Normalize inputs
    fk_column = fk_column.strip().upper()
    pk_column = pk_column.strip().upper()
    pk_table = pk_table.strip().upper()

    if len(pk_column) < 2:
        return False

    # Special handling for "id" primary keys (most restrictive)
    if pk_column == "ID":
        # Reject bare "id" to "id" matches (too generic)
        if fk_column == "ID":
            return False

        # Only allow if FK column has underscore structure and meaningful prefix
        if not fk_column.endswith("_ID"):
            return False

        prefix = fk_column[:-3]  # Remove "_ID" suffix
        if len(prefix) < 2:  # Require meaningful prefix length
            return False

        # Reject generic prefixes that don't indicate relationships
        generic_prefixes = {"PARENT", "CHILD", "REF", "REFERENCE", "FK", "FOREIGN"}
        if prefix in generic_prefixes:
            return False

        # Allow if prefix appears to be a meaningful entity name
        return True

    # For non-"id" columns, use direct pattern matching
    if fk_column == pk_column:
        return True  # Exact match is always valid

    # Universal support: Handle prefix-based naming conventions
    # Example: l_partkey (FK) should match p_partkey (PK) when pk_table is PART
    if "_" in fk_column and "_" in pk_column:
        # Extract the core identifiers (remove table prefixes)
        fk_parts = fk_column.split("_", 1)
        pk_parts = pk_column.split("_", 1)

        if len(fk_parts) == 2 and len(pk_parts) == 2:
            fk_core = fk_parts[1]  # PARTKEY from L_PARTKEY
            pk_core = pk_parts[1]  # PARTKEY from P_PARTKEY

            # Core identifiers match (e.g., PARTKEY == PARTKEY)
            if fk_core == pk_core:
                return True

            # Handle entity-based cores (e.g., CUSTKEY and table CUSTOMER)
            # Check if pk_core relates to pk_table semantically
            if _is_entity_key_match(pk_core, pk_table):
                # Also check if fk_core could reference the same entity
                if _is_entity_key_match(fk_core, pk_table):
                    return True

    # For structured FK columns, check if they reference the PK column
    if "_" in fk_column:
        # Pattern: {entity}_{pk_column}
        if fk_column.endswith(f"_{pk_column}"):
            prefix = fk_column[:-(len(pk_column) + 1)]
            return len(prefix) >= 2  # Meaningful prefix required

        # Pattern: {pk_column}_{suffix}
        if fk_column.startswith(f"{pk_column}_"):
            suffix = fk_column[len(pk_column) + 1:]
            return len(suffix) >= 1  # Some suffix required

    # Allow if FK contains PK column as a component (for compound names)
    if pk_column in fk_column and pk_column != fk_column:
        return len(pk_column) >= 3  # Avoid very short component matches

    return False


def _is_entity_key_match(key_column: str, table_name: str) -> bool:
    """
    Check if a key column semantically relates to a table entity.

    Examples:
    - CUSTKEY relates to CUSTOMER table
    - PARTKEY relates to PART table
    - ORDERKEY relates to ORDERS table
    """
    key_column = key_column.upper()
    table_name = table_name.upper()

    # Direct matches
    if key_column.endswith("KEY"):
        entity_part = key_column[:-3]  # Remove "KEY" suffix

        # Check various entity name patterns
        entity_patterns = [
            entity_part,
            entity_part + "S",  # Plural
            entity_part[:-1] if entity_part.endswith("S") else None,  # Singular
        ]

        for pattern in entity_patterns:
            if pattern and (pattern == table_name or pattern in table_name or table_name in pattern):
                return True

    return False


def _looks_like_foreign_key(fk_table: str, pk_table: str, fk_column: str) -> bool:
    """
    Universal column-pattern-based heuristic to detect foreign key columns.
    Supports multiple naming conventions including prefix-based patterns.

    Args:
        fk_table: Foreign key table name (used for context)
        pk_table: Primary key table name (used for context)
        fk_column: Foreign key column name

    Returns:
        True if the column pattern suggests it's a foreign key

    Universal: Supports both standard patterns and prefix-based naming
    """
    fk_upper = fk_column.strip().upper()
    fk_table_upper = fk_table.strip().upper()
    pk_table_upper = pk_table.strip().upper()

    # Pattern 1: Standard FK patterns {entity}_id, {entity}_key
    if fk_upper.endswith("_ID") or fk_upper.endswith("_KEY"):
        prefix = fk_upper[:-3] if fk_upper.endswith("_ID") else fk_upper[:-4]
        if len(prefix) >= 2:  # Meaningful prefix required
            # Reject generic prefixes that don't indicate relationships
            generic_prefixes = {"PARENT", "CHILD", "REF", "REFERENCE", "FK", "FOREIGN", "MAIN", "SUPER"}
            if prefix not in generic_prefixes:
                if _column_mentions_table(fk_column, pk_table):
                    return True

    # Pattern 2: Universal prefix-based naming
    # Examples: o_custkey (ORDERS -> CUSTOMER), l_partkey (LINEITEM -> PART)
    if "_" in fk_upper:
        parts = fk_upper.split("_", 1)
        if len(parts) == 2:
            fk_prefix, fk_core = parts

            # Check if the FK core represents a meaningful entity reference
            if fk_core.endswith("KEY") and len(fk_core) > 3:
                # Check if this could be referencing the pk_table entity
                if _is_entity_key_match(fk_core, pk_table_upper):
                    return True

    # Pattern 3: Compound FK columns with entity tokens
    tokens = _identifier_tokens(fk_column)
    if len(tokens) >= 2:
        tail = tokens[-1]
        if tail in {"ID", "KEY"}:
            # Check if any token before the tail suggests an entity name
            for i in range(len(tokens) - 1):
                token = tokens[i]
                # Accept meaningful entity-like tokens (non-generic)
                if (
                    len(token) >= 2
                    and token not in _GENERIC_IDENTIFIER_TOKENS
                    and _column_mentions_table(fk_column, pk_table)
                ):
                    return True

    # Pattern 4: Concatenated forms like customerid, orderkey
    if fk_upper.endswith("ID") and len(fk_upper) > 3:
        prefix = fk_upper[:-2]
        if (
            len(prefix) >= 3
            and prefix not in _GENERIC_IDENTIFIER_TOKENS
            and _column_mentions_table(fk_column, pk_table)
        ):
            return True

    if fk_upper.endswith("KEY") and len(fk_upper) > 4:
        prefix = fk_upper[:-3]
        if (
            len(prefix) >= 3
            and prefix not in _GENERIC_IDENTIFIER_TOKENS
            and _column_mentions_table(fk_column, pk_table)
        ):
            return True

    # Pattern 5: Allow meaningful descriptive columns
    meaningful_patterns = {
        "CREATED_BY", "UPDATED_BY", "MODIFIED_BY", "ASSIGNED_TO",
        "OWNED_BY", "MANAGED_BY", "APPROVED_BY", "REQUESTED_BY"
    }
    if fk_upper in meaningful_patterns:
        return True

    return False


def _suggest_filters(
    raw_table: data_types.Table,
) -> List[semantic_model_pb2.NamedFilter]:
    suggestions: List[semantic_model_pb2.NamedFilter] = []
    for col in raw_table.columns:
        base_type = _base_type_from_type(col.column_type)
        values = col.values or []
        distinct_values: List[str] = []
        for value in values:
            if value not in distinct_values:
                distinct_values.append(value)
        if _is_time_like_column(col):
            expr = f"{col.column_name} >= DATEADD('day', -30, CURRENT_DATE())"
            suggestions.append(
                semantic_model_pb2.NamedFilter(
                    name=f"{col.column_name}_last_30_days",
                    synonyms=[_PLACEHOLDER_COMMENT],
                    description=_PLACEHOLDER_COMMENT,
                    expr=expr,
                )
            )
        if 1 < len(distinct_values) <= 5:
            upper_name = col.column_name.upper()
            is_identifier_like = upper_name.endswith(("ID", "_ID", "KEY", "_KEY"))

            categorical_suffixes = (
                "STATUS",
                "FLAG",
                "TYPE",
                "PRIORITY",
                "SEGMENT",
                "CATEGORY",
                "MODE",
                "CODE",
                "LEVEL",
            )
            is_textual = base_type in {"STRING", "TEXT", "VARCHAR", "CHAR", "CHARACTER"}
            is_boolean = base_type in {"BOOLEAN"}
            is_categorical_numeric = base_type in {
                "INT",
                "INTEGER",
                "NUMBER",
                "SMALLINT",
                "BIGINT",
            } and any(upper_name.endswith(suffix) for suffix in categorical_suffixes)

            if not is_identifier_like and (
                is_textual or is_boolean or is_categorical_numeric
            ):
                formatted = [
                    _format_literal(val, base_type) for val in distinct_values[:5]
                ]
                expr = f"{col.column_name} IN ({', '.join(formatted)})"
                suggestions.append(
                    semantic_model_pb2.NamedFilter(
                        name=f"{col.column_name}_include_values",
                        synonyms=[_PLACEHOLDER_COMMENT],
                        description=_PLACEHOLDER_COMMENT,
                        expr=expr,
                    )
                )
    return suggestions


def _infer_relationships(
    raw_tables: List[tuple[data_types.FQNParts, data_types.Table]],
    *,
    session: Optional[Session] = None,
    strict_join_inference: bool = False,
    status: Optional[Dict[str, bool]] = None,
    max_relationships: Optional[int] = None,
    min_confidence: float = 0.2,
    timeout_seconds: Optional[float] = None,
) -> List[semantic_model_pb2.Relationship]:
    status_dict = status if status is not None else {}
    if "limited_by_timeout" not in status_dict:
        status_dict["limited_by_timeout"] = False
    if "limited_by_max_relationships" not in status_dict:
        status_dict["limited_by_max_relationships"] = False

    relationships: List[semantic_model_pb2.Relationship] = []
    if not raw_tables:
        return relationships

    start_time = time.perf_counter()
    min_confidence = max(0.0, min(min_confidence, 1.0))
    limit_reached = False

    def _timed_out() -> bool:
        return (
            timeout_seconds is not None
            and (time.perf_counter() - start_time) >= timeout_seconds
        )

    metadata = {}
    prefix_counter: Dict[str, int] = {}
    for _, raw_table in raw_tables:
        for column in raw_table.columns:
            tokens = _identifier_tokens(column.column_name)
            if tokens:
                prefix = tokens[0]
                prefix_counter[prefix] = prefix_counter.get(prefix, 0) + 1
    global_prefixes = {
        prefix
        for prefix, count in prefix_counter.items()
        if len(prefix) <= 3 and count >= 2
    }

    metadata = {}
    for fqn, raw_table in raw_tables:
        columns_meta: Dict[str, Dict[str, Any]] = {}
        pk_candidates: Dict[str, List[str]] = defaultdict(list)
        table_prefixes = global_prefixes | _table_prefixes(raw_table.name)
        has_explicit_pk = any(getattr(column, "is_primary_key", False) for column in raw_table.columns)

        for column in raw_table.columns:
            base_type = _base_type_from_type(column.column_type)
            normalized = _sanitize_identifier_name(
                column.column_name, prefixes_to_drop=table_prefixes
            )

            # CRITICAL FIX: If normalization results in empty string, use original column name
            # This prevents column collisions when multiple columns normalize to ''
            if not normalized:
                normalized = column.column_name.upper()
            entry = columns_meta.setdefault(
                normalized,
                {
                    "names": [],
                    "base_type": base_type,
                    "values": [],
                    "is_identifier": False,
                    "is_primary": False,
                },
            )
            entry["base_type"] = base_type
            entry["names"].append(column.column_name)
            if column.values:
                entry["values"].extend(column.values)
            entry["is_identifier"] = entry["is_identifier"] or _is_identifier_like(
                column.column_name, base_type
            )
            is_primary = getattr(column, "is_primary_key", False)
            if is_primary:
                entry["is_primary"] = True
                if column.column_name not in pk_candidates[normalized]:
                    pk_candidates[normalized].append(column.column_name)
                continue
            if (
                not has_explicit_pk
                and _looks_like_primary_key(raw_table.name, column.column_name)
            ):
                pk_candidates[normalized].append(column.column_name)

            # ENHANCEMENT: Infer PK from sample data when column naming is poor
            # This allows PK detection for columns like 'uid', 'oid', 'pid' etc.
            # IMPORTANT: Only apply when column name suggests it could be a key (contains id/key/num)
            # CRITICAL: Pass table_name to help distinguish PK from FK patterns
            if (
                not has_explicit_pk
                and column.column_name not in pk_candidates[normalized]
                and column.values
                and _could_be_identifier_column(column.column_name, base_type, raw_table.name)
            ):
                # Check if sample data shows high uniqueness (PK pattern)
                if _infer_pk_from_sample_data(column.values, min_uniqueness=0.95, min_sample_size=20):
                    pk_candidates[normalized].append(column.column_name)
                    # Mark it as inferred from data
                    entry["pk_inferred_from_data"] = True

        metadata[raw_table.name] = {
            "fqn": fqn,
            "columns": columns_meta,
            "pk_candidates": pk_candidates,
        }

    pairs: dict[tuple[str, str], List[tuple[str, str]]] = {}
    null_check_cache: Dict[Tuple[str, str, str, str], bool] = {}

    def _record_pair(
        left_table: str, right_table: str, left_col: str, right_col: str
    ) -> None:
        nonlocal limit_reached
        if limit_reached:
            return
        if _timed_out():
            status_dict["limited_by_timeout"] = True
            limit_reached = True
            return

        key = (left_table, right_table)
        value = (left_col, right_col)
        bucket = pairs.setdefault(key, [])
        if value not in bucket:
            bucket.append(value)
            if (
                max_relationships is not None
                and len(pairs) >= max_relationships
            ):
                status_dict["limited_by_max_relationships"] = True
                limit_reached = True

    # Use optimized relationship candidate generation instead of nested loops
    _generate_optimized_relationship_candidates(metadata, _record_pair, status_dict, _timed_out)

    # Calculate global adaptive thresholds for the entire schema (moved from below)
    all_schema_values = []
    for table_meta in metadata.values():
        for col_meta in table_meta["columns"].values():
            if col_meta.get("values"):
                all_schema_values.append(col_meta["values"])
    global_adaptive_thresholds = _calculate_adaptive_thresholds(
        all_schema_values,
        table_count=len(raw_tables),
        base_sample_size=10,  # Default base
    )

    # DISABLED: Shared column logic and old bidirectional matching
    # The optimized _generate_optimized_relationship_candidates() function
    # already handles all FK-PK matching with proper semantic validation.
    # Skipping the old logic to avoid duplicate and incorrect relationships.

    # All candidate generation is now done by _generate_optimized_relationship_candidates()
    # which was called earlier. We jump directly to "Build relationships with inferred cardinality"

    if False:  # Old shared column logic (permanently disabled)
        for table_a_name, table_a in metadata.items():
            for table_b_name, table_b in metadata.items():
                if table_a_name == table_b_name:
                    continue

            shared = set(table_a["columns"].keys()) & set(table_b["columns"].keys())
            for col_key in shared:
                meta_a = table_a["columns"][col_key]
                meta_b = table_b["columns"][col_key]
                if meta_a["base_type"] != meta_b["base_type"]:
                    continue
                in_pk_a = col_key in table_a["pk_candidates"]
                in_pk_b = col_key in table_b["pk_candidates"]

                # Apply semantic validation for shared column relationships
                # Only create relationships when there's clear PK-FK semantics
                if in_pk_a and not in_pk_b:
                    if _is_valid_shared_column_relationship(
                        col_key, col_key, table_a_name, table_b_name
                    ):
                        _record_pair(
                            table_b_name,
                            table_a_name,
                            meta_b["names"][0],
                            meta_a["names"][0],
                        )
                elif in_pk_b and not in_pk_a:
                    if _is_valid_shared_column_relationship(
                        col_key, col_key, table_b_name, table_a_name
                    ):
                        _record_pair(
                            table_a_name,
                            table_b_name,
                            meta_a["names"][0],
                            meta_b["names"][0],
                        )
                elif in_pk_a and in_pk_b:
                    pk_count_a = len(table_a["pk_candidates"])
                    pk_count_b = len(table_b["pk_candidates"])
                    if pk_count_a >= 2 and pk_count_b == 1:
                        _record_pair(
                            table_a_name,
                            table_b_name,
                            meta_a["names"][0],
                            meta_b["names"][0],
                        )
                    elif pk_count_b >= 2 and pk_count_a == 1:
                        _record_pair(
                            table_b_name,
                            table_a_name,
                            meta_b["names"][0],
                            meta_a["names"][0],
                        )

            # Enhanced suffix-based matches with name similarity
            # Pattern 1: Direct suffix match (e.g. order_date_id -> date_id)
            # Collect all candidate matches and select the best one based on similarity score
            for pk_norm, pk_cols in table_a["pk_candidates"].items():
                pk_meta = table_a["columns"].get(pk_norm)
                if not pk_meta:
                    continue

                # Collect all candidate matches for this PK with their scores
                # Tuple format: (score, similarity, is_single_pk, norm_b, meta_b)
                candidates: List[tuple[float, float, bool, str, Dict[str, Any]]] = []

                # Count primary keys in table_a for tiebreaking
                pk_count_a = len(table_a.get("pk_candidates", {}))
                is_single_pk_a = (pk_count_a == 1)

                for norm_b, meta_b in table_b["columns"].items():
                    if meta_b["base_type"] != pk_meta["base_type"]:
                        continue
                    if norm_b == pk_norm:
                        continue

                    # Calculate name similarity for ranking
                    similarity = _name_similarity(norm_b, pk_norm)

                    # Direct suffix match with improved logic to avoid over-matching
                    if norm_b.endswith(pk_norm) and _is_valid_suffix_match(
                        norm_b, pk_norm, table_a_name
                    ):
                        # Higher score for suffix matches, but still use similarity for ranking
                        # Suffix match gets a bonus, but exact/close name match should still win
                        score = similarity + 0.3  # Bonus for suffix match
                        candidates.append((score, similarity, is_single_pk_a, norm_b, meta_b))
                        # Don't continue - check if there are better matches

                    # Enhanced: Check if column looks like a foreign key to this table
                    elif _looks_like_foreign_key(
                        table_b_name, table_a_name, meta_b["names"][0]
                    ):
                        # CRITICAL FIX: When _looks_like_foreign_key confirms it's a FK,
                        # don't apply similarity threshold - the pattern match is enough evidence
                        # Use similarity as the score for FK pattern matches
                        candidates.append((similarity, similarity, is_single_pk_a, norm_b, meta_b))

                # CRITICAL FIX: Record ALL valid candidates, not just the best one
                # A primary key can be referenced by multiple foreign keys
                # (e.g., PART.P_PARTKEY is referenced by both PARTSUPP.PS_PARTKEY and LINEITEM.L_PARTKEY)
                if candidates:
                    # Sort by: 1) score (desc), 2) similarity (desc), 3) is_single_pk (desc)
                    # This prefers single-column PKs over composite PKs when scores are equal
                    candidates.sort(key=lambda x: (x[0], x[1], x[2]), reverse=True)

                    # Record all candidates that meet minimum quality threshold
                    # Use a lower threshold to avoid missing valid relationships
                    min_quality_threshold = 0.3

                    for score, similarity, is_single_pk, norm_b, meta_b in candidates:
                        # Only record if similarity meets minimum threshold
                        if similarity >= min_quality_threshold:
                            # DEBUG: Log candidate selection for LINEITEM FK columns
                            if table_b_name.upper() == "LINEITEM" and meta_b["names"][0].upper() in ["L_PARTKEY", "L_SUPPKEY"]:
                                print(f"\n  🔍 DEBUG: LINEITEM FK candidate recorded")
                                print(f"     FK column: {meta_b['names'][0].upper()}")
                                print(f"     Target table: {table_a_name}")
                                print(f"     Target PK: {pk_cols[0].upper()}")
                                print(f"     Score: {score:.3f}, Similarity: {similarity:.3f}, Single PK: {is_single_pk}")

                            _record_pair(
                                table_b_name,
                                table_a_name,
                                meta_b["names"][0].upper(),
                                pk_cols[0].upper(),
                            )

            # Pattern 2: Reverse direction
            # Collect all candidate matches and select the best one based on similarity score
            for pk_norm, pk_cols in table_b["pk_candidates"].items():
                pk_meta = table_b["columns"].get(pk_norm)
                if not pk_meta:
                    continue

                # Collect all candidate matches for this PK with their scores
                # Tuple format: (score, similarity, is_single_pk, norm_a, meta_a)
                candidates = []

                # Count primary keys in table_b for tiebreaking
                pk_count_b = len(table_b.get("pk_candidates", {}))
                is_single_pk_b = (pk_count_b == 1)

                for norm_a, meta_a in table_a["columns"].items():
                    if meta_a["base_type"] != pk_meta["base_type"]:
                        continue
                    if norm_a == pk_norm:
                        continue

                    # Calculate name similarity for ranking
                    similarity = _name_similarity(norm_a, pk_norm)

                    # Direct suffix match
                    if norm_a.endswith(pk_norm) and _is_valid_suffix_match(
                        norm_a, pk_norm, table_b_name
                    ):
                        # Higher score for suffix matches, but still use similarity for ranking
                        # Suffix match gets a bonus, but exact/close name match should still win
                        score = similarity + 0.3  # Bonus for suffix match
                        candidates.append((score, similarity, is_single_pk_b, norm_a, meta_a))
                        # Don't continue - check if there are better matches

                    # Enhanced: Check if column looks like a foreign key to this table
                    elif _looks_like_foreign_key(
                        table_a_name, table_b_name, meta_a["names"][0]
                    ):
                        # CRITICAL FIX: When _looks_like_foreign_key confirms it's a FK,
                        # don't apply similarity threshold - the pattern match is enough evidence
                        # Use similarity as the score for FK pattern matches
                        candidates.append((similarity, similarity, is_single_pk_b, norm_a, meta_a))

                # CRITICAL FIX: Record ALL valid candidates, not just the best one
                # A primary key can be referenced by multiple foreign keys
                if candidates:
                    # Sort by: 1) score (desc), 2) similarity (desc), 3) is_single_pk (desc)
                    # This prefers single-column PKs over composite PKs when scores are equal
                    candidates.sort(key=lambda x: (x[0], x[1], x[2]), reverse=True)

                    # Record all candidates that meet minimum quality threshold
                    min_quality_threshold = 0.3

                    for score, similarity, is_single_pk, norm_a, meta_a in candidates:
                        # Only record if similarity meets minimum threshold
                        if similarity >= min_quality_threshold:
                            # DEBUG: Log candidate selection for LINEITEM FK columns
                            if table_a_name.upper() == "LINEITEM" and meta_a["names"][0].upper() in ["L_PARTKEY", "L_SUPPKEY"]:
                                print(f"\n  🔍 DEBUG: LINEITEM FK candidate recorded (reverse)")
                                print(f"     FK column: {meta_a['names'][0].upper()}")
                                print(f"     Target table: {table_b_name}")
                                print(f"     Target PK: {pk_cols[0].upper()}")
                                print(f"     Score: {score:.3f}, Similarity: {similarity:.3f}, Single PK: {is_single_pk}")

                            _record_pair(
                                table_a_name,
                                table_b_name,
                                meta_a["names"][0].upper(),
                                pk_cols[0].upper(),
                            )

    # Calculate global adaptive thresholds for the entire schema
    all_schema_values = []
    for table_meta in metadata.values():
        for col_meta in table_meta["columns"].values():
            if col_meta.get("values"):
                all_schema_values.append(col_meta["values"])

    global_adaptive_thresholds = _calculate_adaptive_thresholds(
        all_schema_values,
        table_count=len(raw_tables),
        base_sample_size=10,  # Default base
    )

    def _pk_name_set(table_meta: Dict[str, Any]) -> set[str]:
        pk_names: set[str] = set()
        for pk_list in table_meta.get("pk_candidates", {}).values():
            for name in pk_list:
                pk_names.add(name.upper())
        return pk_names

    emitted_relationship_signatures: set[tuple[str, str, tuple[tuple[str, str], ...]]] = set()
    domain_patterns_cache = _get_domain_knowledge_patterns()

    # CRITICAL FIX: Deduplicate bidirectional relationships
    # Keep the correct direction (FK -> PK) to avoid duplicates like ORDERS<->LINEITEM
    print(f"\n🔧 Deduplicating relationships: {len(pairs)} candidate pairs before dedup")
    for i, ((lt, rt), cols) in enumerate(pairs.items(), 1):
        col_str = ", ".join(f"{l}→{r}" for l, r in cols)
        print(f"  {i}. {lt} → {rt}: [{col_str}]")
    deduplicated_pairs = {}
    for (left_table, right_table), column_pairs in pairs.items():
        # Create a canonical key (sorted table names)
        canonical_key = tuple(sorted([left_table, right_table]))
        reverse_key = (right_table, left_table)

        # If we already have the reverse direction, decide which one to keep
        if reverse_key in deduplicated_pairs:
            # Get metadata to determine the correct direction
            left_meta = metadata.get(left_table, {})
            right_meta = metadata.get(right_table, {})

            # Extract the columns involved in the relationship
            # column_pairs is a list of tuples: [(left_col, right_col), ...]
            current_pairs_list = list(column_pairs) if column_pairs else []
            reverse_pairs_list = list(deduplicated_pairs[reverse_key]) if deduplicated_pairs[reverse_key] else []

            # For simplicity, check the first pair (most relationships have only one pair)
            current_left_col = current_pairs_list[0][0] if current_pairs_list else None
            current_right_col = current_pairs_list[0][1] if current_pairs_list else None
            reverse_left_col = reverse_pairs_list[0][0] if reverse_pairs_list else None
            reverse_right_col = reverse_pairs_list[0][1] if reverse_pairs_list else None

            # Check which direction is FK -> PK
            # Current direction: left_table.current_left_col -> right_table.current_right_col
            # Reverse direction: right_table.reverse_left_col -> left_table.reverse_right_col

            left_pk_set = _pk_name_set(left_meta)
            right_pk_set = _pk_name_set(right_meta)

            current_right_is_pk = current_right_col and current_right_col.upper() in right_pk_set
            reverse_right_is_pk = reverse_right_col and reverse_right_col.upper() in left_pk_set

            # CRITICAL: When both sides have PK, check which one is a composite PK
            # The correct direction is: composite PK column (FK) -> single PK (referenced PK)
            # Current: left_table.current_left_col -> right_table.current_right_col
            # Check if left_table has composite PK (len > 1)
            current_left_is_composite_pk = current_left_col and current_left_col.upper() in left_pk_set and len(left_pk_set) > 1
            # Reverse: right_table.reverse_left_col -> left_table.reverse_right_col
            # Check if right_table has composite PK (len > 1)
            reverse_left_is_composite_pk = reverse_left_col and reverse_left_col.upper() in right_pk_set and len(right_pk_set) > 1

            # If current direction is FK->PK but reverse is not, replace with current
            if current_right_is_pk and not reverse_right_is_pk:
                print(f"  🔄 Replacing {reverse_key[0]} → {reverse_key[1]} with {left_table} → {right_table} (correct FK->PK direction)")
                del deduplicated_pairs[reverse_key]
                deduplicated_pairs[(left_table, right_table)] = column_pairs
            # If reverse is FK->PK but current is not, keep reverse (skip current)
            elif reverse_right_is_pk and not current_right_is_pk:
                print(f"  ⚠️  Skipping {left_table} → {right_table} (keeping {reverse_key[0]} → {reverse_key[1]} as correct FK->PK direction)")
                continue
            # If both have PK on right side, need to determine which is the true FK->PK relationship
            elif current_right_is_pk and reverse_right_is_pk:
                # Current: left_table.current_left_col -> right_table.current_right_col
                # Reverse: right_table.reverse_left_col -> left_table.reverse_right_col

                # Check if left columns are part of their respective table's PK
                current_left_is_pk = current_left_col and current_left_col.upper() in left_pk_set
                reverse_left_is_pk = reverse_left_col and reverse_left_col.upper() in right_pk_set

                # DEBUG: PK analysis (commented out for production)
                # print(f"  🔍 PK analysis for {left_table}.{current_left_col} → {right_table}.{current_right_col}:")
                # print(f"      Reverse is: {reverse_key[0]}.{reverse_left_col} → {reverse_key[1]}.{reverse_right_col}")
                # print(f"      Current left_is_pk={current_left_is_pk}, reverse left_is_pk={reverse_left_is_pk}")
                # print(f"      left_pk_set={left_pk_set}, right_pk_set={right_pk_set}")

                # Prefer the direction where the left column is NOT a PK (it's a pure FK)
                if not current_left_is_pk and reverse_left_is_pk:
                    # Current direction is better: left is FK, right is PK
                    print(f"  🔄 Replacing {reverse_key[0]} → {reverse_key[1]} with {left_table} → {right_table} (left is FK, not PK)")
                    del deduplicated_pairs[reverse_key]
                    deduplicated_pairs[(left_table, right_table)] = column_pairs
                elif current_left_is_pk and not reverse_left_is_pk:
                    # Reverse direction is better: its left is FK, its right is PK
                    print(f"  ⚠️  Skipping {left_table} → {right_table} (keeping {reverse_key[0]} → {reverse_key[1]}, reverse left is FK)")
                    continue
                # Both left columns are part of composite PKs - prefer composite PK with more columns
                elif current_left_is_composite_pk and not reverse_left_is_composite_pk:
                    print(f"  🔄 Replacing {reverse_key[0]} → {reverse_key[1]} with {left_table} → {right_table} (left has composite PK, correct FK->PK)")
                    del deduplicated_pairs[reverse_key]
                    deduplicated_pairs[(left_table, right_table)] = column_pairs
                elif reverse_left_is_composite_pk and not current_left_is_composite_pk:
                    print(f"  ⚠️  Skipping {left_table} → {right_table} (keeping {reverse_key[0]} → {reverse_key[1]}, reverse has composite PK)")
                    continue
                else:
                    # Both have same structure - check if neither is PK (both are pure FKs)
                    if not current_left_is_pk and not reverse_left_is_pk:
                        # Both left columns are FKs - this is a valid bidirectional relationship!
                        # Keep the first one (reverse), skip current
                        print(f"  ⚠️  Skipping duplicate: {left_table} → {right_table} (both have same FK->PK pattern)")
                        continue
                    else:
                        print(f"  ⚠️  Skipping duplicate: {left_table} → {right_table} (both have same PK structure)")
                        continue
            # If neither has PK, keep the first one encountered
            else:
                print(f"  ⚠️  Skipping duplicate: {left_table} → {right_table} (reverse of {right_table} → {left_table} already exists)")
                continue

        # Check if we already processed this canonical pair (different column combinations)
        if canonical_key in [(tuple(sorted([l, r]))) for (l, r) in deduplicated_pairs.keys()]:
            # Already have this pair, skip
            existing = [(l, r) for (l, r) in deduplicated_pairs.keys()
                       if tuple(sorted([l, r])) == canonical_key]
            if existing:
                print(f"  ⚠️  Skipping duplicate: {left_table} → {right_table} (already have {existing[0][0]} → {existing[0][1]})")
                continue

        deduplicated_pairs[(left_table, right_table)] = column_pairs

    print(f"✅ After deduplication: {len(deduplicated_pairs)} unique relationship pairs\n")
    pairs = deduplicated_pairs

    # Build relationships with inferred cardinality
    for (left_table, right_table), column_pairs in pairs.items():
        if _timed_out():
            status_dict["limited_by_timeout"] = True
            break
        if (
            max_relationships is not None
            and len(relationships) >= max_relationships
        ):
            status_dict["limited_by_max_relationships"] = True
            break

        column_pairs = list(column_pairs)

        # Universal Fix: Validate composite key consistency (works for ANY schema)
        if len(column_pairs) > 1:
            if not _validate_composite_key_consistency(column_pairs):
                # This is a logical error that applies to all schemas
                print(f"🚫 Skipping logically inconsistent composite key {left_table} → {right_table}: {column_pairs}")
                continue  # Skip invalid composite keys

        # P1 Fix: Filter excessive composite key relationships
        if len(column_pairs) > 2:
            # Be very restrictive with composite keys having more than 2 columns
            # Only allow if confidence is very high
            left_meta = metadata[left_table]
            right_meta = metadata[right_table]

            # Check if this forms a strong composite key pattern
            left_analysis = _analyze_composite_key_patterns(
                left_meta, column_pairs, column_index=0
            )
            right_analysis = _analyze_composite_key_patterns(
                right_meta, column_pairs, column_index=1
            )

            # Only allow multi-column relationships if they form clear composite PKs
            if not (left_analysis["is_composite_pk"] or right_analysis["is_composite_pk"]):
                continue  # Skip this relationship
        elif len(column_pairs) > 1:
            # For 2-column composite keys, apply moderate filtering
            # Ensure at least one side has meaningful PK coverage
            left_meta = metadata[left_table]
            right_meta = metadata[right_table]

            left_analysis = _analyze_composite_key_patterns(
                left_meta, column_pairs, column_index=0
            )
            right_analysis = _analyze_composite_key_patterns(
                right_meta, column_pairs, column_index=1
            )

            # Require some PK involvement for composite relationships (relaxed threshold)
            if (left_analysis["pk_coverage_ratio"] < 0.3 and
                right_analysis["pk_coverage_ratio"] < 0.3):
                continue  # Skip this relationship

        # Infer cardinality based on available metadata
        left_meta = metadata[left_table]
        right_meta = metadata[right_table]

        left_pk_names = _pk_name_set(left_meta)
        right_pk_names = _pk_name_set(right_meta)

        left_pk_hits = sum(1 for left_col, _ in column_pairs if left_col.upper() in left_pk_names)
        right_pk_hits = sum(1 for _, right_col in column_pairs if right_col.upper() in right_pk_names)

        # Determine orientation based on PK coverage and naming conventions
        should_swap = False
        if left_pk_hits > right_pk_hits:
            should_swap = True
        elif left_pk_hits == right_pk_hits:
            left_convention = _identify_naming_convention(left_table, domain_patterns_cache)
            right_convention = _identify_naming_convention(right_table, domain_patterns_cache)
            if left_convention == "dimension" and right_convention in {"fact", "bridge"}:
                should_swap = True

        if should_swap:
            column_pairs = [(right_col, left_col) for left_col, right_col in column_pairs]
            left_table, right_table = right_table, left_table
            left_meta, right_meta = right_meta, left_meta
            left_pk_names, right_pk_names = right_pk_names, left_pk_names
            left_pk_hits, right_pk_hits = right_pk_hits, left_pk_hits

        normalized_signature = tuple(
            sorted((left_col.upper(), right_col.upper()) for left_col, right_col in column_pairs)
        )
        signature = (left_table, right_table, normalized_signature)
        if signature in emitted_relationship_signatures:
            continue
        emitted_relationship_signatures.add(signature)

        # Determine if relationship columns form complete primary keys
        # CRITICAL: Only consider it a PK if the relationship columns form a COMPLETE primary key
        # Not just part of a composite key
        def _forms_complete_pk(table_meta, column_names_in_relationship):
            """Check if the relationship columns form a complete PK (not just part of composite key)"""
            pk_candidates = table_meta.get("pk_candidates", {})
            if not pk_candidates:
                return False

            # Get all PK column names (normalized)
            all_pk_cols = set()
            for pk_list in pk_candidates.values():
                all_pk_cols.update(col.upper() for col in pk_list)

            # Get relationship column names (normalized)
            rel_cols = set(col.upper() for col in column_names_in_relationship)

            # DEBUG: Log PK matching details
            is_complete = rel_cols == all_pk_cols
            if len(all_pk_cols) > 0:
                table_name = table_meta.get("fqn", "unknown")
                print(f"    DEBUG _forms_complete_pk for table {table_name}:")
                print(f"      all_pk_cols: {sorted(all_pk_cols)}")
                print(f"      rel_cols: {sorted(rel_cols)}")
                print(f"      is_complete: {is_complete}")

            # Only return True if relationship columns match ALL PK columns
            return is_complete

        left_cols_in_rel = [pair[0] for pair in column_pairs]
        right_cols_in_rel = [pair[1] for pair in column_pairs]

        left_has_pk = _forms_complete_pk(left_meta, left_cols_in_rel)
        right_has_pk = _forms_complete_pk(right_meta, right_cols_in_rel)

        # Enhanced: Get sample values for all columns in the relationship (for composite key analysis)
        left_values_all = []
        right_values_all = []
        left_values = []  # First column only (for backward compatibility)
        right_values = []

        for left_col, right_col in column_pairs:
            left_col_key = _sanitize_identifier_name(
                left_col, prefixes_to_drop=global_prefixes | _table_prefixes(left_table)
            )
            right_col_key = _sanitize_identifier_name(
                right_col,
                prefixes_to_drop=global_prefixes | _table_prefixes(right_table),
            )

            left_col_values = []
            right_col_values = []

            if left_col_key in left_meta["columns"]:
                left_col_values = left_meta["columns"][left_col_key].get("values") or []
            if right_col_key in right_meta["columns"]:
                right_col_values = (
                    right_meta["columns"][right_col_key].get("values") or []
                )

            left_values_all.append(left_col_values)
            right_values_all.append(right_col_values)

            # Keep first column values for backward compatibility
            if not left_values:
                left_values = left_col_values
            if not right_values:
                right_values = right_col_values

        # Enhanced cardinality inference: Use composite key analysis if multiple columns
        if len(column_pairs) > 1:
            left_card, right_card = _infer_composite_cardinality(
                left_meta,
                right_meta,
                column_pairs,
                left_values_all,
                right_values_all,
            )
        else:
            # Fallback to single-column analysis with adaptive thresholds
            # Extract first column names for pattern analysis
            first_left_col = column_pairs[0][0] if column_pairs else ""
            first_right_col = column_pairs[0][1] if column_pairs else ""

            left_card, right_card = _infer_cardinality(
                left_values,
                right_values,
                left_has_pk,
                right_has_pk,
                left_table=left_table,
                right_table=right_table,
                left_column=first_left_col,
                right_column=first_right_col,
                adaptive_thresholds=global_adaptive_thresholds,
            )

        # CRITICAL FIX: Force correct cardinality based on PK metadata
        # Override statistical inference if we have clear PK metadata

        # DEBUG: Always log PK status and initial cardinality
        print(f"  🔍 Cardinality check for {left_table} → {right_table}:")
        print(f"     Initial: {left_card}:{right_card}")
        print(f"     PK status: left_has_pk={left_has_pk}, right_has_pk={right_has_pk}")

        if right_has_pk and not left_has_pk:
            # This is definitely FK -> PK (many-to-one)
            if left_card != "*" or right_card != "1":
                print(f"  🔧 Correcting cardinality for {left_table} → {right_table}: {left_card}:{right_card} -> *:1 (FK->PK)")
                left_card, right_card = ("*", "1")
            else:
                print(f"  ✅ Cardinality already correct: {left_card}:{right_card} (FK->PK)")
        elif left_has_pk and not right_has_pk:
            # This is definitely PK -> FK (one-to-many)
            if left_card != "1" or right_card != "*":
                print(f"  🔧 Correcting cardinality for {left_table} → {right_table}: {left_card}:{right_card} -> 1:* (PK->FK)")
                left_card, right_card = ("1", "*")
            else:
                print(f"  ✅ Cardinality already correct: {left_card}:{right_card} (PK->FK)")
        elif not right_has_pk and not left_has_pk:
            # Neither side is PK, default to many-to-one (safest)
            if left_card == "1" and right_card == "1":
                print(f"  🔧 Correcting cardinality for {left_table} → {right_table}: 1:1 -> *:1 (no PK, use safe default)")
                left_card, right_card = ("*", "1")
            else:
                print(f"  ℹ️  No correction needed: {left_card}:{right_card} (no PK on either side)")
        else:
            # Both have PK - this might be a composite key situation
            print(f"  ⚠️  Both sides have PK - keeping original: {left_card}:{right_card}")

        # Determine if SQL null probe should be executed for stricter inference
        strict_fk_detected = False
        if strict_join_inference and session:
            left_fqn_parts = left_meta.get("fqn")
            if isinstance(left_fqn_parts, data_types.FQNParts):
                strict_fk_detected = any(
                    _column_has_null_via_query(
                        session,
                        left_fqn_parts,
                        left_column,
                        null_check_cache,
                    )
                    for left_column, _ in column_pairs
                )

        # Infer join type based on relationship characteristics
        join_type = _infer_join_type(
            left_table,
            right_table,
            left_card,
            right_card,
            left_has_pk,
            right_has_pk,
            left_values,
            right_values,
            has_null_fk=strict_fk_detected,
            left_table_meta=left_meta,
            right_table_meta=right_meta,
        )

        # Calculate confidence and reasoning for this relationship
        confidence_analysis = _calculate_relationship_confidence(
            left_table=left_table,
            right_table=right_table,
            column_pairs=column_pairs,
            left_meta=left_meta,
            right_meta=right_meta,
            left_has_pk=left_has_pk,
            right_has_pk=right_has_pk,
            left_values=left_values,
            right_values=right_values,
            cardinality_result=(left_card, right_card),
            join_type=join_type,
            adaptive_thresholds=global_adaptive_thresholds,
        )

        # Apply domain knowledge to enhance confidence
        domain_enhancement = _apply_domain_knowledge(
            left_table=left_table,
            right_table=right_table,
            column_pairs=column_pairs,
            left_meta=left_meta,
            right_meta=right_meta,
            current_confidence=confidence_analysis["confidence_score"],
        )

        # Update confidence analysis with domain knowledge
        if domain_enhancement["confidence_boost"] > 0:
            confidence_analysis["confidence_score"] = min(
                1.0,
                confidence_analysis["confidence_score"]
                + domain_enhancement["confidence_boost"],
            )

            # Add domain knowledge factors to reasoning
            for domain_factor in domain_enhancement["domain_factors"]:
                confidence_analysis["reasoning_factors"].append(
                    f"Domain knowledge: {domain_factor}"
                )

            # Update confidence level based on new score
            if confidence_analysis["confidence_score"] >= 0.8:
                confidence_analysis["confidence_level"] = "very_high"
                confidence_analysis["confidence_description"] = "Very High Confidence"
            elif confidence_analysis["confidence_score"] >= 0.6:
                confidence_analysis["confidence_level"] = "high"
                confidence_analysis["confidence_description"] = "High Confidence"
            elif confidence_analysis["confidence_score"] >= 0.4:
                confidence_analysis["confidence_level"] = "medium"
                confidence_analysis["confidence_description"] = "Medium Confidence"
            elif confidence_analysis["confidence_score"] >= 0.2:
                confidence_analysis["confidence_level"] = "low"
                confidence_analysis["confidence_description"] = "Low Confidence"
            else:
                confidence_analysis["confidence_level"] = "very_low"
                confidence_analysis["confidence_description"] = "Very Low Confidence"

        # Enhanced logging with confidence and reasoning
        sample_info = f"samples: L={len(left_values)}, R={len(right_values)}"
        pk_info = f"PKs: L={left_has_pk}, R={right_has_pk}"
        join_type_name = (
            "INNER" if join_type == semantic_model_pb2.JoinType.inner else "LEFT_OUTER"
        )
        confidence_info = f"confidence: {confidence_analysis['confidence_score']:.2f} ({confidence_analysis['confidence_level']})"

        # Add domain knowledge info if applied
        domain_info = ""
        if domain_enhancement["confidence_boost"] > 0:
            domain_info = (
                f", domain boost: +{domain_enhancement['confidence_boost']:.2f}"
            )

        logger.info(
            f"Relationship inference for {left_table} -> {right_table}: "
            f"{left_card}:{right_card}, JOIN={join_type_name}, {confidence_info}{domain_info} "
            f"({sample_info}, {pk_info})"
        )

        # Log domain knowledge patterns if detected
        domain_factors = [
            f
            for f in confidence_analysis["reasoning_factors"]
            if f.startswith("Domain knowledge:")
        ]
        if domain_factors:
            logger.debug(
                f"Domain patterns detected for {left_table} -> {right_table}: {domain_factors}"
            )

        # Log detailed reasoning for medium or lower confidence relationships
        if confidence_analysis["confidence_score"] < 0.6:
            logger.debug(f"Confidence reasoning for {left_table} -> {right_table}:")
            for factor in confidence_analysis["reasoning_factors"]:
                logger.debug(f"  - {factor}")

        # Log very high confidence relationships with their evidence
        elif confidence_analysis["confidence_score"] >= 0.8:
            logger.debug(
                f"High confidence relationship {left_table} -> {right_table} based on:"
            )
            for factor in confidence_analysis["reasoning_factors"][:3]:  # Top 3 factors
                logger.debug(f"  + {factor}")

        if confidence_analysis["confidence_score"] < min_confidence:
            logger.debug(
                "Dropping relationship {} -> {} due to low confidence {:.2f} (threshold {:.2f})",
                left_table,
                right_table,
                confidence_analysis["confidence_score"],
                min_confidence,
            )
            continue

        # Determine relationship type based on cardinality
        if left_card == "1" and right_card == "1":
            rel_type = semantic_model_pb2.RelationshipType.one_to_one
        elif left_card in ("*", "+") and right_card == "1":
            rel_type = semantic_model_pb2.RelationshipType.many_to_one
        elif left_card == "1" and right_card in ("*", "+"):
            rel_type = semantic_model_pb2.RelationshipType.one_to_many
        else:
            # Default to many_to_one for backward compatibility
            rel_type = semantic_model_pb2.RelationshipType.many_to_one

        relationship = semantic_model_pb2.Relationship(
            name=f"{left_table}_to_{right_table}",
            left_table=left_table,
            right_table=right_table,
            join_type=join_type,  # Use inferred join type instead of hardcoded inner
            relationship_type=rel_type,
        )
        for left_column, right_column in column_pairs:
            relationship.relationship_columns.append(
                semantic_model_pb2.RelationKey(
                    left_column=left_column.upper(), right_column=right_column.upper()
                )
            )
        relationships.append(relationship)

    # Phase 2: Detect many-to-many relationships through bridge table analysis
    many_to_many_relationships: List[semantic_model_pb2.Relationship] = []
    if not status_dict["limited_by_timeout"] and (
        max_relationships is None or len(relationships) < max_relationships
    ):
        many_to_many_relationships = _detect_many_to_many_relationships(
            raw_tables, metadata, relationships
        )

        if many_to_many_relationships and max_relationships is not None:
            remaining = max_relationships - len(relationships)
            if remaining <= 0:
                many_to_many_relationships = []
            else:
                many_to_many_relationships = many_to_many_relationships[:remaining]

        if many_to_many_relationships:
            relationships.extend(many_to_many_relationships)
            logger.info(
                f"Detected {len(many_to_many_relationships)} many-to-many relationships via bridge tables"
            )

    logger.info(
        f"Inferred {len(relationships)} total relationships across {len(raw_tables)} tables"
    )
    return relationships


def _raw_table_to_semantic_context_table(
    database: str, schema: str, raw_table: data_types.Table
) -> semantic_model_pb2.Table:
    """
    Converts a raw table representation to a semantic model table in protobuf format.

    Args:
        database (str): The name of the database containing the table.
        schema (str): The name of the schema containing the table.
        raw_table (data_types.Table): The raw table object to be transformed.

    Returns:
        semantic_model_pb2.Table: A protobuf representation of the semantic table.

    This function categorizes table columns into TimeDimensions, Dimensions, or Measures based on their data type,
    populates them with sample values, and sets placeholders for descriptions and filters.
    """

    # For each column, decide if it is a TimeDimension, Measure, or Dimension column.
    # For now, we decide this based on datatype.
    # Any time datatype, is TimeDimension.
    # Any varchar/text is Dimension.
    # Any numerical column is Measure.

    time_dimensions = []
    dimensions = []
    facts: List[semantic_model_pb2.Fact] = []
    used_identifier_names: set[str] = set()
    table_prefixes = _table_prefixes(raw_table.name)

    for col in raw_table.columns:
        base_type = _base_type_from_type(col.column_type)
        if _is_time_like_column(col):
            time_data_type = col.column_type
            if time_data_type.split("(")[0].upper() in {
                "STRING",
                "VARCHAR",
                "TEXT",
                "CHAR",
                "CHARACTER",
                "NVARCHAR",
            }:
                time_data_type = "TIMESTAMP_NTZ"
            time_dimension_name = _safe_semantic_identifier(
                col.column_name,
                used_identifier_names,
                "time_dimension",
                prefixes_to_drop=table_prefixes,
            )
            time_dimensions.append(
                semantic_model_pb2.TimeDimension(
                    name=time_dimension_name,
                    expr=col.column_name,
                    data_type=time_data_type,
                    sample_values=col.values,
                    synonyms=[_PLACEHOLDER_COMMENT],
                    description=col.comment if col.comment else _PLACEHOLDER_COMMENT,
                )
            )

        elif base_type in DIMENSION_DATATYPES:
            dimension_name = _safe_semantic_identifier(
                col.column_name,
                used_identifier_names,
                "dimension",
                prefixes_to_drop=table_prefixes,
            )
            dimensions.append(
                semantic_model_pb2.Dimension(
                    name=dimension_name,
                    expr=col.column_name,
                    data_type=col.column_type,
                    sample_values=col.values,
                    synonyms=[_PLACEHOLDER_COMMENT],
                    description=col.comment if col.comment else _PLACEHOLDER_COMMENT,
                )
            )

        elif base_type in MEASURE_DATATYPES:
            if _is_identifier_like(col.column_name, base_type):
                identifier_dimension_name = _safe_semantic_identifier(
                    col.column_name,
                    used_identifier_names,
                    "dimension",
                    prefixes_to_drop=table_prefixes,
                )
                dimensions.append(
                    semantic_model_pb2.Dimension(
                        name=identifier_dimension_name,
                        expr=col.column_name,
                        data_type=col.column_type,
                        sample_values=col.values,
                        synonyms=[_PLACEHOLDER_COMMENT],
                        description=(
                            col.comment if col.comment else _PLACEHOLDER_COMMENT
                        ),
                    )
                )
                continue
            fact_name = _safe_semantic_identifier(
                col.column_name,
                used_identifier_names,
                "fact",
                prefixes_to_drop=table_prefixes,
            )
            facts.append(
                semantic_model_pb2.Fact(
                    name=fact_name,
                    expr=col.column_name,
                    data_type=col.column_type,
                    sample_values=col.values,
                    synonyms=[_PLACEHOLDER_COMMENT],
                    description=col.comment if col.comment else _PLACEHOLDER_COMMENT,
                )
            )
        elif base_type in OBJECT_DATATYPES:
            logger.warning(
                f"""We don't currently support {col.column_type} as an input column datatype to the Semantic Model. We are skipping column {col.column_name} for now."""
            )
            continue
        else:
            fallback_dimension_name = _safe_semantic_identifier(
                col.column_name,
                used_identifier_names,
                "dimension",
                prefixes_to_drop=table_prefixes,
            )
            logger.warning(
                f"Column datatype does not map to a known datatype. Input was = {col.column_type}. We are going to place as a Dimension for now."
            )
            dimensions.append(
                semantic_model_pb2.Dimension(
                    name=fallback_dimension_name,
                    expr=col.column_name,
                    data_type=col.column_type,
                    sample_values=col.values,
                    synonyms=[_PLACEHOLDER_COMMENT],
                    description=col.comment if col.comment else _PLACEHOLDER_COMMENT,
                )
            )
    if len(time_dimensions) + len(dimensions) + len(facts) == 0:
        raise ValueError(
            f"No valid columns found for table {raw_table.name}. Please verify that this table contains column's datatypes not in {OBJECT_DATATYPES}."
        )

    filters = _suggest_filters(raw_table)

    return semantic_model_pb2.Table(
        name=raw_table.name,
        base_table=semantic_model_pb2.FullyQualifiedTable(
            database=database, schema=schema, table=raw_table.name
        ),
        # For fields we can not automatically infer, leave a comment for the user to fill out.
        description=raw_table.comment if raw_table.comment else _PLACEHOLDER_COMMENT,
        filters=filters,
        dimensions=dimensions,
        time_dimensions=time_dimensions,
        facts=facts,
    )


def raw_schema_to_semantic_context(
    base_tables: List[str],
    semantic_model_name: str,
    conn: Session,
    n_sample_values: int = _DEFAULT_N_SAMPLE_VALUES_PER_COL,
    allow_joins: Optional[bool] = True,
    strict_join_inference: bool = False,
    llm_custom_prompt: str = "",
    enrich_with_llm: bool = False,
    progress_callback: Optional[Callable[[str], None]] = None,
) -> semantic_model_pb2.SemanticModel:
    """
    Converts a list of fully qualified ClickZetta table names into a semantic model.

    Parameters:
    - base_tables  (list[str]): Fully qualified table names to include in the semantic model.
    - semantic_model_name (str): A meaningful semantic model name.
    - conn (Session): ClickZetta session to reuse.
    - n_sample_values (int): The number of sample values per col.
    - allow_joins (bool | None): Whether to infer table relationships.
    - strict_join_inference (bool): If True, runs additional `IS NULL` probes per relationship to tighten join-type detection (requires extra SQL queries).
    - llm_custom_prompt (str): Optional user instructions forwarded to the DashScope enrichment step.

    Returns:
    - The semantic model (semantic_model_pb2.SemanticModel).

    This function fetches metadata for the specified tables, performs schema validation, extracts key information,
    enriches metadata from ClickZetta, and constructs a semantic model in protobuf format.
    It handles different workspaces and schemas within the same account by reusing the provided session.

    Raises:
    - AssertionError: If no valid tables are found in the specified schema.
    """

    # For FQN tables, the connector handles cross-schema access automatically.
    def _notify(message: str) -> None:
        if progress_callback:
            try:
                progress_callback(message)
            except Exception:
                logger.debug("Progress callback failed for message: {}", message)

    table_objects = []
    raw_tables_metadata: List[tuple[data_types.FQNParts, data_types.Table]] = []
    unique_database_schema: List[str] = []
    for table in base_tables:
        # Verify this is a valid FQN table. For now, we check that the table follows the following format.
        # {database}.{schema}.{table}
        fqn_table = create_fqn_table(table)
        fqn_databse_schema = f"{fqn_table.database}.{fqn_table.schema_name}"

        if fqn_databse_schema not in unique_database_schema:
            unique_database_schema.append(fqn_databse_schema)

        logger.info(f"Pulling column information from {fqn_table}")
        _notify(
            f"Fetching metadata for {fqn_table.database}.{fqn_table.schema_name}.{fqn_table.table}..."
        )
        valid_schemas_tables_columns_df = get_valid_schemas_tables_columns_df(
            session=conn,
            workspace=fqn_table.database,
            table_schema=fqn_table.schema_name,
            table_names=[fqn_table.table],
        )
        if valid_schemas_tables_columns_df.empty:
            raise ValueError(
                (
                    "Unable to retrieve column metadata for table "
                    f"{fqn_table.database}.{fqn_table.schema_name}.{fqn_table.table}. "
                    "Shared or external catalogs (category=SHARED/EXTERNAL) do not expose information_schema views; "
                    "please copy the data into a managed workspace or provide manual metadata."
                )
            )

        # get the valid columns for this table.
        valid_columns_df_this_table = valid_schemas_tables_columns_df[
            valid_schemas_tables_columns_df["TABLE_NAME"] == fqn_table.table
        ]

        raw_table = get_table_representation(
            session=conn,
            workspace=fqn_table.database,
            schema_name=fqn_table.schema_name,
            table_name=fqn_table.table,  # Non-qualified table name
            table_index=0,
            ndv_per_column=n_sample_values,  # number of sample values to pull per column.
            columns_df=valid_columns_df_this_table,
            max_workers=1,
        )
        table_object = _raw_table_to_semantic_context_table(
            database=fqn_table.database,
            schema=fqn_table.schema_name,
            raw_table=raw_table,
        )
        table_objects.append(table_object)
        raw_tables_metadata.append((fqn_table, raw_table))
    # TODO(jhilgart): Call cortex model to generate a semantically friendly name here.

    relationships: List[semantic_model_pb2.Relationship] = []
    if allow_joins:
        relationships = _infer_relationships(
            raw_tables_metadata,
            session=conn if strict_join_inference else None,
            strict_join_inference=strict_join_inference,
        )

    context = semantic_model_pb2.SemanticModel(
        name=semantic_model_name,
        tables=table_objects,
        relationships=relationships,
    )
    context.description = _PLACEHOLDER_COMMENT
    context.custom_instructions = _PLACEHOLDER_COMMENT

    if enrich_with_llm:
        settings = get_dashscope_settings()
        if settings:
            actual_model = "qwen-plus-latest"
            logger.info(
                "Running DashScope enrichment for semantic model '{}' using model '{}'",
                semantic_model_name,
                actual_model,
            )
            _notify(
                "Running DashScope enrichment to enhance descriptions and metrics..."
            )

            # Create progress tracker for enrichment
            def enrichment_progress_callback(update):
                stage_messages = {
                    EnrichmentStage.TABLE_ENRICHMENT: "Enriching table descriptions and metrics",
                    EnrichmentStage.MODEL_DESCRIPTION: "Generating model description",
                    EnrichmentStage.MODEL_METRICS: "Generating model-level metrics",
                    EnrichmentStage.VERIFIED_QUERIES: "Generating verified queries",
                    EnrichmentStage.COMPLETE: "Enrichment complete",
                }

                base_message = stage_messages.get(update.stage, "Processing")
                if update.table_name:
                    message = f"{base_message} - {update.table_name} ({update.current_step}/{update.total_steps})"
                elif update.total_steps > 1:
                    message = (
                        f"{base_message} ({update.current_step}/{update.total_steps})"
                    )
                else:
                    message = base_message

                if update.details and update.details.get("message"):
                    message += f": {update.details['message']}"
                elif update.message:
                    message += f": {update.message}"

                _notify(f"[{update.percentage:.0f}%] {message}")

            progress_tracker = EnrichmentProgressTracker(enrichment_progress_callback)

            settings = DashscopeSettings(
                api_key=settings.api_key,
                model=actual_model,
                base_url=settings.base_url,
                temperature=settings.temperature,
                top_p=settings.top_p,
                max_output_tokens=settings.max_output_tokens,
                timeout_seconds=settings.timeout_seconds,
            )
            client = DashscopeClient(settings)
            enrich_semantic_model(
                context,
                raw_tables_metadata,
                client,
                placeholder=_PLACEHOLDER_COMMENT,
                custom_prompt=llm_custom_prompt,
                session=conn,
                progress_tracker=progress_tracker,
            )
            _notify("DashScope enrichment complete.")
        else:
            logger.warning(
                "LLM enrichment was requested but DashScope is not configured; skipping enrichment."
            )
            _notify("DashScope configuration missing; skipped enrichment.")
    return context


def append_comment_to_placeholders(yaml_str: str) -> str:
    """
    Finds all instances of a specified placeholder in a YAML string and appends a given text to these placeholders.
    This is the homework to fill out after your yaml is generated.

    Parameters:
    - yaml_str (str): The YAML string to process.

    Returns:
    - str: The modified YAML string with appended text to placeholders.
    """
    updated_yaml = []
    # Split the string into lines to process each line individually
    lines = yaml_str.split("\n")

    for line in lines:
        # Check if the placeholder is in the current line.
        # Strip the last quote to match.
        if line.rstrip("'").endswith(_PLACEHOLDER_COMMENT):
            # Replace the _PLACEHOLDER_COMMENT with itself plus the append_text
            updated_line = line + _FILL_OUT_TOKEN
            updated_yaml.append(updated_line)
        elif line.rstrip("'").endswith(AUTOGEN_TOKEN):
            updated_line = line + _AUTOGEN_COMMENT_TOKEN
            updated_yaml.append(updated_line)
        # Add comments to specific fields in certain sections.
        elif line.lstrip().startswith("join_type"):
            updated_yaml.append(line)
        elif line.lstrip().startswith("relationship_type"):
            updated_yaml.append(line)
        else:
            updated_yaml.append(line)

    # Join the lines back together into a single string
    return "\n".join(updated_yaml)


def _to_snake_case(s: str) -> str:
    """
    Convert a string into snake case.

    Parameters:
    s (str): The string to convert.

    Returns:
    str: The snake case version of the string.
    """
    # Replace common delimiters with spaces
    s = s.replace("-", " ").replace("_", " ")

    words = s.split(" ")

    # Convert each word to lowercase and join with underscores
    snake_case_str = "_".join([word.lower() for word in words if word]).strip()

    return snake_case_str


def generate_base_semantic_model_from_clickzetta(
    base_tables: List[str],
    conn: Session,
    semantic_model_name: str,
    n_sample_values: int = _DEFAULT_N_SAMPLE_VALUES_PER_COL,
    output_yaml_path: Optional[str] = None,
) -> None:
    """
    Generates a base semantic context from specified ClickZetta tables and exports it to a YAML file.

    Parameters:
        base_tables : Fully qualified names of ClickZetta tables to include in the semantic context.
        conn: ClickZetta session to reuse.
        semantic_model_name: The human readable model name. This should be semantically meaningful to an organization.
        output_yaml_path: Path for the output YAML file. If None, defaults to 'semantic_model_generator/output_models/YYYYMMDDHHMMSS_<semantic_model_name>.yaml'.
        n_sample_values: The number of sample values to populate for all columns.

    Returns:
        None. Writes the semantic context to a YAML file.
    """
    formatted_datetime = datetime.now().strftime("%Y%m%d%H%M%S")
    if not output_yaml_path:
        file_name = f"{formatted_datetime}_{_to_snake_case(semantic_model_name)}.yaml"
        if os.path.exists("semantic_model_generator/output_models"):
            write_path = f"semantic_model_generator/output_models/{file_name}"
        else:
            write_path = f"./{file_name}"
    else:  # Assume user gives correct path.
        write_path = output_yaml_path

    yaml_str = generate_model_str_from_clickzetta(
        base_tables,
        n_sample_values=n_sample_values if n_sample_values > 0 else 1,
        semantic_model_name=semantic_model_name,
        conn=conn,
    )

    with open(write_path, "w") as f:
        # Clarify that the YAML was autogenerated and that placeholders should be filled out/deleted.
        f.write(_AUTOGEN_COMMENT_WARNING)
        f.write(yaml_str)

    logger.info(f"Semantic model saved to {write_path}")

    return None


def generate_model_str_from_clickzetta(
    base_tables: List[str],
    semantic_model_name: str,
    conn: Session,
    n_sample_values: int = _DEFAULT_N_SAMPLE_VALUES_PER_COL,
    allow_joins: Optional[bool] = True,
    strict_join_inference: bool = False,
    enrich_with_llm: bool = False,
    llm_custom_prompt: str = "",
    progress_callback: Optional[Callable[[str], None]] = None,
) -> str:
    """
    Generates a base semantic context from specified ClickZetta tables and returns the raw string.

    Parameters:
        base_tables : Fully qualified names of ClickZetta tables to include in the semantic context.
        semantic_model_name: The human readable model name. This should be semantically meaningful to an organization.
        conn: ClickZetta session to reuse.
        n_sample_values: The number of sample values to populate for all columns.
        allow_joins: Whether to allow joins in the semantic context.
        strict_join_inference: When True, executes additional null-probe SQL queries to improve join type accuracy.
        llm_custom_prompt: Optional user instructions forwarded to the DashScope enrichment step.
        progress_callback: Optional callable invoked with human-readable progress updates.

    Returns:
        str: The raw string of the semantic context.
    """

    def _notify(message: str) -> None:
        if progress_callback:
            try:
                progress_callback(message)
            except Exception:
                logger.debug("Progress callback failed for message: {}", message)

    table_list = ", ".join(base_tables)
    logger.info(
        "Generating semantic model '{}' from tables: {}",
        semantic_model_name,
        table_list,
    )
    _notify("Collecting metadata from ClickZetta tables...")

    context = raw_schema_to_semantic_context(
        base_tables,
        n_sample_values=n_sample_values if n_sample_values > 0 else 1,
        semantic_model_name=semantic_model_name,
        allow_joins=allow_joins,
        strict_join_inference=strict_join_inference,
        llm_custom_prompt=llm_custom_prompt,
        enrich_with_llm=enrich_with_llm,
        conn=conn,
        progress_callback=_notify,
    )
    _notify("Constructing semantic model structure...")
    # Validate the generated yaml is within context limits.
    # We just throw a warning here to allow users to update.
    validate_context_length(context)
    _notify("Validating semantic model context length...")

    _notify("Converting semantic model to YAML...")
    yaml_str = proto_utils.proto_to_yaml(context)
    # Once we have the yaml, update to include to # <FILL-OUT> tokens.
    yaml_str = append_comment_to_placeholders(yaml_str)

    _notify("Semantic model generation complete.")
    logger.info("Semantic model '{}' generated successfully.", semantic_model_name)
    return yaml_str
