import os
import re
import math
from collections import defaultdict
from datetime import datetime
from typing import Any, Callable, Dict, List, Optional, Tuple

from clickzetta.zettapark.session import Session
from loguru import logger

from semantic_model_generator.data_processing import data_types, proto_utils
from semantic_model_generator.protos import semantic_model_pb2
from semantic_model_generator.clickzetta_utils.clickzetta_connector import (
    AUTOGEN_TOKEN,
    DIMENSION_DATATYPES,
    MEASURE_DATATYPES,
    OBJECT_DATATYPES,
    TIME_MEASURE_DATATYPES,
    get_table_representation,
    get_valid_schemas_tables_columns_df,
)
from semantic_model_generator.clickzetta_utils.utils import create_fqn_table
from semantic_model_generator.validate.context_length import validate_context_length
from semantic_model_generator.llm import (
    DashscopeClient,
    DashscopeSettings,
    enrich_semantic_model,
    get_dashscope_settings,
)
from semantic_model_generator.llm.progress_tracker import EnrichmentProgressTracker, EnrichmentStage
from semantic_model_generator.validate.keywords import CZ_RESERVED_WORDS

_PLACEHOLDER_COMMENT = "  "
_FILL_OUT_TOKEN = " # <FILL-OUT>"
# TODO add _AUTO_GEN_TOKEN to the end of the auto generated descriptions.
_AUTOGEN_COMMENT_TOKEN = (
    " # <AUTO-GENERATED DESCRIPTION, PLEASE MODIFY AND REMOVE THE __ AT THE END>"
)
_DEFAULT_N_SAMPLE_VALUES_PER_COL = 10
_AUTOGEN_COMMENT_WARNING = f"# NOTE: This file was auto-generated by the semantic model generator. Please fill out placeholders marked with {_FILL_OUT_TOKEN} (or remove if not relevant) and verify autogenerated comments.\n"

def _singularize(token: str) -> str:
    if token.endswith("IES") and len(token) > 3:
        return token[:-3] + "Y"
    if token.endswith(("SES", "XES", "ZES", "CHES", "SHES")) and len(token) > 4:
        return token[:-2]
    if token.endswith("S") and len(token) > 3:
        return token[:-1]
    return token


def _clean_column_type(raw: Any) -> str:
    if raw is None:
        return ""
    text = str(raw).strip()
    if not text:
        return ""
    text = re.sub(r"\s+", " ", text.upper())
    for suffix in (" NOT NULL", " NULL"):
        if text.endswith(suffix):
            text = text[: -len(suffix)].strip()
    return text


def _base_type_from_type(column_type: str) -> str:
    cleaned = _clean_column_type(column_type)
    token = cleaned.split(" ")[0] if cleaned else ""
    return token.split("(")[0]


def _identifier_tokens(name: str, prefixes_to_drop: Optional[set[str]] = None) -> List[str]:
    name = name.replace("-", "_")
    raw_tokens = re.split(r"[^0-9A-Za-z]+", name)
    tokens: List[str] = []
    for token in raw_tokens:
        if not token:
            continue
        split = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", token).split()
        for part in split:
            tokens.append(part.upper())
    tokens = [token for token in tokens if token]
    if prefixes_to_drop and len(tokens) >= 2 and tokens[0] in prefixes_to_drop:
        tokens = tokens[1:]
    return tokens


def _sanitize_identifier_name(name: str, prefixes_to_drop: Optional[set[str]] = None) -> str:
    if not name:
        return ""

    cleaned = name.replace("-", "_")
    cleaned = re.sub(r"\s+", "_", cleaned)
    parts = [part for part in cleaned.split("_") if part]
    if not parts:
        return ""

    if len(parts) >= 2 and len(parts[0]) == 1:
        parts = parts[1:]

    if prefixes_to_drop:
        drop_tokens = {token.upper() for token in prefixes_to_drop}
        while parts and parts[0].upper() in drop_tokens:
            parts = parts[1:]

    if not parts:
        return ""

    rebuilt = "_".join(parts)
    tokens = _identifier_tokens(rebuilt)
    if not tokens:
        return ""

    if len(parts) == 1:
        return "".join(tokens)
    return "_".join(tokens)


_INVALID_ID_CHARS = re.compile(r"[^0-9A-Za-z_$]")


def _is_valid_identifier(name: str) -> bool:
    if not name:
        return False
    candidate = name.strip()
    if not candidate:
        return False
    return candidate.replace("_", "").replace("$", "").isalnum()


def _normalize_identifier(name: str) -> str:
    if not name:
        return ""
    text = re.sub(r"\s+", "_", name.strip())
    text = text.replace("-", "_")
    text = _INVALID_ID_CHARS.sub("", text)
    text = re.sub(r"_+", "_", text)
    return text.strip("_")


def _safe_semantic_identifier(
    raw_name: str,
    used_names: set[str],
    fallback_prefix: str,
    prefixes_to_drop: Optional[set[str]] = None,
) -> str:
    """
    Produce a YAML-safe identifier that avoids reserved keywords and duplicates.
    """

    variants: List[str] = []
    normalized = _normalize_identifier(raw_name)
    if normalized:
        variants.append(normalized)
    sanitized = _sanitize_identifier_name(raw_name, prefixes_to_drop=prefixes_to_drop)
    if sanitized:
        variants.append(sanitized.lower())
    variants.append(f"{fallback_prefix}_field")

    for variant in variants:
        candidate = _normalize_identifier(variant)
        if not candidate:
            continue
        if candidate[0].isdigit():
            candidate = f"{fallback_prefix}_{candidate}"
        candidate = candidate.lower()
        base = candidate
        suffix = 2
        while (
            base in used_names
            or base.upper() in CZ_RESERVED_WORDS
            or not _is_valid_identifier(base)
        ):
            base = f"{candidate}_{suffix}"
            suffix += 1
        if base.upper() in CZ_RESERVED_WORDS or not _is_valid_identifier(base):
            continue
        used_names.add(base)
        return base

    suffix = 1
    while True:
        candidate = f"{fallback_prefix}_{suffix}"
        if (
            candidate not in used_names
            and candidate.upper() not in CZ_RESERVED_WORDS
            and _is_valid_identifier(candidate)
        ):
            used_names.add(candidate)
            return candidate
        suffix += 1


def _is_identifier_like(column_name: str, base_type: str) -> bool:
    """
    Heuristic to detect identifier-style numeric columns that should stay as dimensions.
    """

    tokens = _identifier_tokens(column_name)
    if not tokens:
        return False

    tail = tokens[-1]
    if tail in {"ID", "KEY", "ROWID", "ROW_ID", "PK", "PRIMARY", "PRIMARYKEY"}:
        return True
    if tail.endswith("ID") and len(tail) > 2:
        return True
    if tail.endswith("KEY") and len(tail) > 3:
        return True

    return False


def _table_variants(table_name: str) -> set[str]:
    table_upper = table_name.upper()
    tokens = _identifier_tokens(table_name)
    variants = {table_upper}
    variants.update(tokens)
    for token in list(variants):
        variants.add(_singularize(token))
        if len(token) > 3:
            variants.add(token[:4])
        if len(token) > 3:
            variants.add(token[:3])
        if len(token) > 3:
            variants.add(token[-4:])
        if len(token) > 2:
            variants.add(token[-3:])
    return {variant for variant in variants if variant}


_GENERIC_PREFIXES = {
    "DIM",
    "FACT",
    "FCT",
    "BRIDGE",
    "BRG",
    "STG",
    "ODS",
    "OD",
    "DW",
    "VW",
    "VIEW",
    "HUB",
    "SAT",
    "LNK",
    "TMP",
    "TMPV",
}


def _table_prefixes(table_name: str) -> set[str]:
    prefixes: set[str] = set()
    tokens = _identifier_tokens(table_name)
    if not tokens:
        tokens = [table_name.upper()]

    first_token = tokens[0].upper()
    if first_token in _GENERIC_PREFIXES:
        for length in range(1, min(len(first_token), 4) + 1):
            prefixes.add(first_token[:length])
        prefixes.add(first_token)
    return prefixes


def _looks_like_primary_key(table_name: str, column_name: str) -> bool:
    upper_name = column_name.strip().upper()
    variants = _table_variants(table_name)
    direct_matches = {
        "ID",
        "PK",
        "PRIMARY_KEY",
    }
    for variant in variants:
        direct_matches.update({f"{variant}_ID", f"{variant}ID", f"{variant}_KEY", f"{variant}KEY"})
    if upper_name in direct_matches:
        return True

    tokens = _identifier_tokens(column_name)
    if not tokens:
        return False
    last = tokens[-1]
    if last in {"ID", "KEY"} and len(tokens) >= 2 and tokens[-2] in variants:
        return True
    if last.endswith("ID") and last[:-2] in variants:
        return True
    if last.endswith("KEY") and last[:-3] in variants:
        return True
    if len(tokens) == 1 and last.endswith(("ID", "KEY")):
        stem = last[:-2] if last.endswith("ID") else last[:-3]
        if stem in variants:
            return True

    return False


_TIME_NAME_HINTS = ("DATE", "TIME", "TIMESTAMP", "DT", "DAY", "HOUR")
_ISO_DATE_REGEX = re.compile(r"^\d{4}[-/]\d{2}[-/]\d{2}")


def _is_time_like_column(column: data_types.Column) -> bool:
    base_type = _base_type_from_type(column.column_type)
    if base_type in TIME_MEASURE_DATATYPES:
        return True
    if base_type not in {"STRING", "VARCHAR", "TEXT", "CHAR", "CHARACTER", "NVARCHAR"}:
        return False

    column_name = column.column_name.upper()
    if any(hint in column_name for hint in _TIME_NAME_HINTS):
        return True

    for value in (column.values or [])[:5]:
        val = str(value).strip()
        if not val:
            continue
        candidate = val
        if candidate.endswith("Z"):
            candidate = candidate[:-1]
        candidate = candidate.replace(" ", "T", 1)
        try:
            datetime.fromisoformat(candidate)
            return True
        except ValueError:
            pass
        if _ISO_DATE_REGEX.match(val):
            return True
    return False


def _format_literal(value: str, base_type: str) -> str:
    text = "" if value is None else str(value)
    if base_type in MEASURE_DATATYPES:
        try:
            if "." in text:
                return str(float(text))
            return str(int(text))
        except ValueError:
            pass
    lowered = text.lower()
    if lowered in {"true", "false"}:
        return lowered.upper()
    escaped = text.replace("'", "''")
    return f"'{escaped}'"


def _format_sql_identifier(name: str) -> str:
    """
    Formats an identifier for SQL (without quoting) by stripping quotes and uppercasing.
    """
    if not name:
        return ""
    return str(name).replace('"', "").replace("`", "").strip().upper()


def _qualified_table_name(fqn: data_types.FQNParts) -> str:
    """
    Builds a fully qualified table name without quoting.
    """
    parts = [part for part in (fqn.database, fqn.schema_name, fqn.table) if part]
    return ".".join(_format_sql_identifier(part) for part in parts if part)


def _levenshtein_distance(s1: str, s2: str) -> int:
    """
    Calculate Levenshtein distance between two strings.
    Used for fuzzy column name matching.
    """
    if len(s1) < len(s2):
        return _levenshtein_distance(s2, s1)
    if len(s2) == 0:
        return len(s1)
    
    previous_row = range(len(s2) + 1)
    for i, c1 in enumerate(s1):
        current_row = [i + 1]
        for j, c2 in enumerate(s2):
            insertions = previous_row[j + 1] + 1
            deletions = current_row[j] + 1
            substitutions = previous_row[j] + (c1 != c2)
            current_row.append(min(insertions, deletions, substitutions))
        previous_row = current_row
    
    return previous_row[-1]


def _name_similarity(name1: str, name2: str) -> float:
    """
    Calculate normalized similarity score between two column names.
    Returns a score between 0.0 (completely different) and 1.0 (identical).
    """
    if not name1 or not name2:
        return 0.0
    
    # Exact match
    if name1.upper() == name2.upper():
        return 1.0
    
    # Normalize names for comparison
    norm1 = name1.upper().replace("_", "").replace("-", "")
    norm2 = name2.upper().replace("_", "").replace("-", "")
    
    if norm1 == norm2:
        return 0.95
    
    # Calculate Levenshtein-based similarity
    max_len = max(len(norm1), len(norm2))
    if max_len == 0:
        return 0.0
    
    distance = _levenshtein_distance(norm1, norm2)
    similarity = 1.0 - (distance / max_len)
    
    return max(0.0, similarity)


def _analyze_composite_key_patterns(
    table_meta: Dict[str, Any],
    column_pairs: List[tuple[str, str]],
) -> Dict[str, Any]:
    """
    Analyze composite key patterns for multi-column relationships.

    Args:
        table_meta: Metadata about the table
        column_pairs: List of column pairs in the relationship

    Returns:
        Dict with composite key analysis results
    """
    pk_candidates = table_meta.get("pk_candidates", {})
    columns_meta = table_meta.get("columns", {})

    # Check if all relationship columns form a composite key
    relationship_cols = [pair[0] if isinstance(pair, tuple) else pair for pair in column_pairs]

    # Normalize column names for comparison
    global_prefixes = set()  # This should come from context but we'll handle it locally
    table_prefixes = _table_prefixes(list(table_meta.get("columns", {}).keys())[0] if table_meta.get("columns") else "")

    normalized_rel_cols = [
        _sanitize_identifier_name(col, prefixes_to_drop=global_prefixes | table_prefixes)
        for col in relationship_cols
    ]

    # Check if these columns together might form a composite PK
    pk_col_count = sum(1 for col in normalized_rel_cols if col in pk_candidates)
    total_pk_candidates = len(pk_candidates)

    analysis = {
        "is_composite_pk": pk_col_count > 1 and pk_col_count == total_pk_candidates,
        "partial_pk": pk_col_count > 0 and pk_col_count < total_pk_candidates,
        "pk_coverage_ratio": pk_col_count / total_pk_candidates if total_pk_candidates > 0 else 0,
        "relationship_column_count": len(relationship_cols),
        "pk_column_count": pk_col_count,
    }

    # Additional analysis: check for sequential ID patterns (common in composite keys)
    if len(relationship_cols) > 1:
        sequential_patterns = []
        for col in relationship_cols:
            if any(pattern in col.upper() for pattern in ["_ID", "ID", "_KEY", "KEY", "_NUM", "NUM"]):
                sequential_patterns.append(col)

        analysis["sequential_id_pattern"] = len(sequential_patterns) >= 2

    return analysis


def _infer_composite_cardinality(
    left_meta: Dict[str, Any],
    right_meta: Dict[str, Any],
    column_pairs: List[tuple[str, str]],
    left_values_all: List[List[str]],
    right_values_all: List[List[str]],
) -> tuple[str, str]:
    """
    Enhanced cardinality inference that considers composite keys.

    Args:
        left_meta: Left table metadata
        right_meta: Right table metadata
        column_pairs: All column pairs in the relationship
        left_values_all: Sample values for all left columns
        right_values_all: Sample values for all right columns

    Returns:
        tuple: (left_cardinality, right_cardinality)
    """
    # Analyze composite key patterns
    left_analysis = _analyze_composite_key_patterns(left_meta, column_pairs)
    right_analysis = _analyze_composite_key_patterns(right_meta, column_pairs)

    # Rule 1: If one side is a complete composite PK, it's likely "1"
    if right_analysis["is_composite_pk"]:
        return ("*", "1")
    elif left_analysis["is_composite_pk"]:
        return ("1", "*")

    # Rule 2: High PK coverage suggests uniqueness
    if right_analysis["pk_coverage_ratio"] >= 0.8:
        return ("*", "1")
    elif left_analysis["pk_coverage_ratio"] >= 0.8:
        return ("1", "*")

    # Rule 3: Composite key uniqueness analysis (if we have sufficient samples)
    MIN_SAMPLE_SIZE = 20  # Lower threshold for composite keys

    if (left_values_all and right_values_all and
        len(left_values_all) >= MIN_SAMPLE_SIZE and
        len(right_values_all) >= MIN_SAMPLE_SIZE):

        # Create composite keys by concatenating values
        left_composite_keys = []
        right_composite_keys = []

        sample_size = min(len(left_values_all), len(right_values_all))

        for i in range(sample_size):
            left_key = "|".join(str(vals[i]) if i < len(vals) else ""
                              for vals in left_values_all)
            right_key = "|".join(str(vals[i]) if i < len(vals) else ""
                               for vals in right_values_all)

            if left_key and not _is_nullish(left_key):
                left_composite_keys.append(left_key)
            if right_key and not _is_nullish(right_key):
                right_composite_keys.append(right_key)

        if left_composite_keys and right_composite_keys:
            left_unique_ratio = len(set(left_composite_keys)) / len(left_composite_keys)
            right_unique_ratio = len(set(right_composite_keys)) / len(right_composite_keys)

            # Lower threshold for composite key uniqueness
            if right_unique_ratio > 0.9:
                return ("*", "1")
            elif left_unique_ratio > 0.9:
                return ("1", "*")

    # Rule 4: Fall back to single-column analysis with adaptive thresholds
    if column_pairs:
        first_left_vals = left_values_all[0] if left_values_all else []
        first_right_vals = right_values_all[0] if right_values_all else []

        # Calculate adaptive thresholds for composite key analysis
        all_values = []
        if left_values_all:
            all_values.extend(left_values_all)
        if right_values_all:
            all_values.extend(right_values_all)

        adaptive_thresholds = _calculate_adaptive_thresholds(
            all_values,
            table_count=2,  # For pairwise comparison
            base_sample_size=len(first_left_vals) if first_left_vals else 10,
        )

        return _infer_cardinality(
            first_left_vals,
            first_right_vals,
            left_analysis.get("pk_column_count", 0) > 0,
            right_analysis.get("pk_column_count", 0) > 0,
            adaptive_thresholds=adaptive_thresholds,
        )

def _detect_bridge_table_pattern(
    table_meta: Dict[str, Any],
    all_tables_meta: Dict[str, Dict[str, Any]],
) -> Dict[str, Any]:
    """
    Detect if a table follows bridge/junction table patterns for many-to-many relationships.

    Args:
        table_meta: Metadata for the table being analyzed
        all_tables_meta: Metadata for all tables in the schema

    Returns:
        Dict with bridge table analysis results
    """
    table_name = ""
    for table, meta in all_tables_meta.items():
        if meta == table_meta:
            table_name = table
            break

    if not table_name:
        return {"is_bridge": False, "confidence": 0.0, "connected_tables": []}

    columns_meta = table_meta.get("columns", {})
    pk_candidates = table_meta.get("pk_candidates", {})

    # Rule 1: Bridge table characteristics
    total_columns = len(columns_meta)
    if total_columns == 0:
        return {"is_bridge": False, "confidence": 0.0, "connected_tables": []}

    # Count foreign key-like columns
    fk_like_columns = []
    id_columns = []
    descriptive_columns = 0

    for col_name, col_info in columns_meta.items():
        names = col_info.get("names", [])
        if not names:
            continue

        original_name = names[0]
        base_type = col_info.get("base_type", "")

        # Check if column looks like an ID/foreign key
        if any(pattern in original_name.upper() for pattern in ["_ID", "ID", "_KEY", "KEY"]):
            id_columns.append(original_name)

            # Check if this could be a foreign key to another table
            for other_table_name, other_meta in all_tables_meta.items():
                if other_table_name == table_name:
                    continue

                if _looks_like_foreign_key(table_name, other_table_name, original_name):
                    fk_like_columns.append({
                        "column": original_name,
                        "references_table": other_table_name,
                        "confidence": 0.8
                    })
                    break

                # Check if column name contains the other table name
                other_variants = _table_variants(other_table_name)
                col_tokens = _identifier_tokens(original_name)

                for variant in other_variants:
                    if variant in col_tokens:
                        fk_like_columns.append({
                            "column": original_name,
                            "references_table": other_table_name,
                            "confidence": 0.6
                        })
                        break
        else:
            # Count descriptive/non-ID columns
            if base_type in DIMENSION_DATATYPES:
                descriptive_columns += 1

    # Rule 2: Bridge table scoring
    fk_ratio = len(fk_like_columns) / total_columns if total_columns > 0 else 0
    id_ratio = len(id_columns) / total_columns if total_columns > 0 else 0

    # Bridge table criteria:
    # - High ratio of foreign key columns (>= 50%)
    # - Small number of total columns (typically 2-6)
    # - Low number of descriptive columns
    # - No clear single primary key (composite key pattern)

    confidence = 0.0

    # High FK ratio is a strong indicator
    if fk_ratio >= 0.7:
        confidence += 0.4
    elif fk_ratio >= 0.5:
        confidence += 0.3
    elif fk_ratio >= 0.3:
        confidence += 0.2

    # Size factor - bridge tables are typically small
    if total_columns <= 4:
        confidence += 0.3
    elif total_columns <= 6:
        confidence += 0.2
    elif total_columns <= 8:
        confidence += 0.1

    # Few descriptive columns
    if descriptive_columns == 0:
        confidence += 0.2
    elif descriptive_columns <= 1:
        confidence += 0.1

    # Composite key pattern (multiple PKs or IDs)
    if len(pk_candidates) >= 2 or len(id_columns) >= 2:
        confidence += 0.1

    # Name-based heuristics
    table_upper = table_name.upper()
    bridge_keywords = {
        "BRIDGE", "JUNCTION", "LINK", "ASSOC", "ASSOCIATION", "REL", "RELATIONSHIP",
        "MAP", "MAPPING", "XREF", "CROSS_REF", "CONNECTOR"
    }

    for keyword in bridge_keywords:
        if keyword in table_upper:
            confidence += 0.2
            break

    # Multi-word table names that combine other table names (e.g., CUSTOMER_ORDER)
    table_tokens = _identifier_tokens(table_name)
    if len(table_tokens) >= 2:
        matching_tables = []
        for token in table_tokens:
            for other_table_name in all_tables_meta.keys():
                if other_table_name != table_name:
                    other_variants = _table_variants(other_table_name)
                    if token in other_variants:
                        matching_tables.append(other_table_name)

        if len(set(matching_tables)) >= 2:
            confidence += 0.15

    # Normalize confidence
    confidence = min(confidence, 1.0)

    is_bridge = confidence >= 0.6  # Threshold for bridge table classification

    connected_tables = [fk["references_table"] for fk in fk_like_columns if fk["confidence"] >= 0.5]

    return {
        "is_bridge": is_bridge,
        "confidence": confidence,
        "connected_tables": connected_tables,
        "fk_like_columns": fk_like_columns,
        "fk_ratio": fk_ratio,
        "id_ratio": id_ratio,
        "total_columns": total_columns,
        "descriptive_columns": descriptive_columns
    }


def _detect_many_to_many_relationships(
    raw_tables: List[tuple[data_types.FQNParts, data_types.Table]],
    metadata: Dict[str, Dict[str, Any]],
    existing_relationships: List[semantic_model_pb2.Relationship]
) -> List[semantic_model_pb2.Relationship]:
    """
    Detect many-to-many relationships through bridge table analysis.

    Args:
        raw_tables: List of raw table metadata
        metadata: Processed table metadata
        existing_relationships: Already detected relationships

    Returns:
        List of additional many-to-many relationships
    """
    many_to_many_relationships = []

    # Analyze each table for bridge patterns
    bridge_tables = {}

    for table_name, table_meta in metadata.items():
        bridge_analysis = _detect_bridge_table_pattern(table_meta, metadata)

        if bridge_analysis["is_bridge"] and len(bridge_analysis["connected_tables"]) >= 2:
            bridge_tables[table_name] = bridge_analysis

            logger.debug(
                f"Detected bridge table: {table_name} "
                f"(confidence: {bridge_analysis['confidence']:.2f}, "
                f"connects: {bridge_analysis['connected_tables']})"
            )

    # Create many-to-many relationships through bridge tables
    for bridge_table, bridge_info in bridge_tables.items():
        connected_tables = bridge_info["connected_tables"]

        # Create relationships for each pair of connected tables
        for i in range(len(connected_tables)):
            for j in range(i + 1, len(connected_tables)):
                left_table = connected_tables[i]
                right_table = connected_tables[j]

                # Skip if direct relationship already exists
                existing_pair_names = {
                    (rel.left_table, rel.right_table) for rel in existing_relationships
                } | {
                    (rel.right_table, rel.left_table) for rel in existing_relationships
                }

                if (left_table, right_table) in existing_pair_names:
                    continue

                # Find the foreign key columns in the bridge table
                left_fk_cols = []
                right_fk_cols = []

                for fk_info in bridge_info["fk_like_columns"]:
                    if fk_info["references_table"] == left_table and fk_info["confidence"] >= 0.5:
                        left_fk_cols.append(fk_info["column"])
                    elif fk_info["references_table"] == right_table and fk_info["confidence"] >= 0.5:
                        right_fk_cols.append(fk_info["column"])

                if not left_fk_cols or not right_fk_cols:
                    continue

                # Create many-to-many relationship
                relationship = semantic_model_pb2.Relationship(
                    name=f"{left_table}_to_{right_table}_via_{bridge_table}",
                    left_table=left_table,
                    right_table=right_table,
                    join_type=semantic_model_pb2.JoinType.inner,  # Bridge tables typically require INNER JOINs
                    relationship_type=semantic_model_pb2.RelationshipType.many_to_many,
                )

                # For now, we'll document the bridge table in the relationship name
                # In a more advanced implementation, we could add bridge table support to the protobuf schema

                # Try to infer the actual FK relationships
                # This is a simplified approach - in practice, you'd need more sophisticated FK detection
                if left_fk_cols and right_fk_cols:
                    # Use the first detected FK columns as a representative
                    relationship.relationship_columns.append(
                        semantic_model_pb2.RelationKey(
                            left_column=left_fk_cols[0],  # This is actually in the bridge table
                            right_column=right_fk_cols[0],  # This is also in the bridge table
                        )
                    )

                many_to_many_relationships.append(relationship)

                logger.debug(
                    f"Created many-to-many relationship: {left_table} <-> {right_table} via {bridge_table}"
                )

    return many_to_many_relationships


def _calculate_relationship_confidence(
    left_table: str,
    right_table: str,
    column_pairs: List[tuple[str, str]],
    left_meta: Dict[str, Any],
    right_meta: Dict[str, Any],
    left_has_pk: bool,
    right_has_pk: bool,
    left_values: List[str],
    right_values: List[str],
    cardinality_result: tuple[str, str],
    join_type: int,
    adaptive_thresholds: Optional[Dict[str, float]] = None,
) -> Dict[str, Any]:
    """
    Calculate confidence score and reasoning for relationship inference.

    Args:
        left_table: Name of the left table
        right_table: Name of the right table
        column_pairs: Column pairs in the relationship
        left_meta: Left table metadata
        right_meta: Right table metadata
        left_has_pk: Whether left side has primary key
        right_has_pk: Whether right side has primary key
        left_values: Sample values from left column
        right_values: Sample values from right column
        cardinality_result: Inferred cardinality tuple
        join_type: Inferred join type
        adaptive_thresholds: Adaptive thresholds used

    Returns:
        Dict with confidence score and detailed reasoning
    """
    confidence_score = 0.0
    reasoning_factors = []
    evidence_details = {}

    # Factor 1: Primary key metadata evidence (highest confidence)
    if left_has_pk or right_has_pk:
        pk_confidence = 0.4
        confidence_score += pk_confidence
        if left_has_pk and right_has_pk:
            reasoning_factors.append("Both sides have primary key metadata (very strong evidence)")
            evidence_details["pk_evidence"] = "both_pk"
        elif right_has_pk:
            reasoning_factors.append("Right side has primary key metadata (strong evidence)")
            evidence_details["pk_evidence"] = "right_pk"
        elif left_has_pk:
            reasoning_factors.append("Left side has primary key metadata (strong evidence)")
            evidence_details["pk_evidence"] = "left_pk"

    # Factor 2: Name similarity and pattern matching
    if column_pairs:
        total_name_similarity = 0.0
        for left_col, right_col in column_pairs:
            similarity = _name_similarity(left_col, right_col)
            total_name_similarity += similarity

        avg_name_similarity = total_name_similarity / len(column_pairs)
        evidence_details["avg_name_similarity"] = avg_name_similarity

        if avg_name_similarity >= 0.9:
            name_confidence = 0.25
            reasoning_factors.append(f"Very high column name similarity ({avg_name_similarity:.2f})")
        elif avg_name_similarity >= 0.7:
            name_confidence = 0.2
            reasoning_factors.append(f"High column name similarity ({avg_name_similarity:.2f})")
        elif avg_name_similarity >= 0.5:
            name_confidence = 0.15
            reasoning_factors.append(f"Moderate column name similarity ({avg_name_similarity:.2f})")
        elif avg_name_similarity >= 0.3:
            name_confidence = 0.1
            reasoning_factors.append(f"Low column name similarity ({avg_name_similarity:.2f})")
        else:
            name_confidence = 0.05
            reasoning_factors.append(f"Very low column name similarity ({avg_name_similarity:.2f})")

        confidence_score += name_confidence

        # Check for foreign key naming patterns
        fk_pattern_confidence = 0.0
        for left_col, right_col in column_pairs:
            if _looks_like_foreign_key(left_table, right_table, left_col):
                fk_pattern_confidence += 0.1
                reasoning_factors.append(f"Column '{left_col}' follows FK naming pattern")

        confidence_score += min(fk_pattern_confidence, 0.2)

    # Factor 3: Sample data uniqueness consistency
    if left_values and right_values:
        sample_size = min(len(left_values), len(right_values))
        evidence_details["sample_size"] = sample_size

        if sample_size >= 20:  # Sufficient sample size
            left_non_null = [v for v in left_values if not _is_nullish(v)]
            right_non_null = [v for v in right_values if not _is_nullish(v)]

            if left_non_null and right_non_null:
                left_unique_ratio = len(set(left_non_null)) / len(left_non_null)
                right_unique_ratio = len(set(right_non_null)) / len(right_non_null)

                evidence_details["left_unique_ratio"] = left_unique_ratio
                evidence_details["right_unique_ratio"] = right_unique_ratio

                # Check if uniqueness pattern matches inferred cardinality
                left_card, right_card = cardinality_result
                uniqueness_threshold = adaptive_thresholds.get("uniqueness_threshold", 0.95) if adaptive_thresholds else 0.95

                cardinality_consistency = False
                if left_card == "1" and left_unique_ratio > uniqueness_threshold:
                    cardinality_consistency = True
                elif left_card in ("*", "+") and left_unique_ratio <= uniqueness_threshold:
                    cardinality_consistency = True

                if right_card == "1" and right_unique_ratio > uniqueness_threshold:
                    cardinality_consistency = cardinality_consistency and True
                elif right_card in ("*", "+") and right_unique_ratio <= uniqueness_threshold:
                    cardinality_consistency = cardinality_consistency and True

                if cardinality_consistency:
                    uniqueness_confidence = 0.2
                    reasoning_factors.append("Sample uniqueness patterns support inferred cardinality")
                else:
                    uniqueness_confidence = 0.1
                    reasoning_factors.append("Sample uniqueness patterns partially support cardinality")

                confidence_score += uniqueness_confidence
        else:
            reasoning_factors.append(f"Limited sample size ({sample_size}) reduces confidence")

    # Factor 4: Data type compatibility
    if column_pairs and left_meta and right_meta:
        type_compatible_count = 0
        for left_col, right_col in column_pairs:
            # Get column types
            left_col_key = _sanitize_identifier_name(left_col)
            right_col_key = _sanitize_identifier_name(right_col)

            left_col_meta = left_meta.get("columns", {}).get(left_col_key, {})
            right_col_meta = right_meta.get("columns", {}).get(right_col_key, {})

            left_type = left_col_meta.get("base_type", "")
            right_type = right_col_meta.get("base_type", "")

            if left_type == right_type:
                type_compatible_count += 1

        type_compatibility_ratio = type_compatible_count / len(column_pairs)
        evidence_details["type_compatibility_ratio"] = type_compatibility_ratio

        if type_compatibility_ratio == 1.0:
            type_confidence = 0.1
            reasoning_factors.append("All column types are compatible")
        elif type_compatibility_ratio >= 0.8:
            type_confidence = 0.08
            reasoning_factors.append("Most column types are compatible")
        elif type_compatibility_ratio >= 0.5:
            type_confidence = 0.05
            reasoning_factors.append("Some column types are compatible")
        else:
            type_confidence = 0.0
            reasoning_factors.append("Column type compatibility is questionable")

        confidence_score += type_confidence

    # Factor 5: Table role consistency (business logic)
    left_role = _detect_table_role(left_table, left_meta)
    right_role = _detect_table_role(right_table, right_meta)
    evidence_details["left_table_role"] = left_role
    evidence_details["right_table_role"] = right_role

    relationship_context = _get_business_relationship_context(left_table, right_table, left_role, right_role)
    evidence_details["relationship_context"] = relationship_context

    if relationship_context in ["fact_to_dimension", "dimension_to_fact"]:
        role_confidence = 0.15
        reasoning_factors.append(f"Strong business relationship pattern: {relationship_context}")
    elif relationship_context in ["dimension_hierarchy", "bridge_relationship"]:
        role_confidence = 0.1
        reasoning_factors.append(f"Valid business relationship pattern: {relationship_context}")
    elif relationship_context == "fact_to_fact":
        role_confidence = 0.05
        reasoning_factors.append("Unusual but possible fact-to-fact relationship")
    else:
        role_confidence = 0.0
        reasoning_factors.append("Unclear business relationship pattern")

    confidence_score += role_confidence

    # Factor 6: Multiple column relationships (composite keys)
    if len(column_pairs) > 1:
        composite_confidence = 0.1
        reasoning_factors.append(f"Multi-column relationship ({len(column_pairs)} columns) increases confidence")
        confidence_score += composite_confidence

    # Normalize confidence score to 0-1 range
    confidence_score = min(confidence_score, 1.0)

    # Determine confidence level category
    if confidence_score >= 0.8:
        confidence_level = "very_high"
        confidence_description = "Very High Confidence"
    elif confidence_score >= 0.6:
        confidence_level = "high"
        confidence_description = "High Confidence"
    elif confidence_score >= 0.4:
        confidence_level = "medium"
        confidence_description = "Medium Confidence"
    elif confidence_score >= 0.2:
        confidence_level = "low"
        confidence_description = "Low Confidence"
    else:
        confidence_level = "very_low"
        confidence_description = "Very Low Confidence"

    return {
        "confidence_score": confidence_score,
        "confidence_level": confidence_level,
        "confidence_description": confidence_description,
        "reasoning_factors": reasoning_factors,
        "evidence_details": evidence_details,
        "inferred_cardinality": f"{cardinality_result[0]}:{cardinality_result[1]}",
        "join_type": "INNER" if join_type == semantic_model_pb2.JoinType.inner else "LEFT_OUTER",
        "column_count": len(column_pairs),
    }


def _get_domain_knowledge_patterns() -> Dict[str, Any]:
    """
    Define common data warehouse patterns and domain knowledge.

    Returns:
        Dict containing predefined patterns and business rules
    """
    return {
        # Common business entity patterns
        "business_entities": {
            "customer": {
                "table_patterns": ["CUSTOMER", "CUST", "CLIENT", "ACCOUNT_HOLDER", "USER", "MEMBER"],
                "pk_patterns": ["CUSTOMER_ID", "CUST_ID", "CLIENT_ID", "USER_ID", "MEMBER_ID"],
                "typical_attributes": ["NAME", "EMAIL", "PHONE", "ADDRESS", "STATUS", "TYPE", "SEGMENT"],
                "role": "dimension"
            },
            "product": {
                "table_patterns": ["PRODUCT", "ITEM", "SKU", "INVENTORY", "CATALOG"],
                "pk_patterns": ["PRODUCT_ID", "ITEM_ID", "SKU", "PRODUCT_KEY"],
                "typical_attributes": ["NAME", "DESCRIPTION", "CATEGORY", "PRICE", "BRAND", "STATUS"],
                "role": "dimension"
            },
            "order": {
                "table_patterns": ["ORDER", "TRANSACTION", "SALE", "PURCHASE"],
                "pk_patterns": ["ORDER_ID", "TRANSACTION_ID", "SALE_ID", "ORDER_NUMBER"],
                "typical_attributes": ["DATE", "AMOUNT", "STATUS", "QUANTITY", "TOTAL"],
                "role": "fact"
            },
            "date": {
                "table_patterns": ["DATE", "TIME", "CALENDAR", "DIM_DATE"],
                "pk_patterns": ["DATE_ID", "DATE_KEY", "TIME_ID"],
                "typical_attributes": ["YEAR", "MONTH", "DAY", "QUARTER", "WEEK", "WEEKDAY"],
                "role": "dimension"
            },
            "location": {
                "table_patterns": ["LOCATION", "GEOGRAPHY", "ADDRESS", "REGION", "TERRITORY"],
                "pk_patterns": ["LOCATION_ID", "GEO_ID", "ADDRESS_ID", "REGION_ID"],
                "typical_attributes": ["COUNTRY", "STATE", "CITY", "ZIP", "LATITUDE", "LONGITUDE"],
                "role": "dimension"
            },
            "employee": {
                "table_patterns": ["EMPLOYEE", "STAFF", "WORKER", "PERSONNEL"],
                "pk_patterns": ["EMPLOYEE_ID", "STAFF_ID", "EMP_ID"],
                "typical_attributes": ["NAME", "DEPARTMENT", "TITLE", "MANAGER", "HIRE_DATE"],
                "role": "dimension"
            }
        },

        # Common relationship patterns in data warehouses
        "relationship_patterns": {
            "star_schema": {
                "pattern": "fact_to_dimension",
                "confidence_boost": 0.2,
                "description": "Standard star schema fact-to-dimension relationship"
            },
            "snowflake_schema": {
                "pattern": "dimension_hierarchy",
                "confidence_boost": 0.15,
                "description": "Snowflake schema dimension hierarchy"
            },
            "bridge_table": {
                "pattern": "many_to_many_via_bridge",
                "confidence_boost": 0.1,
                "description": "Many-to-many relationship through bridge table"
            },
            "time_dimension": {
                "pattern": "temporal_relationship",
                "confidence_boost": 0.25,
                "description": "Time-based relationship (very common in warehouses)"
            }
        },

        # Known FK patterns that often appear in real data warehouses
        "common_fk_patterns": {
            "customer_references": [
                "CUSTOMER_ID", "CUST_ID", "CLIENT_ID", "ACCOUNT_ID", "USER_ID"
            ],
            "product_references": [
                "PRODUCT_ID", "ITEM_ID", "SKU", "PROD_ID", "CATALOG_ID"
            ],
            "date_references": [
                "DATE_ID", "ORDER_DATE_ID", "SHIP_DATE_ID", "CREATE_DATE_ID",
                "TRANSACTION_DATE_ID", "DATE_KEY"
            ],
            "location_references": [
                "LOCATION_ID", "ADDRESS_ID", "SHIP_TO_ID", "BILL_TO_ID",
                "WAREHOUSE_ID", "STORE_ID"
            ]
        },

        # Table naming conventions that indicate specific patterns
        "naming_conventions": {
            "fact_indicators": [
                "FACT_", "FCT_", "F_", "SALES_", "ORDERS_", "TRANSACTIONS_",
                "REVENUE_", "METRICS_", "EVENTS_", "ACTIVITY_"
            ],
            "dimension_indicators": [
                "DIM_", "D_", "REF_", "LKP_", "LOOKUP_", "MASTER_"
            ],
            "bridge_indicators": [
                "BRG_", "BRIDGE_", "XREF_", "MAP_", "ASSOC_", "LINK_"
            ],
            "staging_indicators": [
                "STG_", "STAGING_", "TMP_", "TEMP_", "RAW_", "LANDING_"
            ]
        }
    }


def _apply_domain_knowledge(
    left_table: str,
    right_table: str,
    column_pairs: List[tuple[str, str]],
    left_meta: Dict[str, Any],
    right_meta: Dict[str, Any],
    current_confidence: float,
) -> Dict[str, Any]:
    """
    Apply domain knowledge to enhance relationship inference confidence.

    Args:
        left_table: Name of the left table
        right_table: Name of the right table
        column_pairs: Column pairs in the relationship
        left_meta: Left table metadata
        right_meta: Right table metadata
        current_confidence: Current confidence score

    Returns:
        Dict with enhanced confidence and domain knowledge insights
    """
    domain_patterns = _get_domain_knowledge_patterns()
    enhancement_factors = []
    confidence_boost = 0.0

    # Factor 1: Recognize business entity patterns
    left_entity = _identify_business_entity(left_table, left_meta, domain_patterns)
    right_entity = _identify_business_entity(right_table, right_meta, domain_patterns)

    if left_entity and right_entity:
        # Check for common business relationship patterns
        entity_pair = f"{left_entity}-{right_entity}"
        common_pairs = {
            "order-customer": 0.25,
            "order-product": 0.25,
            "order-date": 0.3,
            "customer-location": 0.2,
            "product-location": 0.15,
            "employee-location": 0.15,
            "order-employee": 0.2,
        }

        # Check both directions
        if entity_pair in common_pairs:
            boost = common_pairs[entity_pair]
            confidence_boost += boost
            enhancement_factors.append(f"Recognized common business pattern: {entity_pair} (+{boost:.2f})")
        elif f"{right_entity}-{left_entity}" in common_pairs:
            boost = common_pairs[f"{right_entity}-{left_entity}"]
            confidence_boost += boost
            enhancement_factors.append(f"Recognized common business pattern: {right_entity}-{left_entity} (+{boost:.2f})")

    # Factor 2: Check for standard FK naming patterns
    for left_col, right_col in column_pairs:
        fk_pattern_match = _check_standard_fk_patterns(left_col, right_col, domain_patterns)
        if fk_pattern_match:
            confidence_boost += 0.15
            enhancement_factors.append(f"Standard FK pattern detected: {fk_pattern_match}")

    # Factor 3: Table naming convention analysis
    left_convention = _identify_naming_convention(left_table, domain_patterns)
    right_convention = _identify_naming_convention(right_table, domain_patterns)

    if left_convention and right_convention:
        # Boost confidence for expected patterns
        if (left_convention == "fact" and right_convention == "dimension") or \
           (left_convention == "dimension" and right_convention == "fact"):
            confidence_boost += 0.2
            enhancement_factors.append("Standard fact-dimension naming pattern (+0.20)")
        elif left_convention == "dimension" and right_convention == "dimension":
            confidence_boost += 0.1
            enhancement_factors.append("Dimension hierarchy naming pattern (+0.10)")

    # Factor 4: Time dimension special handling
    if _is_time_dimension_pattern(right_table, right_meta, domain_patterns):
        confidence_boost += 0.2
        enhancement_factors.append("Time dimension relationship (very common) (+0.20)")

    # Factor 5: Schema pattern recognition (star vs snowflake)
    schema_pattern = _detect_schema_pattern(left_table, right_table, left_meta, right_meta, domain_patterns)
    if schema_pattern:
        pattern_boost = domain_patterns["relationship_patterns"][schema_pattern]["confidence_boost"]
        confidence_boost += pattern_boost
        pattern_desc = domain_patterns["relationship_patterns"][schema_pattern]["description"]
        enhancement_factors.append(f"Schema pattern: {pattern_desc} (+{pattern_boost:.2f})")

    # Apply the boost but cap the final confidence at 1.0
    enhanced_confidence = min(current_confidence + confidence_boost, 1.0)

    return {
        "enhanced_confidence": enhanced_confidence,
        "confidence_boost": confidence_boost,
        "domain_factors": enhancement_factors,
        "left_entity": left_entity,
        "right_entity": right_entity,
        "left_convention": left_convention,
        "right_convention": right_convention,
        "schema_pattern": schema_pattern,
    }


def _identify_business_entity(table_name: str, table_meta: Dict[str, Any], domain_patterns: Dict[str, Any]) -> Optional[str]:
    """Identify what business entity a table represents."""
    table_upper = table_name.upper()
    business_entities = domain_patterns["business_entities"]

    for entity_type, entity_info in business_entities.items():
        # Check table name patterns
        for pattern in entity_info["table_patterns"]:
            if pattern in table_upper:
                return entity_type

        # Check primary key patterns
        pk_candidates = table_meta.get("pk_candidates", {})
        for pk_pattern in entity_info["pk_patterns"]:
            for pk_norm in pk_candidates.keys():
                if pk_pattern.replace("_", "").upper() in pk_norm.replace("_", "").upper():
                    return entity_type

    return None


def _check_standard_fk_patterns(left_col: str, right_col: str, domain_patterns: Dict[str, Any]) -> Optional[str]:
    """Check if column pair matches standard FK patterns."""
    common_fks = domain_patterns["common_fk_patterns"]

    left_upper = left_col.upper()
    right_upper = right_col.upper()

    for pattern_type, patterns in common_fks.items():
        for pattern in patterns:
            if pattern in left_upper or pattern in right_upper:
                return f"{pattern_type}: {pattern}"

    return None


def _identify_naming_convention(table_name: str, domain_patterns: Dict[str, Any]) -> Optional[str]:
    """Identify the naming convention used for a table."""
    table_upper = table_name.upper()
    naming_conventions = domain_patterns["naming_conventions"]

    for convention_type, indicators in naming_conventions.items():
        for indicator in indicators:
            if table_upper.startswith(indicator) or indicator in table_upper:
                return convention_type.replace("_indicators", "")

    return None


def _is_time_dimension_pattern(table_name: str, table_meta: Dict[str, Any], domain_patterns: Dict[str, Any]) -> bool:
    """Check if table follows time dimension patterns."""
    table_upper = table_name.upper()
    time_patterns = domain_patterns["business_entities"]["date"]["table_patterns"]

    for pattern in time_patterns:
        if pattern in table_upper:
            return True

    # Check for typical time dimension attributes
    time_attributes = ["YEAR", "MONTH", "DAY", "QUARTER", "WEEK", "DATE"]
    columns = table_meta.get("columns", {})

    time_attr_count = 0
    for col_info in columns.values():
        names = col_info.get("names", [])
        if names:
            col_name = names[0].upper()
            for time_attr in time_attributes:
                if time_attr in col_name:
                    time_attr_count += 1
                    break

    # If more than 30% of columns are time-related, likely a time dimension
    total_columns = len(columns)
    if total_columns > 0 and time_attr_count / total_columns > 0.3:
        return True

    return False


def _detect_schema_pattern(
    left_table: str,
    right_table: str,
    left_meta: Dict[str, Any],
    right_meta: Dict[str, Any],
    domain_patterns: Dict[str, Any]
) -> Optional[str]:
    """Detect common schema patterns (star, snowflake, etc.)."""
    left_role = _detect_table_role(left_table, left_meta)
    right_role = _detect_table_role(right_table, right_meta)

    # Star schema pattern: fact table to dimension
    if (left_role == "fact" and right_role == "dimension") or \
       (left_role == "dimension" and right_role == "fact"):
        return "star_schema"

    # Snowflake schema pattern: dimension to dimension
    if left_role == "dimension" and right_role == "dimension":
        return "snowflake_schema"

    # Time dimension pattern (very common)
    if _is_time_dimension_pattern(right_table, right_meta, domain_patterns) or \
       _is_time_dimension_pattern(left_table, left_meta, domain_patterns):
        return "time_dimension"

    # Bridge table pattern
    if left_role == "bridge" or right_role == "bridge":
        return "bridge_table"

    return None


def _calculate_adaptive_thresholds(
    values_list: List[List[str]],
    table_count: int = 2,
    base_sample_size: int = 10,
) -> Dict[str, float]:
    """
    Calculate adaptive thresholds based on data characteristics.

    Args:
        values_list: List of sample value lists from multiple columns
        table_count: Number of tables in the schema
        base_sample_size: Base sample size from configuration

    Returns:
        Dict with adaptive threshold values
    """
    if not values_list or not any(values_list):
        return {
            "min_sample_size": 50,
            "uniqueness_threshold": 0.95,
            "similarity_threshold": 0.6,
        }

    # Calculate sample statistics
    sample_sizes = [len(vals) for vals in values_list if vals]
    max_sample_size = max(sample_sizes) if sample_sizes else base_sample_size
    avg_sample_size = sum(sample_sizes) / len(sample_sizes) if sample_sizes else base_sample_size

    # Calculate data distribution characteristics
    total_unique_values = 0
    total_values = 0
    skew_ratios = []

    for values in values_list:
        if not values:
            continue

        unique_count = len(set(str(v) for v in values if not _is_nullish(v)))
        total_count = len([v for v in values if not _is_nullish(v)])

        if total_count > 0:
            total_unique_values += unique_count
            total_values += total_count

            # Calculate value distribution skew
            value_counts = {}
            for v in values:
                if not _is_nullish(v):
                    val_str = str(v)
                    value_counts[val_str] = value_counts.get(val_str, 0) + 1

            if len(value_counts) > 1:
                max_freq = max(value_counts.values())
                min_freq = min(value_counts.values())
                skew = max_freq / min_freq if min_freq > 0 else float('inf')
                skew_ratios.append(skew)

    # Calculate overall uniqueness ratio
    overall_uniqueness = total_unique_values / total_values if total_values > 0 else 0
    avg_skew = sum(skew_ratios) / len(skew_ratios) if skew_ratios else 1.0

    # Adaptive threshold calculation
    thresholds = {}

    # 1. Minimum sample size: adjust based on data characteristics
    base_min_size = 50

    # Reduce threshold for highly unique data (likely keys)
    if overall_uniqueness > 0.9:
        min_size_adj = 0.6  # More confident with unique data
    elif overall_uniqueness > 0.7:
        min_size_adj = 0.8
    else:
        min_size_adj = 1.0  # Keep conservative for low uniqueness

    # Adjust for data skew - more skewed data needs larger samples
    if avg_skew > 10:  # Highly skewed
        min_size_adj *= 1.5
    elif avg_skew > 5:  # Moderately skewed
        min_size_adj *= 1.2

    # Adjust for schema complexity - more tables = need more confidence
    if table_count > 10:
        min_size_adj *= 1.3
    elif table_count > 5:
        min_size_adj *= 1.1

    # Scale with base sample size from configuration
    size_scale_factor = min(max_sample_size / base_sample_size, 3.0) if base_sample_size > 0 else 1.0
    min_size_adj *= size_scale_factor

    thresholds["min_sample_size"] = max(int(base_min_size * min_size_adj), 10)

    # 2. Uniqueness threshold: adjust based on data characteristics
    base_uniqueness = 0.95

    # For highly skewed data, be more strict
    if avg_skew > 10:
        uniqueness_adj = 1.02  # Require higher uniqueness
    elif avg_skew > 5:
        uniqueness_adj = 1.01
    else:
        uniqueness_adj = 1.0

    # For small samples, be more conservative
    if avg_sample_size < 20:
        uniqueness_adj *= 1.02

    thresholds["uniqueness_threshold"] = min(base_uniqueness * uniqueness_adj, 0.99)

    # 3. Name similarity threshold: adjust based on naming patterns
    base_similarity = 0.6

    # If we have very structured naming (consistent prefixes/suffixes), be more lenient
    prefix_consistency = _calculate_naming_consistency(values_list)
    if prefix_consistency > 0.8:
        similarity_adj = 0.9  # Lower threshold for consistent naming
    elif prefix_consistency > 0.6:
        similarity_adj = 0.95
    else:
        similarity_adj = 1.0

    thresholds["similarity_threshold"] = max(base_similarity * similarity_adj, 0.4)

    return thresholds


def _calculate_naming_consistency(values_list: List[List[str]]) -> float:
    """
    Calculate consistency in naming patterns across sample values.

    Returns:
        float: Consistency score between 0.0 and 1.0
    """
    if not values_list:
        return 0.0

    # Look for common prefixes and suffixes patterns
    all_values = []
    for values in values_list:
        if values:
            all_values.extend([str(v).upper() for v in values if not _is_nullish(v)])

    if len(all_values) < 2:
        return 0.0

    # Check for common prefixes/suffixes
    prefix_patterns = {}
    suffix_patterns = {}

    for value in all_values:
        # Extract potential prefixes (first 1-3 chars)
        for i in range(1, min(4, len(value))):
            prefix = value[:i]
            prefix_patterns[prefix] = prefix_patterns.get(prefix, 0) + 1

        # Extract potential suffixes (last 1-3 chars)
        for i in range(1, min(4, len(value))):
            suffix = value[-i:]
            suffix_patterns[suffix] = suffix_patterns.get(suffix, 0) + 1

    # Calculate consistency scores
    total_values = len(all_values)
    max_prefix_freq = max(prefix_patterns.values()) if prefix_patterns else 0
    max_suffix_freq = max(suffix_patterns.values()) if suffix_patterns else 0

    prefix_consistency = max_prefix_freq / total_values
    suffix_consistency = max_suffix_freq / total_values

    return max(prefix_consistency, suffix_consistency)


def _infer_cardinality(
    left_values: List[str],
    right_values: List[str],
    left_is_pk: bool,
    right_is_pk: bool,
    *,
    adaptive_thresholds: Optional[Dict[str, float]] = None,
) -> tuple[str, str]:
    """
    Infer relationship cardinality based on uniqueness ratios with adaptive thresholds.

    Returns:
        tuple: (left_cardinality, right_cardinality) where values can be:
               "*" (many), "1" (one), "?" (zero or one), "+" (one or more)

    Note:
        Uses adaptive thresholds based on data characteristics to improve accuracy.
        Falls back to conservative defaults when data is insufficient or ambiguous.
    """
    # Use adaptive thresholds if provided, otherwise fall back to defaults
    if adaptive_thresholds:
        min_sample_size = int(adaptive_thresholds.get("min_sample_size", 50))
        uniqueness_threshold = adaptive_thresholds.get("uniqueness_threshold", 0.95)
    else:
        min_sample_size = 50
        uniqueness_threshold = 0.95

    # RULE 1: Trust explicit primary key metadata (highest priority)
    if right_is_pk and not left_is_pk:
        return ("*", "1")

    if left_is_pk and not right_is_pk:
        return ("1", "*")

    if left_is_pk and right_is_pk:
        return ("1", "1")

    # RULE 2: Use adaptive uniqueness heuristics with sufficient sample size
    if left_values and right_values:
        sample_size = min(len(left_values), len(right_values))

        # Apply adaptive minimum sample size
        if sample_size < min_sample_size:
            # Conservative default: assume many-to-one (most common pattern)
            # This prevents false 1:1 inference from small samples
            return ("*", "1")

        # Large enough sample: trust uniqueness ratios with adaptive threshold
        left_non_null = [v for v in left_values if not _is_nullish(v)]
        right_non_null = [v for v in right_values if not _is_nullish(v)]

        left_unique_ratio = len(set(left_non_null)) / len(left_non_null) if left_non_null else 0
        right_unique_ratio = len(set(right_non_null)) / len(right_non_null) if right_non_null else 0

        # Apply adaptive uniqueness threshold
        left_is_unique = left_unique_ratio > uniqueness_threshold
        right_is_unique = right_unique_ratio > uniqueness_threshold

        if left_is_unique and right_is_unique:
            return ("1", "1")
        elif right_is_unique:
            return ("*", "1")
        elif left_is_unique:
            return ("1", "*")

    # RULE 3: Default to many-to-one (most common case in data warehouses)
    return ("*", "1")


def _is_nullish(value: Any) -> bool:
    """
    Determines whether a sampled value should be treated as NULL/empty.
    Handles common ClickZetta sampling formats where NULL becomes NaN or an empty string.
    """
    if value is None:
        return True
    if isinstance(value, float) and math.isnan(value):
        return True
    text = str(value).strip()
    if not text:
        return True
    normalized = text.upper()
    return normalized in {"NULL", "NONE", "NAN", "NA", "N/A", "NOT AVAILABLE"}


def _column_has_null_via_query(
    session: Session,
    fqn: data_types.FQNParts,
    column_name: str,
    cache: Dict[Tuple[str, str, str, str], bool],
) -> bool:
    """
    Executes a targeted IS NULL query to conclusively detect nullable foreign keys.
    Results are cached per table/column to avoid redundant lookups.
    """
    key = (
        (fqn.database or "").upper(),
        (fqn.schema_name or "").upper(),
        (fqn.table or "").upper(),
        column_name.upper(),
    )
    if key in cache:
        return cache[key]

    if not column_name:
        cache[key] = False
        return False

    qualified_table = _qualified_table_name(fqn)
    column_identifier = _format_sql_identifier(column_name)
    if not column_identifier or not qualified_table:
        cache[key] = False
        return False

    query = (
        f"SELECT {column_identifier} FROM {qualified_table} "
        f"WHERE {column_identifier} IS NULL LIMIT 1"
    )

    try:
        df = session.sql(query).to_pandas()
        has_null = not df.empty
    except Exception as exc:  # pragma: no cover - best-effort diagnostic query
        logger.debug(
            "Strict null check query failed for %s.%s.%s.%s: %s",
            fqn.database,
            fqn.schema_name,
            fqn.table,
            column_name,
            exc,
        )
        has_null = False

    cache[key] = has_null
    return has_null


def _detect_table_role(table_name: str, columns_info: Dict[str, Any]) -> str:
    """
    Detect the semantic role of a table (fact, dimension, bridge, etc.).

    Args:
        table_name: Name of the table
        columns_info: Dictionary containing column metadata

    Returns:
        str: Table role ('fact', 'dimension', 'bridge', 'staging', 'unknown')
    """
    upper_name = table_name.upper()
    tokens = _identifier_tokens(table_name)

    # Rule 1: Explicit prefixes/suffixes
    fact_indicators = {"FACT", "FCT", "TXN", "TRANSACTION", "EVENT", "LOG", "SALES", "ORDER"}
    dim_indicators = {"DIM", "DIMENSION", "LOOKUP", "REF", "REFERENCE", "MASTER"}
    bridge_indicators = {"BRIDGE", "BRG", "LINK", "JUNCTION", "ASSOC", "ASSOCIATION"}
    staging_indicators = {"STG", "STAGING", "TMP", "TEMP", "WORK", "LANDING", "RAW"}

    # Check for explicit indicators in table name
    for token in tokens:
        if token in fact_indicators:
            return "fact"
        elif token in dim_indicators:
            return "dimension"
        elif token in bridge_indicators:
            return "bridge"
        elif token in staging_indicators:
            return "staging"

    # Rule 2: Analyze column patterns
    pk_count = len(columns_info.get("pk_candidates", {}))
    total_columns = len(columns_info.get("columns", {}))

    if total_columns == 0:
        return "unknown"

    # Count different column types
    measure_like_count = 0
    dimension_like_count = 0
    id_count = 0

    for col_info in columns_info.get("columns", {}).values():
        base_type = col_info.get("base_type", "")
        names = col_info.get("names", [])

        if names:
            col_name = names[0].upper()
            # Count ID/key columns
            if any(suffix in col_name for suffix in ["_ID", "ID", "_KEY", "KEY"]):
                id_count += 1

            # Count measure-like columns (amounts, counts, quantities)
            if any(word in col_name for word in ["AMOUNT", "QTY", "QUANTITY", "COUNT", "TOTAL", "SUM", "AVG"]):
                measure_like_count += 1
            elif base_type in MEASURE_DATATYPES and not col_info.get("is_identifier", False):
                measure_like_count += 1
            else:
                dimension_like_count += 1

    # Rule 3: Heuristic classification
    id_ratio = id_count / total_columns if total_columns > 0 else 0
    measure_ratio = measure_like_count / total_columns if total_columns > 0 else 0

    # Bridge table: mostly IDs with few other columns
    if id_ratio > 0.6 and total_columns <= 10:
        return "bridge"

    # Fact table: has measures and multiple foreign keys
    if measure_ratio > 0.2 and id_count >= 2:
        return "fact"

    # Dimension table: mostly descriptive attributes, single PK
    if pk_count == 1 and measure_ratio < 0.1:
        return "dimension"

    # Default to unknown if no clear pattern
    return "unknown"


def _get_business_relationship_context(left_table: str, right_table: str, left_role: str, right_role: str) -> str:
    """
    Determine business relationship context between tables based on their roles.

    Returns:
        str: Context hint for join type inference
    """
    # Fact to Dimension relationships
    if left_role == "fact" and right_role == "dimension":
        return "fact_to_dimension"
    elif left_role == "dimension" and right_role == "fact":
        return "dimension_to_fact"

    # Bridge table relationships
    elif left_role == "bridge" or right_role == "bridge":
        return "bridge_relationship"

    # Dimension to Dimension (hierarchy or lookup)
    elif left_role == "dimension" and right_role == "dimension":
        return "dimension_hierarchy"

    # Fact to Fact (rare but possible)
    elif left_role == "fact" and right_role == "fact":
        return "fact_to_fact"

    # Staging relationships
    elif "staging" in [left_role, right_role]:
        return "staging_relationship"

    return "unknown_relationship"


def _infer_join_type(
    left_table: str,
    right_table: str,
    left_card: str,
    right_card: str,
    left_is_pk: bool,
    right_is_pk: bool,
    left_values: List[str],
    right_values: List[str],
    *,
    has_null_fk: bool = False,
    left_table_meta: Optional[Dict[str, Any]] = None,
    right_table_meta: Optional[Dict[str, Any]] = None,
) -> int:
    """
    Enhanced JOIN type inference based on table roles and business semantics.

    Args:
        left_table: Name of the left table
        right_table: Name of the right table
        left_card: Left cardinality ("1", "*", etc.)
        right_card: Right cardinality ("1", "*", etc.)
        left_is_pk: Whether left column is a primary key
        right_is_pk: Whether right column is a primary key
        left_values: Sample values from left column
        right_values: Sample values from right column
        has_null_fk: Whether strict null detection found NULLs
        left_table_meta: Metadata about the left table
        right_table_meta: Metadata about the right table

    Returns:
        semantic_model_pb2.JoinType value (inner or left_outer)

    Enhanced Decision Logic:
        1. Strict null detection (highest priority)
        2. Business semantic rules based on table roles
        3. NULL detection in sample values
        4. Naming pattern heuristics
        5. Conservative INNER JOIN default
    """
    
    # RULE 1: Default to INNER JOIN (most common and safest)
    default_join = semantic_model_pb2.JoinType.inner

    # RULE 2: Strict mode override - explicit null detection via SQL (highest priority)
    if has_null_fk:
        reason = "strict null probe"
        logger.debug(
            f"Join type inference for {left_table} -> {right_table}: "
            f"LEFT_OUTER ({reason})"
        )
        return semantic_model_pb2.JoinType.left_outer

    # RULE 3: Enhanced business semantics based on table roles
    left_role = "unknown"
    right_role = "unknown"
    relationship_context = "unknown_relationship"

    if left_table_meta and right_table_meta:
        left_role = _detect_table_role(left_table, left_table_meta)
        right_role = _detect_table_role(right_table, right_table_meta)
        relationship_context = _get_business_relationship_context(
            left_table, right_table, left_role, right_role
        )

        # Apply business rules based on relationship context
        if relationship_context == "fact_to_dimension":
            # Fact → Dimension: usually INNER, but check for optional dimensions
            if any(keyword in right_table.upper() for keyword in [
                "PROMO", "PROMOTION", "DISCOUNT", "COUPON", "OPTIONAL", "SECONDARY"
            ]):
                logger.debug(
                    f"Join type inference for {left_table} -> {right_table}: "
                    f"LEFT_OUTER (fact to optional dimension: {right_role})"
                )
                return semantic_model_pb2.JoinType.left_outer

        elif relationship_context == "dimension_hierarchy":
            # Dimension → Dimension: often optional (parent-child, category hierarchies)
            if left_card in ("*", "+") and right_card == "1":
                logger.debug(
                    f"Join type inference for {left_table} -> {right_table}: "
                    f"LEFT_OUTER (dimension hierarchy: child to parent)"
                )
                return semantic_model_pb2.JoinType.left_outer

        elif relationship_context == "bridge_relationship":
            # Bridge tables: typically INNER to maintain referential integrity
            logger.debug(
                f"Join type inference for {left_table} -> {right_table}: "
                f"INNER (bridge table relationship)"
            )
            return semantic_model_pb2.JoinType.inner

        elif relationship_context == "staging_relationship":
            # Staging tables: often have incomplete data, prefer LEFT OUTER
            logger.debug(
                f"Join type inference for {left_table} -> {right_table}: "
                f"LEFT_OUTER (staging table relationship)"
            )
            return semantic_model_pb2.JoinType.left_outer

    # RULE 4: Check for NULL values in foreign key (left side) using samples
    if left_values and left_card in ("*", "+"):
        sample_window = left_values[: min(len(left_values), 25)]
        has_nulls = any(_is_nullish(val) for val in sample_window)
        if has_nulls:
            logger.debug(
                f"Join type inference for {left_table} -> {right_table}: "
                f"LEFT_OUTER (detected NULL values in FK column)"
            )
            return semantic_model_pb2.JoinType.left_outer

    # RULE 5: Naming pattern heuristics for optional relationships
    left_upper = left_table.upper()
    right_upper = right_table.upper()
    optional_keywords = {
        "OPTIONAL", "ALTERNATE", "SECONDARY", "BACKUP", "FALLBACK",
        "PROMO", "PROMOTION", "DISCOUNT", "COUPON", "TEMP", "TMP"
    }

    for keyword in optional_keywords:
        if keyword in right_upper:
            logger.debug(
                f"Join type inference for {left_table} -> {right_table}: "
                f"LEFT_OUTER (optional relationship pattern: {keyword})"
            )
            return semantic_model_pb2.JoinType.left_outer

    # RULE 6: Time dimension special case - often required
    if any(time_hint in right_upper for time_hint in ["DATE", "TIME", "CALENDAR"]):
        logger.debug(
            f"Join type inference for {left_table} -> {right_table}: "
            f"INNER (time dimension - typically required)"
        )
        return semantic_model_pb2.JoinType.inner

    # RULE 7: Default to INNER JOIN
    # Most conservative choice that ensures data integrity
    logger.debug(
        f"Join type inference for {left_table} -> {right_table}: "
        f"INNER (default - roles: {left_role}->{right_role}, context: {relationship_context})"
    )
    return default_join


def _looks_like_foreign_key(fk_table: str, pk_table: str, fk_column: str) -> bool:
    """
    Enhanced heuristic to detect if a column looks like a foreign key.
    Checks patterns like: customer_id, cust_id, customer_key, etc.
    """
    fk_upper = fk_column.strip().upper()
    pk_table_variants = _table_variants(pk_table)
    
    # Pattern 1: {table_name}_id or {table_name}_key
    for variant in pk_table_variants:
        if fk_upper in {f"{variant}_ID", f"{variant}ID", f"{variant}_KEY", f"{variant}KEY"}:
            return True
    
    # Pattern 2: Column ends with table name variants
    tokens = _identifier_tokens(fk_column)
    if len(tokens) >= 2:
        # e.g., order_customer_id -> check if CUSTOMER is in pk_table_variants
        for i in range(len(tokens) - 1):
            if tokens[i] in pk_table_variants:
                tail = tokens[-1]
                if tail in {"ID", "KEY"}:
                    return True
    
    # Pattern 3: Similar to primary key column but with FK table prefix
    # e.g., order_id in order_items table referencing orders.id
    fk_table_variants = _table_variants(fk_table)
    for fk_variant in fk_table_variants:
        if fk_upper.startswith(fk_variant):
            remainder = fk_upper[len(fk_variant):].lstrip("_")
            for pk_variant in pk_table_variants:
                if remainder.startswith(pk_variant):
                    return True
    
    return False


def _suggest_filters(raw_table: data_types.Table) -> List[semantic_model_pb2.NamedFilter]:
    suggestions: List[semantic_model_pb2.NamedFilter] = []
    for col in raw_table.columns:
        base_type = _base_type_from_type(col.column_type)
        values = col.values or []
        distinct_values: List[str] = []
        for value in values:
            if value not in distinct_values:
                distinct_values.append(value)
        if _is_time_like_column(col):
            expr = f"{col.column_name} >= DATEADD('day', -30, CURRENT_DATE())"
            suggestions.append(
                semantic_model_pb2.NamedFilter(
                    name=f"{col.column_name}_last_30_days",
                    synonyms=[_PLACEHOLDER_COMMENT],
                    description=_PLACEHOLDER_COMMENT,
                    expr=expr,
                )
            )
        if 1 < len(distinct_values) <= 5:
            upper_name = col.column_name.upper()
            is_identifier_like = upper_name.endswith(("ID", "_ID", "KEY", "_KEY"))

            categorical_suffixes = (
                "STATUS",
                "FLAG",
                "TYPE",
                "PRIORITY",
                "SEGMENT",
                "CATEGORY",
                "MODE",
                "CODE",
                "LEVEL",
            )
            is_textual = base_type in {"STRING", "TEXT", "VARCHAR", "CHAR", "CHARACTER"}
            is_boolean = base_type in {"BOOLEAN"}
            is_categorical_numeric = base_type in {"INT", "INTEGER", "NUMBER", "SMALLINT", "BIGINT"} and any(
                upper_name.endswith(suffix) for suffix in categorical_suffixes
            )

            if not is_identifier_like and (is_textual or is_boolean or is_categorical_numeric):
                formatted = [_format_literal(val, base_type) for val in distinct_values[:5]]
                expr = f"{col.column_name} IN ({', '.join(formatted)})"
                suggestions.append(
                    semantic_model_pb2.NamedFilter(
                        name=f"{col.column_name}_include_values",
                        synonyms=[_PLACEHOLDER_COMMENT],
                        description=_PLACEHOLDER_COMMENT,
                        expr=expr,
                    )
                )
    return suggestions


def _infer_relationships(
    raw_tables: List[tuple[data_types.FQNParts, data_types.Table]],
    *,
    session: Optional[Session] = None,
    strict_join_inference: bool = False,
) -> List[semantic_model_pb2.Relationship]:
    relationships: List[semantic_model_pb2.Relationship] = []
    if not raw_tables:
        return relationships

    metadata = {}
    prefix_counter: Dict[str, int] = {}
    for _, raw_table in raw_tables:
        for column in raw_table.columns:
            tokens = _identifier_tokens(column.column_name)
            if tokens:
                prefix = tokens[0]
                prefix_counter[prefix] = prefix_counter.get(prefix, 0) + 1
    global_prefixes = {
        prefix
        for prefix, count in prefix_counter.items()
        if len(prefix) <= 3 and count >= 2
    }

    metadata = {}
    for fqn, raw_table in raw_tables:
        columns_meta: Dict[str, Dict[str, Any]] = {}
        pk_candidates: Dict[str, List[str]] = defaultdict(list)
        table_prefixes = global_prefixes | _table_prefixes(raw_table.name)
        for column in raw_table.columns:
            base_type = _base_type_from_type(column.column_type)
            normalized = _sanitize_identifier_name(column.column_name, prefixes_to_drop=table_prefixes)
            entry = columns_meta.setdefault(
                normalized,
                {
                    "names": [],
                    "base_type": base_type,
                    "values": [],
                    "is_identifier": False,
                    "is_primary": False,
                },
            )
            entry["base_type"] = base_type
            entry["names"].append(column.column_name)
            if column.values:
                entry["values"].extend(column.values)
            entry["is_identifier"] = entry["is_identifier"] or _is_identifier_like(column.column_name, base_type)
            is_primary = getattr(column, "is_primary_key", False)
            if is_primary:
                entry["is_primary"] = True
                if column.column_name not in pk_candidates[normalized]:
                    pk_candidates[normalized].append(column.column_name)
                continue
            if _looks_like_primary_key(raw_table.name, column.column_name):
                pk_candidates[normalized].append(column.column_name)
        metadata[raw_table.name] = {
            "fqn": fqn,
            "columns": columns_meta,
            "pk_candidates": pk_candidates,
        }

    pairs: dict[tuple[str, str], List[tuple[str, str]]] = {}
    null_check_cache: Dict[Tuple[str, str, str, str], bool] = {}

    def _record_pair(left_table: str, right_table: str, left_col: str, right_col: str) -> None:
        key = (left_table, right_table)
        value = (left_col, right_col)
        if value not in pairs.setdefault(key, []):
            pairs[key].append(value)

    table_names = list(metadata.keys())
    for i in range(len(table_names)):
        for j in range(i + 1, len(table_names)):
            table_a_name = table_names[i]
            table_b_name = table_names[j]
            table_a = metadata[table_a_name]
            table_b = metadata[table_b_name]

            shared = set(table_a["columns"].keys()) & set(table_b["columns"].keys())
            for col_key in shared:
                meta_a = table_a["columns"][col_key]
                meta_b = table_b["columns"][col_key]
                if meta_a["base_type"] != meta_b["base_type"]:
                    continue
                in_pk_a = col_key in table_a["pk_candidates"]
                in_pk_b = col_key in table_b["pk_candidates"]
                if in_pk_a and not in_pk_b:
                    _record_pair(
                        table_b_name,
                        table_a_name,
                        meta_b["names"][0],
                        meta_a["names"][0],
                    )
                elif in_pk_b and not in_pk_a:
                    _record_pair(
                        table_a_name,
                        table_b_name,
                        meta_a["names"][0],
                        meta_b["names"][0],
                    )
                elif in_pk_a and in_pk_b:
                    pk_count_a = len(table_a["pk_candidates"])
                    pk_count_b = len(table_b["pk_candidates"])
                    if pk_count_a >= 2 and pk_count_b == 1:
                        _record_pair(
                            table_a_name,
                            table_b_name,
                            meta_a["names"][0],
                            meta_b["names"][0],
                        )
                    elif pk_count_b >= 2 and pk_count_a == 1:
                        _record_pair(
                            table_b_name,
                            table_a_name,
                            meta_b["names"][0],
                            meta_a["names"][0],
                        )

            # Enhanced suffix-based matches with name similarity
            # Pattern 1: Direct suffix match (e.g. order_date_id -> date_id)
            for pk_norm, pk_cols in table_a["pk_candidates"].items():
                pk_meta = table_a["columns"].get(pk_norm)
                if not pk_meta:
                    continue
                for norm_b, meta_b in table_b["columns"].items():
                    if meta_b["base_type"] != pk_meta["base_type"]:
                        continue
                    if norm_b == pk_norm:
                        continue
                    
                    # Direct suffix match
                    if norm_b.endswith(pk_norm):
                        _record_pair(
                            table_b_name,
                            table_a_name,
                            meta_b["names"][0],
                            pk_cols[0],
                        )
                        continue
                    
                    # Enhanced: Check if column looks like a foreign key to this table
                    if _looks_like_foreign_key(table_b_name, table_a_name, meta_b["names"][0]):
                        # Additional check: name similarity with adaptive threshold
                        similarity = _name_similarity(norm_b, pk_norm)
                        # Calculate adaptive threshold for this relationship
                        all_sample_values = []
                        for col_values in [pk_meta.get("values", []), meta_b.get("values", [])]:
                            if col_values:
                                all_sample_values.append(col_values)

                        adaptive_thresholds = _calculate_adaptive_thresholds(
                            all_sample_values,
                            table_count=len(raw_tables),
                            base_sample_size=len(pk_meta.get("values", [])) if pk_meta.get("values") else 10,
                        )
                        similarity_threshold = adaptive_thresholds.get("similarity_threshold", 0.6)

                        if similarity >= similarity_threshold:
                            _record_pair(
                                table_b_name,
                                table_a_name,
                                meta_b["names"][0],
                                pk_cols[0],
                            )

            # Pattern 2: Reverse direction
            for pk_norm, pk_cols in table_b["pk_candidates"].items():
                pk_meta = table_b["columns"].get(pk_norm)
                if not pk_meta:
                    continue
                for norm_a, meta_a in table_a["columns"].items():
                    if meta_a["base_type"] != pk_meta["base_type"]:
                        continue
                    if norm_a == pk_norm:
                        continue
                    
                    # Direct suffix match
                    if norm_a.endswith(pk_norm):
                        _record_pair(
                            table_a_name,
                            table_b_name,
                            meta_a["names"][0],
                            pk_cols[0],
                        )
                        continue
                    
                    # Enhanced: Check if column looks like a foreign key to this table
                    if _looks_like_foreign_key(table_a_name, table_b_name, meta_a["names"][0]):
                        # Additional check: name similarity with adaptive threshold
                        similarity = _name_similarity(norm_a, pk_norm)
                        # Calculate adaptive threshold for this relationship
                        all_sample_values = []
                        for col_values in [pk_meta.get("values", []), meta_a.get("values", [])]:
                            if col_values:
                                all_sample_values.append(col_values)

                        adaptive_thresholds = _calculate_adaptive_thresholds(
                            all_sample_values,
                            table_count=len(raw_tables),
                            base_sample_size=len(pk_meta.get("values", [])) if pk_meta.get("values") else 10,
                        )
                        similarity_threshold = adaptive_thresholds.get("similarity_threshold", 0.6)

                        if similarity >= similarity_threshold:
                            _record_pair(
                                table_a_name,
                                table_b_name,
                                meta_a["names"][0],
                                pk_cols[0],
                            )

    # Calculate global adaptive thresholds for the entire schema
    all_schema_values = []
    for table_meta in metadata.values():
        for col_meta in table_meta["columns"].values():
            if col_meta.get("values"):
                all_schema_values.append(col_meta["values"])

    global_adaptive_thresholds = _calculate_adaptive_thresholds(
        all_schema_values,
        table_count=len(raw_tables),
        base_sample_size=10,  # Default base
    )

    # Build relationships with inferred cardinality
    for (left_table, right_table), column_pairs in pairs.items():
        # Infer cardinality based on available metadata
        left_meta = metadata[left_table]
        right_meta = metadata[right_table]
        
        # Determine if tables have primary keys in the relationship
        left_has_pk = any(
            col_name in [pair[0] for pair in column_pairs]
            for pk_list in left_meta["pk_candidates"].values()
            for col_name in pk_list
        )
        right_has_pk = any(
            col_name in [pair[1] for pair in column_pairs]
            for pk_list in right_meta["pk_candidates"].values()
            for col_name in pk_list
        )
        
        # Enhanced: Get sample values for all columns in the relationship (for composite key analysis)
        left_values_all = []
        right_values_all = []
        left_values = []  # First column only (for backward compatibility)
        right_values = []

        for left_col, right_col in column_pairs:
            left_col_key = _sanitize_identifier_name(
                left_col,
                prefixes_to_drop=global_prefixes | _table_prefixes(left_table)
            )
            right_col_key = _sanitize_identifier_name(
                right_col,
                prefixes_to_drop=global_prefixes | _table_prefixes(right_table)
            )

            left_col_values = []
            right_col_values = []

            if left_col_key in left_meta["columns"]:
                left_col_values = left_meta["columns"][left_col_key].get("values") or []
            if right_col_key in right_meta["columns"]:
                right_col_values = right_meta["columns"][right_col_key].get("values") or []

            left_values_all.append(left_col_values)
            right_values_all.append(right_col_values)

            # Keep first column values for backward compatibility
            if not left_values:
                left_values = left_col_values
            if not right_values:
                right_values = right_col_values

        # Enhanced cardinality inference: Use composite key analysis if multiple columns
        if len(column_pairs) > 1:
            left_card, right_card = _infer_composite_cardinality(
                left_meta,
                right_meta,
                column_pairs,
                left_values_all,
                right_values_all,
            )
        else:
            # Fallback to single-column analysis with adaptive thresholds
            left_card, right_card = _infer_cardinality(
                left_values,
                right_values,
                left_has_pk,
                right_has_pk,
                adaptive_thresholds=global_adaptive_thresholds,
            )
        
        # Determine if SQL null probe should be executed for stricter inference
        strict_fk_detected = False
        if strict_join_inference and session:
            left_fqn_parts = left_meta.get("fqn")
            if isinstance(left_fqn_parts, data_types.FQNParts):
                strict_fk_detected = any(
                    _column_has_null_via_query(
                        session,
                        left_fqn_parts,
                        left_column,
                        null_check_cache,
                    )
                    for left_column, _ in column_pairs
                )

        # Infer join type based on relationship characteristics
        join_type = _infer_join_type(
            left_table,
            right_table,
            left_card,
            right_card,
            left_has_pk,
            right_has_pk,
            left_values,
            right_values,
            has_null_fk=strict_fk_detected,
            left_table_meta=left_meta,
            right_table_meta=right_meta,
        )
        
        # Calculate confidence and reasoning for this relationship
        confidence_analysis = _calculate_relationship_confidence(
            left_table=left_table,
            right_table=right_table,
            column_pairs=column_pairs,
            left_meta=left_meta,
            right_meta=right_meta,
            left_has_pk=left_has_pk,
            right_has_pk=right_has_pk,
            left_values=left_values,
            right_values=right_values,
            cardinality_result=(left_card, right_card),
            join_type=join_type,
            adaptive_thresholds=global_adaptive_thresholds,
        )

        # Apply domain knowledge to enhance confidence
        domain_enhancement = _apply_domain_knowledge(
            left_table=left_table,
            right_table=right_table,
            column_pairs=column_pairs,
            left_meta=left_meta,
            right_meta=right_meta,
            current_confidence=confidence_analysis['confidence_score']
        )

        # Update confidence analysis with domain knowledge
        if domain_enhancement['confidence_boost'] > 0:
            confidence_analysis['confidence_score'] = min(1.0,
                confidence_analysis['confidence_score'] + domain_enhancement['confidence_boost'])

            # Add domain knowledge factors to reasoning
            for domain_factor in domain_enhancement['domain_factors']:
                confidence_analysis['reasoning_factors'].append(f"Domain knowledge: {domain_factor}")

            # Update confidence level based on new score
            if confidence_analysis['confidence_score'] >= 0.8:
                confidence_analysis['confidence_level'] = 'very_high'
                confidence_analysis['confidence_description'] = 'Very High Confidence'
            elif confidence_analysis['confidence_score'] >= 0.6:
                confidence_analysis['confidence_level'] = 'high'
                confidence_analysis['confidence_description'] = 'High Confidence'
            elif confidence_analysis['confidence_score'] >= 0.4:
                confidence_analysis['confidence_level'] = 'medium'
                confidence_analysis['confidence_description'] = 'Medium Confidence'
            elif confidence_analysis['confidence_score'] >= 0.2:
                confidence_analysis['confidence_level'] = 'low'
                confidence_analysis['confidence_description'] = 'Low Confidence'
            else:
                confidence_analysis['confidence_level'] = 'very_low'
                confidence_analysis['confidence_description'] = 'Very Low Confidence'

        # Enhanced logging with confidence and reasoning
        sample_info = f"samples: L={len(left_values)}, R={len(right_values)}"
        pk_info = f"PKs: L={left_has_pk}, R={right_has_pk}"
        join_type_name = "INNER" if join_type == semantic_model_pb2.JoinType.inner else "LEFT_OUTER"
        confidence_info = f"confidence: {confidence_analysis['confidence_score']:.2f} ({confidence_analysis['confidence_level']})"

        # Add domain knowledge info if applied
        domain_info = ""
        if domain_enhancement['confidence_boost'] > 0:
            domain_info = f", domain boost: +{domain_enhancement['confidence_boost']:.2f}"

        logger.info(
            f"Relationship inference for {left_table} -> {right_table}: "
            f"{left_card}:{right_card}, JOIN={join_type_name}, {confidence_info}{domain_info} "
            f"({sample_info}, {pk_info})"
        )

        # Log domain knowledge patterns if detected
        domain_factors = [f for f in confidence_analysis['reasoning_factors'] if f.startswith("Domain knowledge:")]
        if domain_factors:
            logger.debug(f"Domain patterns detected for {left_table} -> {right_table}: {domain_factors}")

        # Log detailed reasoning for medium or lower confidence relationships
        if confidence_analysis['confidence_score'] < 0.6:
            logger.debug(f"Confidence reasoning for {left_table} -> {right_table}:")
            for factor in confidence_analysis['reasoning_factors']:
                logger.debug(f"  - {factor}")

        # Log very high confidence relationships with their evidence
        elif confidence_analysis['confidence_score'] >= 0.8:
            logger.debug(f"High confidence relationship {left_table} -> {right_table} based on:")
            for factor in confidence_analysis['reasoning_factors'][:3]:  # Top 3 factors
                logger.debug(f"  + {factor}")
        
        # Determine relationship type based on cardinality
        if left_card == "1" and right_card == "1":
            rel_type = semantic_model_pb2.RelationshipType.one_to_one
        elif left_card in ("*", "+") and right_card == "1":
            rel_type = semantic_model_pb2.RelationshipType.many_to_one
        elif left_card == "1" and right_card in ("*", "+"):
            rel_type = semantic_model_pb2.RelationshipType.one_to_many
        else:
            # Default to many_to_one for backward compatibility
            rel_type = semantic_model_pb2.RelationshipType.many_to_one
        
        relationship = semantic_model_pb2.Relationship(
            name=f"{left_table}_to_{right_table}",
            left_table=left_table,
            right_table=right_table,
            join_type=join_type,  # Use inferred join type instead of hardcoded inner
            relationship_type=rel_type,
        )
        for left_column, right_column in column_pairs:
            relationship.relationship_columns.append(
                semantic_model_pb2.RelationKey(
                    left_column=left_column, right_column=right_column
                )
            )
        relationships.append(relationship)

    # Phase 2: Detect many-to-many relationships through bridge table analysis
    many_to_many_relationships = _detect_many_to_many_relationships(
        raw_tables, metadata, relationships
    )

    if many_to_many_relationships:
        relationships.extend(many_to_many_relationships)
        logger.info(f"Detected {len(many_to_many_relationships)} many-to-many relationships via bridge tables")

    logger.info(f"Inferred {len(relationships)} total relationships across {len(raw_tables)} tables")
    return relationships


def _raw_table_to_semantic_context_table(
    database: str, schema: str, raw_table: data_types.Table
) -> semantic_model_pb2.Table:
    """
    Converts a raw table representation to a semantic model table in protobuf format.

    Args:
        database (str): The name of the database containing the table.
        schema (str): The name of the schema containing the table.
        raw_table (data_types.Table): The raw table object to be transformed.

    Returns:
        semantic_model_pb2.Table: A protobuf representation of the semantic table.

    This function categorizes table columns into TimeDimensions, Dimensions, or Measures based on their data type,
    populates them with sample values, and sets placeholders for descriptions and filters.
    """

    # For each column, decide if it is a TimeDimension, Measure, or Dimension column.
    # For now, we decide this based on datatype.
    # Any time datatype, is TimeDimension.
    # Any varchar/text is Dimension.
    # Any numerical column is Measure.

    time_dimensions = []
    dimensions = []
    facts: List[semantic_model_pb2.Fact] = []
    used_identifier_names: set[str] = set()
    table_prefixes = _table_prefixes(raw_table.name)

    for col in raw_table.columns:
        base_type = _base_type_from_type(col.column_type)
        if _is_time_like_column(col):
            time_data_type = col.column_type
            if time_data_type.split("(")[0].upper() in {"STRING", "VARCHAR", "TEXT", "CHAR", "CHARACTER", "NVARCHAR"}:
                time_data_type = "TIMESTAMP_NTZ"
            time_dimension_name = _safe_semantic_identifier(
                col.column_name,
                used_identifier_names,
                "time_dimension",
                prefixes_to_drop=table_prefixes,
            )
            time_dimensions.append(
                semantic_model_pb2.TimeDimension(
                    name=time_dimension_name,
                    expr=col.column_name,
                    data_type=time_data_type,
                    sample_values=col.values,
                    synonyms=[_PLACEHOLDER_COMMENT],
                    description=col.comment if col.comment else _PLACEHOLDER_COMMENT,
                )
            )

        elif base_type in DIMENSION_DATATYPES:
            dimension_name = _safe_semantic_identifier(
                col.column_name,
                used_identifier_names,
                "dimension",
                prefixes_to_drop=table_prefixes,
            )
            dimensions.append(
                semantic_model_pb2.Dimension(
                    name=dimension_name,
                    expr=col.column_name,
                    data_type=col.column_type,
                    sample_values=col.values,
                    synonyms=[_PLACEHOLDER_COMMENT],
                    description=col.comment if col.comment else _PLACEHOLDER_COMMENT,
                )
            )

        elif base_type in MEASURE_DATATYPES:
            if _is_identifier_like(col.column_name, base_type):
                identifier_dimension_name = _safe_semantic_identifier(
                    col.column_name,
                    used_identifier_names,
                    "dimension",
                    prefixes_to_drop=table_prefixes,
                )
                dimensions.append(
                    semantic_model_pb2.Dimension(
                        name=identifier_dimension_name,
                        expr=col.column_name,
                        data_type=col.column_type,
                        sample_values=col.values,
                        synonyms=[_PLACEHOLDER_COMMENT],
                        description=col.comment if col.comment else _PLACEHOLDER_COMMENT,
                    )
                )
                continue
            fact_name = _safe_semantic_identifier(
                col.column_name,
                used_identifier_names,
                "fact",
                prefixes_to_drop=table_prefixes,
            )
            facts.append(
                semantic_model_pb2.Fact(
                    name=fact_name,
                    expr=col.column_name,
                    data_type=col.column_type,
                    sample_values=col.values,
                    synonyms=[_PLACEHOLDER_COMMENT],
                    description=col.comment if col.comment else _PLACEHOLDER_COMMENT,
                )
            )
        elif base_type in OBJECT_DATATYPES:
            logger.warning(
                f"""We don't currently support {col.column_type} as an input column datatype to the Semantic Model. We are skipping column {col.column_name} for now."""
            )
            continue
        else:
            fallback_dimension_name = _safe_semantic_identifier(
                col.column_name,
                used_identifier_names,
                "dimension",
                prefixes_to_drop=table_prefixes,
            )
            logger.warning(
                f"Column datatype does not map to a known datatype. Input was = {col.column_type}. We are going to place as a Dimension for now."
            )
            dimensions.append(
                semantic_model_pb2.Dimension(
                    name=fallback_dimension_name,
                    expr=col.column_name,
                    data_type=col.column_type,
                    sample_values=col.values,
                    synonyms=[_PLACEHOLDER_COMMENT],
                    description=col.comment if col.comment else _PLACEHOLDER_COMMENT,
                )
            )
    if len(time_dimensions) + len(dimensions) + len(facts) == 0:
        raise ValueError(
            f"No valid columns found for table {raw_table.name}. Please verify that this table contains column's datatypes not in {OBJECT_DATATYPES}."
        )

    filters = _suggest_filters(raw_table)

    return semantic_model_pb2.Table(
        name=raw_table.name,
        base_table=semantic_model_pb2.FullyQualifiedTable(
            database=database, schema=schema, table=raw_table.name
        ),
        # For fields we can not automatically infer, leave a comment for the user to fill out.
        description=raw_table.comment if raw_table.comment else _PLACEHOLDER_COMMENT,
        filters=filters,
        dimensions=dimensions,
        time_dimensions=time_dimensions,
        facts=facts,
    )


def raw_schema_to_semantic_context(
    base_tables: List[str],
    semantic_model_name: str,
    conn: Session,
    n_sample_values: int = _DEFAULT_N_SAMPLE_VALUES_PER_COL,
    allow_joins: Optional[bool] = True,
    strict_join_inference: bool = False,
    llm_custom_prompt: str = "",
    enrich_with_llm: bool = False,
    progress_callback: Optional[Callable[[str], None]] = None,
) -> semantic_model_pb2.SemanticModel:
    """
    Converts a list of fully qualified ClickZetta table names into a semantic model.

    Parameters:
    - base_tables  (list[str]): Fully qualified table names to include in the semantic model.
    - semantic_model_name (str): A meaningful semantic model name.
    - conn (Session): ClickZetta session to reuse.
    - n_sample_values (int): The number of sample values per col.
    - allow_joins (bool | None): Whether to infer table relationships.
    - strict_join_inference (bool): If True, runs additional `IS NULL` probes per relationship to tighten join-type detection (requires extra SQL queries).
    - llm_custom_prompt (str): Optional user instructions forwarded to the DashScope enrichment step.

    Returns:
    - The semantic model (semantic_model_pb2.SemanticModel).

    This function fetches metadata for the specified tables, performs schema validation, extracts key information,
    enriches metadata from ClickZetta, and constructs a semantic model in protobuf format.
    It handles different workspaces and schemas within the same account by reusing the provided session.

    Raises:
    - AssertionError: If no valid tables are found in the specified schema.
    """

    # For FQN tables, the connector handles cross-schema access automatically.
    def _notify(message: str) -> None:
        if progress_callback:
            try:
                progress_callback(message)
            except Exception:
                logger.debug("Progress callback failed for message: {}", message)

    table_objects = []
    raw_tables_metadata: List[tuple[data_types.FQNParts, data_types.Table]] = []
    unique_database_schema: List[str] = []
    for table in base_tables:
        # Verify this is a valid FQN table. For now, we check that the table follows the following format.
        # {database}.{schema}.{table}
        fqn_table = create_fqn_table(table)
        fqn_databse_schema = f"{fqn_table.database}.{fqn_table.schema_name}"

        if fqn_databse_schema not in unique_database_schema:
            unique_database_schema.append(fqn_databse_schema)

        logger.info(f"Pulling column information from {fqn_table}")
        _notify(f"Fetching metadata for {fqn_table.database}.{fqn_table.schema_name}.{fqn_table.table}...")
        valid_schemas_tables_columns_df = get_valid_schemas_tables_columns_df(
            session=conn,
            workspace=fqn_table.database,
            table_schema=fqn_table.schema_name,
            table_names=[fqn_table.table],
        )
        if valid_schemas_tables_columns_df.empty:
            raise ValueError(
                (
                    "Unable to retrieve column metadata for table "
                    f"{fqn_table.database}.{fqn_table.schema_name}.{fqn_table.table}. "
                    "Shared or external catalogs (category=SHARED/EXTERNAL) do not expose information_schema views; "
                    "please copy the data into a managed workspace or provide manual metadata."
                )
            )

        # get the valid columns for this table.
        valid_columns_df_this_table = valid_schemas_tables_columns_df[
            valid_schemas_tables_columns_df["TABLE_NAME"] == fqn_table.table
        ]

        raw_table = get_table_representation(
            session=conn,
            workspace=fqn_table.database,
            schema_name=fqn_table.schema_name,
            table_name=fqn_table.table,  # Non-qualified table name
            table_index=0,
            ndv_per_column=n_sample_values,  # number of sample values to pull per column.
            columns_df=valid_columns_df_this_table,
            max_workers=1,
        )
        table_object = _raw_table_to_semantic_context_table(
            database=fqn_table.database,
            schema=fqn_table.schema_name,
            raw_table=raw_table,
        )
        table_objects.append(table_object)
        raw_tables_metadata.append((fqn_table, raw_table))
    # TODO(jhilgart): Call cortex model to generate a semantically friendly name here.

    relationships: List[semantic_model_pb2.Relationship] = []
    if allow_joins:
        relationships = _infer_relationships(
            raw_tables_metadata,
            session=conn if strict_join_inference else None,
            strict_join_inference=strict_join_inference,
        )

    context = semantic_model_pb2.SemanticModel(
        name=semantic_model_name,
        tables=table_objects,
        relationships=relationships,
    )
    context.description = _PLACEHOLDER_COMMENT
    context.custom_instructions = _PLACEHOLDER_COMMENT

    if enrich_with_llm:
        settings = get_dashscope_settings()
        if settings:
            actual_model = "qwen-plus-latest"
            logger.info(
                "Running DashScope enrichment for semantic model '{}' using model '{}'",
                semantic_model_name,
                actual_model,
            )
            _notify("Running DashScope enrichment to enhance descriptions and metrics...")

            # Create progress tracker for enrichment
            def enrichment_progress_callback(update):
                stage_messages = {
                    EnrichmentStage.TABLE_ENRICHMENT: "Enriching table descriptions and metrics",
                    EnrichmentStage.MODEL_DESCRIPTION: "Generating model description",
                    EnrichmentStage.MODEL_METRICS: "Generating model-level metrics",
                    EnrichmentStage.VERIFIED_QUERIES: "Generating verified queries",
                    EnrichmentStage.COMPLETE: "Enrichment complete"
                }

                base_message = stage_messages.get(update.stage, "Processing")
                if update.table_name:
                    message = f"{base_message} - {update.table_name} ({update.current_step}/{update.total_steps})"
                elif update.total_steps > 1:
                    message = f"{base_message} ({update.current_step}/{update.total_steps})"
                else:
                    message = base_message

                if update.details and update.details.get("message"):
                    message += f": {update.details['message']}"
                elif update.message:
                    message += f": {update.message}"

                _notify(f"[{update.percentage:.0f}%] {message}")

            progress_tracker = EnrichmentProgressTracker(enrichment_progress_callback)

            settings = DashscopeSettings(
                api_key=settings.api_key,
                model=actual_model,
                base_url=settings.base_url,
                temperature=settings.temperature,
                top_p=settings.top_p,
                max_output_tokens=settings.max_output_tokens,
                timeout_seconds=settings.timeout_seconds,
            )
            client = DashscopeClient(settings)
            enrich_semantic_model(
                context,
                raw_tables_metadata,
                client,
                placeholder=_PLACEHOLDER_COMMENT,
                custom_prompt=llm_custom_prompt,
                session=conn,
                progress_tracker=progress_tracker,
            )
            _notify("DashScope enrichment complete.")
        else:
            logger.warning("LLM enrichment was requested but DashScope is not configured; skipping enrichment.")
            _notify("DashScope configuration missing; skipped enrichment.")
    return context


def append_comment_to_placeholders(yaml_str: str) -> str:
    """
    Finds all instances of a specified placeholder in a YAML string and appends a given text to these placeholders.
    This is the homework to fill out after your yaml is generated.

    Parameters:
    - yaml_str (str): The YAML string to process.

    Returns:
    - str: The modified YAML string with appended text to placeholders.
    """
    updated_yaml = []
    # Split the string into lines to process each line individually
    lines = yaml_str.split("\n")

    for line in lines:
        # Check if the placeholder is in the current line.
        # Strip the last quote to match.
        if line.rstrip("'").endswith(_PLACEHOLDER_COMMENT):
            # Replace the _PLACEHOLDER_COMMENT with itself plus the append_text
            updated_line = line + _FILL_OUT_TOKEN
            updated_yaml.append(updated_line)
        elif line.rstrip("'").endswith(AUTOGEN_TOKEN):
            updated_line = line + _AUTOGEN_COMMENT_TOKEN
            updated_yaml.append(updated_line)
        # Add comments to specific fields in certain sections.
        elif line.lstrip().startswith("join_type"):
            updated_yaml.append(line)
        elif line.lstrip().startswith("relationship_type"):
            updated_yaml.append(line)
        else:
            updated_yaml.append(line)

    # Join the lines back together into a single string
    return "\n".join(updated_yaml)


def _to_snake_case(s: str) -> str:
    """
    Convert a string into snake case.

    Parameters:
    s (str): The string to convert.

    Returns:
    str: The snake case version of the string.
    """
    # Replace common delimiters with spaces
    s = s.replace("-", " ").replace("_", " ")

    words = s.split(" ")

    # Convert each word to lowercase and join with underscores
    snake_case_str = "_".join([word.lower() for word in words if word]).strip()

    return snake_case_str


def generate_base_semantic_model_from_clickzetta(
    base_tables: List[str],
    conn: Session,
    semantic_model_name: str,
    n_sample_values: int = _DEFAULT_N_SAMPLE_VALUES_PER_COL,
    output_yaml_path: Optional[str] = None,
) -> None:
    """
    Generates a base semantic context from specified ClickZetta tables and exports it to a YAML file.

    Parameters:
        base_tables : Fully qualified names of ClickZetta tables to include in the semantic context.
        conn: ClickZetta session to reuse.
        semantic_model_name: The human readable model name. This should be semantically meaningful to an organization.
        output_yaml_path: Path for the output YAML file. If None, defaults to 'semantic_model_generator/output_models/YYYYMMDDHHMMSS_<semantic_model_name>.yaml'.
        n_sample_values: The number of sample values to populate for all columns.

    Returns:
        None. Writes the semantic context to a YAML file.
    """
    formatted_datetime = datetime.now().strftime("%Y%m%d%H%M%S")
    if not output_yaml_path:
        file_name = f"{formatted_datetime}_{_to_snake_case(semantic_model_name)}.yaml"
        if os.path.exists("semantic_model_generator/output_models"):
            write_path = f"semantic_model_generator/output_models/{file_name}"
        else:
            write_path = f"./{file_name}"
    else:  # Assume user gives correct path.
        write_path = output_yaml_path

    yaml_str = generate_model_str_from_clickzetta(
        base_tables,
        n_sample_values=n_sample_values if n_sample_values > 0 else 1,
        semantic_model_name=semantic_model_name,
        conn=conn,
    )

    with open(write_path, "w") as f:
        # Clarify that the YAML was autogenerated and that placeholders should be filled out/deleted.
        f.write(_AUTOGEN_COMMENT_WARNING)
        f.write(yaml_str)

    logger.info(f"Semantic model saved to {write_path}")

    return None


def generate_model_str_from_clickzetta(
    base_tables: List[str],
    semantic_model_name: str,
    conn: Session,
    n_sample_values: int = _DEFAULT_N_SAMPLE_VALUES_PER_COL,
    allow_joins: Optional[bool] = True,
    strict_join_inference: bool = False,
    enrich_with_llm: bool = False,
    llm_custom_prompt: str = "",
    progress_callback: Optional[Callable[[str], None]] = None,
) -> str:
    """
    Generates a base semantic context from specified ClickZetta tables and returns the raw string.

    Parameters:
        base_tables : Fully qualified names of ClickZetta tables to include in the semantic context.
        semantic_model_name: The human readable model name. This should be semantically meaningful to an organization.
        conn: ClickZetta session to reuse.
        n_sample_values: The number of sample values to populate for all columns.
        allow_joins: Whether to allow joins in the semantic context.
        strict_join_inference: When True, executes additional null-probe SQL queries to improve join type accuracy.
        llm_custom_prompt: Optional user instructions forwarded to the DashScope enrichment step.
        progress_callback: Optional callable invoked with human-readable progress updates.

    Returns:
        str: The raw string of the semantic context.
    """
    def _notify(message: str) -> None:
        if progress_callback:
            try:
                progress_callback(message)
            except Exception:
                logger.debug("Progress callback failed for message: {}", message)

    table_list = ", ".join(base_tables)
    logger.info("Generating semantic model '{}' from tables: {}", semantic_model_name, table_list)
    _notify("Collecting metadata from ClickZetta tables...")

    context = raw_schema_to_semantic_context(
        base_tables,
        n_sample_values=n_sample_values if n_sample_values > 0 else 1,
        semantic_model_name=semantic_model_name,
        allow_joins=allow_joins,
        strict_join_inference=strict_join_inference,
        llm_custom_prompt=llm_custom_prompt,
        enrich_with_llm=enrich_with_llm,
        conn=conn,
        progress_callback=_notify,
    )
    _notify("Constructing semantic model structure...")
    # Validate the generated yaml is within context limits.
    # We just throw a warning here to allow users to update.
    validate_context_length(context)
    _notify("Validating semantic model context length...")

    _notify("Converting semantic model to YAML...")
    yaml_str = proto_utils.proto_to_yaml(context)
    # Once we have the yaml, update to include to # <FILL-OUT> tokens.
    yaml_str = append_comment_to_placeholders(yaml_str)

    _notify("Semantic model generation complete.")
    logger.info("Semantic model '{}' generated successfully.", semantic_model_name)
    return yaml_str
