"""
SQLAlchemy-based materializer for Mock Spark lazy evaluation.

This module uses SQLAlchemy with DuckDB to provide SQL generation
and execution capabilities for complex DataFrame operations.
"""

# mypy: disable-error-code="arg-type"

from typing import Any, Dict, List, Tuple
from sqlalchemy import (
    create_engine,
    select,
    func,
    desc,
    asc,
    and_,
    or_,
    MetaData,
    Table,
    Column,
    Integer,
    String,
    Float,
    Double,
    Boolean,
    literal,
    insert,
    text,
)
from sqlalchemy.orm import Session
from mock_spark.spark_types import (
    MockStructType,
    MockRow,
)
from mock_spark.functions import MockColumn, MockColumnOperation, MockLiteral


class SQLAlchemyMaterializer:
    """Materializes lazy DataFrames using SQLAlchemy with DuckDB."""

    def __init__(self, engine_url: str = "duckdb:///:memory:"):
        # Create DuckDB engine with SQLAlchemy
        self.engine = create_engine(engine_url, echo=False)
        self._temp_table_counter = 0
        self._created_tables: Dict[str, Any] = {}  # Track created tables
        self.metadata = MetaData()

    def materialize(
        self, data: List[Dict[str, Any]], schema: MockStructType, operations: List[Tuple[str, Any]]
    ) -> List[MockRow]:
        """
        Materializes the DataFrame by building and executing operations using SQLAlchemy.
        """
        if not operations:
            # No operations to apply, return original data as rows
            return [MockRow(row) for row in data]

        # Create initial table with data
        current_table_name = f"temp_table_{self._temp_table_counter}"
        self._temp_table_counter += 1

        # Create table and insert data
        self._create_table_with_data(current_table_name, data)

        # Apply operations step by step
        temp_counter = 1
        for op_name, op_val in operations:
            next_table_name = f"temp_table_{self._temp_table_counter}_{temp_counter}"
            temp_counter += 1

            if op_name == "filter":
                self._apply_filter(current_table_name, next_table_name, op_val)
            elif op_name == "select":
                self._apply_select(current_table_name, next_table_name, op_val)
            elif op_name == "withColumn":
                col_name, col = op_val
                self._apply_with_column(current_table_name, next_table_name, col_name, col)
            elif op_name == "orderBy":
                self._apply_order_by(current_table_name, next_table_name, op_val)
            elif op_name == "limit":
                self._apply_limit(current_table_name, next_table_name, op_val)
            elif op_name == "join":
                other_df, on, how = op_val
                self._apply_join(current_table_name, next_table_name, op_val)
            elif op_name == "union":
                other_df = op_val
                self._apply_union(current_table_name, next_table_name, other_df)

            current_table_name = next_table_name

        # Get final results
        return self._get_table_results(current_table_name)

    def _create_table_with_data(self, table_name: str, data: List[Dict[str, Any]]) -> None:
        """Create a table and insert data using SQLAlchemy Table."""
        if not data:
            # Create a minimal table with at least one column to avoid "Table must have at least one column!" error
            columns = [Column("id", Integer)]
            table = Table(table_name, self.metadata, *columns)
            table.create(self.engine, checkfirst=True)
            self._created_tables[table_name] = table
            return

        # Create table using SQLAlchemy Table approach
        columns = []
        if data:
            for key, value in data[0].items():
                if isinstance(value, int):
                    columns.append(Column(key, Integer))
                elif isinstance(value, float):
                    columns.append(Column(key, Float))
                elif isinstance(value, bool):
                    columns.append(Column(key, Boolean))
                else:
                    columns.append(Column(key, String))

        # Create table using SQLAlchemy Table
        table = Table(table_name, self.metadata, *columns)
        table.create(self.engine, checkfirst=True)
        self._created_tables[table_name] = table

        # Insert data using raw SQL
        with Session(self.engine) as session:
            for row_data in data:
                # Convert row data to values for insert
                values = []
                for col in columns:
                    values.append(row_data[col.name])

                # Insert using raw SQL
                insert_stmt = table.insert().values(
                    dict(zip([col.name for col in columns], values))
                )
                session.execute(insert_stmt)
            session.commit()

    def _apply_filter(self, source_table: str, target_table: str, condition: Any) -> None:
        """Apply a filter operation using SQLAlchemy expressions."""
        source_table_obj = self._created_tables[source_table]

        # Check if source table has any rows
        with Session(self.engine) as session:
            row_count = session.execute(select(func.count()).select_from(source_table_obj)).scalar()

        # Set flag to enable strict column validation for filters
        # Only validate if table has rows (errors should only occur when processing actual data)
        self._strict_column_validation = bool(row_count and row_count > 0)

        # Convert condition to SQLAlchemy expression
        try:
            filter_expr = self._condition_to_sqlalchemy(source_table_obj, condition)
        finally:
            self._strict_column_validation = False

        # Create target table with same structure
        self._copy_table_structure(source_table, target_table)
        target_table_obj = self._created_tables[target_table]

        # Execute filter and insert results
        with Session(self.engine) as session:
            # Build raw SQL query
            column_names = [col.name for col in source_table_obj.columns]
            sql = f"SELECT {', '.join(column_names)} FROM {source_table}"

            if filter_expr is not None:
                # Convert SQLAlchemy expression to SQL string
                filter_sql = str(filter_expr.compile(compile_kwargs={"literal_binds": True}))
                sql += f" WHERE {filter_sql}"

            results = session.execute(text(sql)).all()

            # Insert into target table
            for result in results:
                # Convert result to dict using column names
                result_dict = {}
                for i, column in enumerate(source_table_obj.columns):
                    result_dict[column.name] = result[i]
                insert_stmt = target_table_obj.insert().values(result_dict)
                session.execute(insert_stmt)
            session.commit()

    def _apply_select(self, source_table: str, target_table: str, columns: Tuple[Any, ...]) -> None:
        """Apply a select operation."""
        source_table_obj = self._created_tables[source_table]

        # print(f"DEBUG: _apply_select called with columns: {[str(col) for col in columns]}")

        # Check if we have window functions or aggregate functions - if so, use raw SQL
        has_window_functions = any(
            (hasattr(col, "function_name") and hasattr(col, "window_spec"))  # MockWindowFunction
            or (
                hasattr(col, "function_name")
                and hasattr(col, "column")
                and col.__class__.__name__ == "MockAggregateFunction"
            )
            for col in columns
        )

        # print(f"DEBUG: has_window_functions: {has_window_functions}")

        if has_window_functions:
            # Use raw SQL for window functions
            # print("DEBUG: Using window functions path")
            self._apply_select_with_window_functions(source_table, target_table, columns)
            return

        # Build select columns and new table structure
        select_columns = []
        new_columns: List[Any] = []

        for col in columns:
            # print(f"DEBUG _apply_select: Processing {type(col).__name__}, name={getattr(col, 'name', 'N/A')}, has_operation={hasattr(col, 'operation')}")
            if isinstance(col, str):
                if col == "*":
                    # Select all columns
                    for column in source_table_obj.columns:
                        select_columns.append(column)
                        new_columns.append(Column(column.name, column.type, primary_key=False))
                else:
                    # Select specific column
                    try:
                        source_column = source_table_obj.c[col]
                        select_columns.append(source_column)
                        new_columns.append(Column(col, source_column.type, primary_key=False))
                    except KeyError:
                        # Column not found - raise AnalysisException
                        from mock_spark.core.exceptions import AnalysisException

                        raise AnalysisException(
                            f"Column '{col}' not found. Available columns: {list(source_table_obj.c.keys())}"
                        )
            elif hasattr(col, "value") and hasattr(col, "data_type"):
                # Handle MockLiteral objects (literal values) - check this before MockColumn
                if isinstance(col.value, str):
                    select_columns.append(text(f"'{col.value}'"))
                else:
                    select_columns.append(text(str(col.value)))
                # Use appropriate column type based on the literal value
                if isinstance(col.value, int):
                    new_columns.append(Column(col.name, Integer, primary_key=False))
                elif isinstance(col.value, float):
                    new_columns.append(Column(col.name, Double, primary_key=False))
                elif isinstance(col.value, str):
                    new_columns.append(Column(col.name, String, primary_key=False))
                else:
                    new_columns.append(Column(col.name, String, primary_key=False))
            elif hasattr(col, "name") and hasattr(col, "column_type"):
                # Handle MockColumn objects
                col_name = col.name
                # print(f"DEBUG: Handling MockColumn: {col_name}")

                # Check if this is an aliased column (check both _original_column and original_column)
                original_col = getattr(col, "_original_column", None) or getattr(
                    col, "original_column", None
                )
                if original_col is not None:
                    # Use original column name for lookup, alias name for output
                    original_name = original_col.name
                    alias_name = col.name
                    try:
                        source_column = source_table_obj.c[original_name]
                        select_columns.append(source_column.label(alias_name))
                        new_columns.append(
                            Column(alias_name, source_column.type, primary_key=False)
                        )
                    except KeyError:
                        print(
                            f"Warning: Column '{original_name}' not found in table {source_table}"
                        )
                        continue
                # Check if this is a wildcard selector
                elif col_name == "*":
                    # Select all columns
                    for column in source_table_obj.columns:
                        select_columns.append(column)
                        new_columns.append(Column(column.name, column.type, primary_key=False))
                else:
                    # Check if column exists in source table (might come from join)
                    if col_name in source_table_obj.c:
                        source_column = source_table_obj.c[col_name]
                        select_columns.append(source_column)
                        new_columns.append(Column(col_name, source_column.type, primary_key=False))
                    else:
                        # Column doesn't exist in source, might come from join
                        # Add as text column reference with default String type
                        select_columns.append(text(f'"{col_name}"'))
                        new_columns.append(Column(col_name, String, primary_key=False))
            elif hasattr(col, "conditions") and hasattr(col, "default_value"):
                # Handle MockCaseWhen objects
                # print(f"DEBUG: Handling MockCaseWhen: {col.name}")
                try:
                    # Build CASE WHEN SQL expression
                    case_expr = self._build_case_when_sql(col, source_table_obj)
                    select_columns.append(text(case_expr))
                    new_columns.append(Column(col.name, String, primary_key=False))
                except Exception as e:
                    print(f"Warning: Error handling MockCaseWhen: {e}")
                    continue
            elif (
                hasattr(col, "operation")
                and hasattr(col, "column")
                and hasattr(col, "function_name")
            ):
                # Check if this is an arithmetic operation
                if col.function_name in ["+", "-", "*", "/", "%"]:
                    # Handle arithmetic operations
                    # print(f"DEBUG: Handling arithmetic operation: {col.function_name}")
                    try:
                        left_col = source_table_obj.c[col.column.name]
                        # Extract value from MockLiteral or MockColumn
                        if hasattr(col.value, "value") and hasattr(col.value, "data_type"):
                            # This is a MockLiteral
                            right_val = col.value.value
                        elif hasattr(col.value, "name"):
                            # This is a MockColumn - convert to SQL column reference
                            right_val = source_table_obj.c[col.value.name]
                        else:
                            right_val = col.value

                        # Apply arithmetic operation
                        if col.function_name == "+":
                            expr = left_col + right_val
                        elif col.function_name == "-":
                            expr = left_col - right_val
                        elif col.function_name == "*":
                            expr = left_col * right_val
                        elif col.function_name == "/":
                            expr = left_col / right_val
                        elif col.function_name == "%":
                            expr = left_col % right_val

                        select_columns.append(expr.label(col.name))
                        # For arithmetic operations, determine result type based on operand types
                        if col.function_name == "/":
                            # Division always returns float
                            new_columns.append(Column(col.name, Float, primary_key=False))
                        else:
                            # For other operations, if either operand is float, result is float
                            left_is_float = (
                                "FLOAT" in str(left_col.type).upper()
                                or "DOUBLE" in str(left_col.type).upper()
                                or "REAL" in str(left_col.type).upper()
                            )
                            right_is_float = False
                            if hasattr(right_val, "type"):
                                right_is_float = (
                                    "FLOAT" in str(right_val.type).upper()
                                    or "DOUBLE" in str(right_val.type).upper()
                                    or "REAL" in str(right_val.type).upper()
                                )

                            if left_is_float or right_is_float:
                                new_columns.append(Column(col.name, Float, primary_key=False))
                            else:
                                new_columns.append(
                                    Column(col.name, left_col.type, primary_key=False)
                                )
                        # print(f"DEBUG: Successfully handled arithmetic operation: {col.function_name}")
                    except KeyError:
                        print(
                            f"Warning: Column '{col.column.name}' not found in table {source_table}"
                        )
                        continue
                elif col.function_name in ["==", "!=", ">", "<", ">=", "<="]:
                    # Handle comparison operations
                    try:
                        left_col = source_table_obj.c[col.column.name]
                        # Extract value from MockLiteral if needed
                        if hasattr(col.value, "value") and hasattr(col.value, "data_type"):
                            # This is a MockLiteral
                            right_val = col.value.value
                        else:
                            right_val = col.value

                        # Apply comparison operation
                        if col.function_name == "==":
                            if right_val is None:
                                expr = left_col.is_(None)
                            else:
                                expr = left_col == right_val
                        elif col.function_name == "!=":
                            if right_val is None:
                                expr = left_col.isnot(None)
                            else:
                                expr = left_col != right_val
                        elif col.function_name == ">":
                            expr = left_col > right_val
                        elif col.function_name == "<":
                            expr = left_col < right_val
                        elif col.function_name == ">=":
                            expr = left_col >= right_val
                        elif col.function_name == "<=":
                            expr = left_col <= right_val

                        select_columns.append(expr.label(col.name))
                        new_columns.append(Column(col.name, Boolean, primary_key=False))
                    except KeyError:
                        print(
                            f"Warning: Column '{col.column.name}' not found in table {source_table}"
                        )
                        continue
                else:
                    # Handle function operations like F.upper(F.col("name"))
                    # print(f"DEBUG: Handling function operation: {col.function_name} on column {col.column.name}")

                    # Special handling for expr() which doesn't reference a real column
                    if (
                        col.function_name == "expr"
                        and hasattr(col, "value")
                        and col.value is not None
                    ):
                        # Use the SQL expression directly
                        func_expr: Any = text(col.value)  # Can be TextClause or Function[Any]
                        # Handle labeling
                        try:
                            select_columns.append(func_expr.label(col.name))
                        except (NotImplementedError, AttributeError):
                            select_columns.append(text(f"({col.value}) AS {col.name}"))
                        # Infer column type as String for now
                        new_columns.append(Column(col.name, String, primary_key=False))
                        continue

                    # Special handling for cast() which needs CAST(column AS type) syntax
                    if (
                        col.function_name == "cast"
                        and hasattr(col, "value")
                        and col.value is not None
                    ):
                        # Map Mock Spark data types to DuckDB types
                        type_mapping = {
                            "StringType": "VARCHAR",
                            "IntegerType": "INTEGER",
                            "LongType": "BIGINT",
                            "DoubleType": "DOUBLE",
                            "FloatType": "DOUBLE",
                            "BooleanType": "BOOLEAN",
                            "DateType": "DATE",
                            "TimestampType": "TIMESTAMP",
                            # Also support lowercase string versions
                            "string": "VARCHAR",
                            "int": "INTEGER",
                            "integer": "INTEGER",
                            "long": "BIGINT",
                            "bigint": "BIGINT",
                            "double": "DOUBLE",
                            "float": "DOUBLE",
                            "boolean": "BOOLEAN",
                            "date": "DATE",
                            "timestamp": "TIMESTAMP",
                        }
                        # Get the data type name
                        # Check if col.value is a string (e.g., "double", "integer")
                        if isinstance(col.value, str):
                            type_name = col.value
                        elif hasattr(col.value, "__class__"):
                            type_name = col.value.__class__.__name__
                        else:
                            type_name = str(col.value)

                        sql_type = type_mapping.get(type_name, "VARCHAR")

                        # Build CAST expression
                        try:
                            source_column = source_table_obj.c[col.column.name]
                            # Use raw SQL with AS clause since text expressions don't support .label()
                            cast_sql = f"CAST({col.column.name} AS {sql_type}) AS {col.name}"
                            select_columns.append(text(cast_sql))
                            # Infer column type based on cast target
                            if sql_type in ["INTEGER", "BIGINT"]:
                                new_columns.append(Column(col.name, Integer, primary_key=False))
                            elif sql_type == "DOUBLE":
                                new_columns.append(Column(col.name, Double, primary_key=False))
                            elif sql_type == "BOOLEAN":
                                new_columns.append(Column(col.name, Boolean, primary_key=False))
                            else:
                                new_columns.append(Column(col.name, String, primary_key=False))
                            continue
                        except KeyError:
                            print(
                                f"Warning: Column '{col.column.name}' not found in table {source_table}"
                            )
                            continue

                    try:
                        # Check if the column is a complex expression (e.g., arithmetic operation)
                        if hasattr(col.column, "function_name") and col.column.function_name in [
                            "+",
                            "-",
                            "*",
                            "/",
                            "%",
                        ]:
                            # Build the arithmetic expression first
                            left_col = source_table_obj.c[col.column.column.name]
                            if isinstance(col.column.value, (int, float)):
                                right_val = col.column.value
                            else:
                                right_val = (
                                    source_table_obj.c[col.column.value.name]
                                    if hasattr(col.column.value, "name")
                                    else col.column.value
                                )

                            if col.column.function_name == "+":
                                base_expr = left_col + right_val
                            elif col.column.function_name == "-":
                                base_expr = left_col - right_val
                            elif col.column.function_name == "*":
                                base_expr = left_col * right_val
                            elif col.column.function_name == "/":
                                base_expr = left_col / right_val
                            elif col.column.function_name == "%":
                                base_expr = left_col % right_val
                            else:
                                base_expr = left_col
                        else:
                            # Simple column reference
                            base_expr = source_table_obj.c[col.column.name]

                        source_column = base_expr
                        # Apply the function using SQLAlchemy
                        if col.function_name == "upper":
                            func_expr = func.upper(source_column)
                        elif col.function_name == "lower":
                            func_expr = func.lower(source_column)
                        elif col.function_name == "length":
                            func_expr = func.length(source_column)
                        elif col.function_name == "abs":
                            func_expr = func.abs(source_column)
                        elif col.function_name == "round":
                            # For round function, check if there's a precision parameter
                            if hasattr(col, "value") and col.value is not None:
                                func_expr = func.round(source_column, col.value)
                            else:
                                func_expr = func.round(source_column)
                        elif col.function_name == "ceil":
                            func_expr = func.ceil(source_column)
                        elif col.function_name == "floor":
                            func_expr = func.floor(source_column)
                        elif col.function_name == "sqrt":
                            func_expr = func.sqrt(source_column)
                        else:
                            # Map PySpark function names to DuckDB function names
                            function_mapping = {
                                "signum": "sign",
                                "greatest": "greatest",
                                "least": "least",
                                "format_string": "format",
                                "translate": "translate",
                                "base64": "base64",
                                "ascii": "ascii",
                                "months_between": "months_between",
                                "minute": "minute",
                                "second": "second",
                                "add_months": "date_add",
                                "date_add": "date_add",
                                "date_sub": "date_sub",
                                "coalesce": "coalesce",
                                "isnull": "isnull",  # Handled by special_functions with IS NULL syntax
                                "isnan": "isnan",
                                # Add more mappings as needed
                            }

                            # Functions that require type casting
                            type_casting_functions = {
                                "base64": "CAST({} AS BLOB)",  # DuckDB base64 expects BLOB
                                "ascii": "CAST({} AS VARCHAR)",  # Ensure VARCHAR for ascii
                                "minute": "CAST({} AS TIMESTAMP)",  # DuckDB minute expects TIMESTAMP
                                "second": "CAST({} AS TIMESTAMP)",  # DuckDB second expects TIMESTAMP
                                "add_months": "CAST({} AS DATE)",  # DuckDB add_months expects DATE
                                "date_add": "CAST({} AS DATE)",  # DuckDB date_add expects DATE
                                "date_sub": "CAST({} AS DATE)",  # DuckDB date_sub expects DATE
                            }

                            # Special handling for functions that need custom SQL generation
                            special_functions = {
                                "add_months": "({} + INTERVAL {} MONTH)",
                                "months_between": "DATEDIFF('MONTH', {}, {})",
                                "date_add": "({} + INTERVAL {} DAY)",
                                "date_sub": "({} - INTERVAL {} DAY)",
                                "isnull": "({} IS NULL)",
                                "expr": "{}",  # expr() function directly uses the SQL expression
                                "coalesce": "coalesce",  # Mark for special handling
                            }
                            duckdb_function_name = function_mapping.get(
                                col.function_name, col.function_name
                            )

                            # Check if this function needs type casting
                            column_expr = col.column.name
                            if col.function_name in type_casting_functions:
                                column_expr = type_casting_functions[col.function_name].format(
                                    col.column.name
                                )

                            # Handle functions with parameters
                            if hasattr(col, "value") and col.value is not None:
                                # Check if this is a special function that needs custom SQL generation
                                if col.function_name in special_functions:
                                    # Handle special functions like add_months, months_between, date_add, date_sub, expr
                                    if col.function_name == "expr":
                                        # For expr(), use the SQL expression directly
                                        func_expr = text(col.value)
                                    elif col.function_name == "months_between":
                                        # For months_between, we need both column names with DATE casting
                                        column1_name = f"CAST({col.column.name} AS DATE)"
                                        column2_name = (
                                            f"CAST({col.value.name} AS DATE)"
                                            if hasattr(col.value, "name")
                                            else f"CAST({col.value} AS DATE)"
                                        )
                                        special_sql = special_functions[col.function_name].format(
                                            column1_name, column2_name
                                        )
                                        func_expr = text(special_sql)
                                    elif col.function_name in ["date_add", "date_sub"]:
                                        # For date_add and date_sub, we need the column and the number of days
                                        if isinstance(col.value, str):
                                            param_value = f"'{col.value}'"
                                        elif hasattr(col.value, "value") and hasattr(
                                            col.value, "data_type"
                                        ):
                                            # Handle MockLiteral objects
                                            if isinstance(col.value.value, str):
                                                param_value = f"'{col.value.value}'"
                                            else:
                                                param_value = str(col.value.value)
                                        else:
                                            param_value = str(col.value)

                                        special_sql = special_functions[col.function_name].format(
                                            column_expr, param_value
                                        )
                                        func_expr = text(special_sql)
                                    elif col.function_name == "isnull":
                                        # For isnull, we only need the column name (no parameters)
                                        special_sql = special_functions[col.function_name].format(
                                            column_expr
                                        )
                                        func_expr = text(special_sql)
                                    elif col.function_name == "coalesce":
                                        # For coalesce, cast all arguments to VARCHAR to ensure type compatibility
                                        params = []
                                        params.append(f"CAST({column_expr} AS VARCHAR)")

                                        # Handle the additional parameters
                                        if isinstance(col.value, (tuple, list)):
                                            for param in col.value:
                                                if hasattr(param, "value") and hasattr(
                                                    param, "data_type"
                                                ):
                                                    # MockLiteral - check if it's a string literal
                                                    if isinstance(param.value, str):
                                                        params.append(
                                                            f"CAST('{param.value}' AS VARCHAR)"
                                                        )
                                                    else:
                                                        params.append(
                                                            f"CAST({param.value} AS VARCHAR)"
                                                        )
                                                elif hasattr(param, "name"):
                                                    # MockColumn
                                                    params.append(
                                                        f'CAST("{param.name}" AS VARCHAR)'
                                                    )
                                                else:
                                                    params.append(f"CAST({param} AS VARCHAR)")
                                        else:
                                            # Single parameter
                                            if hasattr(col.value, "value") and hasattr(
                                                col.value, "data_type"
                                            ):
                                                # MockLiteral - check if it's a string literal
                                                if isinstance(col.value.value, str):
                                                    params.append(
                                                        f"CAST('{col.value.value}' AS VARCHAR)"
                                                    )
                                                else:
                                                    params.append(
                                                        f"CAST({col.value.value} AS VARCHAR)"
                                                    )
                                            elif hasattr(col.value, "name"):
                                                params.append(
                                                    f'CAST("{col.value.name}" AS VARCHAR)'
                                                )
                                            else:
                                                params.append(f"CAST({col.value} AS VARCHAR)")

                                        coalesce_sql = f"coalesce({', '.join(params)})"
                                        func_expr = text(coalesce_sql)
                                    else:
                                        # Handle other special functions like add_months
                                        if isinstance(col.value, str):
                                            param_value = f"'{col.value}'"
                                        elif hasattr(col.value, "value") and hasattr(
                                            col.value, "data_type"
                                        ):
                                            # Handle MockLiteral objects
                                            if isinstance(col.value.value, str):
                                                param_value = f"'{col.value.value}'"
                                            else:
                                                param_value = str(col.value.value)
                                        else:
                                            param_value = str(col.value)

                                        special_sql = special_functions[col.function_name].format(
                                            column_expr, param_value
                                        )
                                        func_expr = text(special_sql)
                                elif isinstance(col.value, (tuple, list)):
                                    # Flatten nested tuples/lists and process parameters
                                    flattened_params: List[Any] = []
                                    for param in col.value:
                                        if isinstance(param, (tuple, list)):
                                            # Handle nested tuples/lists (like format_string)
                                            flattened_params.extend(param)
                                        else:
                                            flattened_params.append(param)

                                    # Filter out empty tuples/lists and None values
                                    filtered_params = [
                                        p
                                        for p in flattened_params
                                        if p is not None and p != () and p != []
                                    ]

                                    if filtered_params:
                                        # Join parameters with commas, handling MockLiteral and MockColumn objects
                                        formatted_params = []
                                        for param in filtered_params:
                                            if isinstance(param, str):
                                                formatted_params.append(f"'{param}'")
                                            elif hasattr(param, "value") and hasattr(
                                                param, "data_type"
                                            ):
                                                # Handle MockLiteral objects
                                                if isinstance(param.value, str):
                                                    formatted_params.append(f"'{param.value}'")
                                                else:
                                                    formatted_params.append(str(param.value))
                                            elif hasattr(param, "name"):
                                                # Handle MockColumn objects
                                                formatted_params.append(f'"{param.name}"')
                                            else:
                                                formatted_params.append(str(param))
                                        param_str = ", ".join(formatted_params)
                                        func_expr = text(
                                            f"{duckdb_function_name}({column_expr}, {param_str})"
                                        )
                                    else:
                                        # No valid parameters - single argument function
                                        func_expr = text(f"{duckdb_function_name}({column_expr})")
                                else:
                                    # Single parameter
                                    if isinstance(col.value, str):
                                        func_expr = text(
                                            f"{duckdb_function_name}({column_expr}, '{col.value}')"
                                        )
                                    elif hasattr(col.value, "value") and hasattr(
                                        col.value, "data_type"
                                    ):
                                        # Handle MockLiteral objects
                                        if isinstance(col.value.value, str):
                                            func_expr = text(
                                                f"{duckdb_function_name}({column_expr}, '{col.value.value}')"
                                            )
                                        else:
                                            func_expr = text(
                                                f"{duckdb_function_name}({column_expr}, {col.value.value})"
                                            )
                                    else:
                                        # Handle MockColumn objects in parameters
                                        if hasattr(col.value, "name"):
                                            # This is a MockColumn object, extract its name
                                            param_name = col.value.name
                                            func_expr = text(
                                                f"{duckdb_function_name}({column_expr}, {param_name})"
                                            )
                                        else:
                                            func_expr = text(
                                                f"{duckdb_function_name}({column_expr}, {col.value})"
                                            )
                            else:
                                # No parameters - single argument function
                                # Check if this is a special function that doesn't use standard function syntax
                                if col.function_name in special_functions:
                                    special_sql = special_functions[col.function_name].format(
                                        column_expr
                                    )
                                    func_expr = text(special_sql)
                                else:
                                    func_expr = text(f"{duckdb_function_name}({column_expr})")

                        # Handle labeling more carefully to avoid NotImplementedError
                        try:
                            select_columns.append(func_expr.label(col.name))
                        except (NotImplementedError, AttributeError):
                            # For expressions that don't support .label() or .alias(),
                            # create a raw SQL expression with AS clause
                            if hasattr(func_expr, "text"):
                                # For TextClause objects, create a new text expression with alias
                                select_columns.append(text(f"({func_expr.text}) AS {col.name}"))
                            else:
                                # Fallback: try to convert to string and wrap in parentheses
                                select_columns.append(text(f"({str(func_expr)}) AS {col.name}"))
                        # Infer column type based on function
                        if col.function_name in ["length", "abs", "ceil", "floor"]:
                            new_columns.append(Column(col.name, Integer, primary_key=False))
                        elif col.function_name in ["round", "sqrt"]:
                            new_columns.append(Column(col.name, Float, primary_key=False))
                        elif col.function_name in ["isnull", "isnan", "isnotnull"]:
                            new_columns.append(Column(col.name, Boolean, primary_key=False))
                        else:
                            new_columns.append(Column(col.name, String, primary_key=False))
                        # print(f"DEBUG: Successfully handled function operation: {col.function_name}")
                    except KeyError:
                        print(
                            f"Warning: Column '{col.column.name}' not found in table {source_table}"
                        )
                        continue
            elif hasattr(col, "name"):
                # Handle F.col("column_name") case
                # Check for wildcard first
                if col.name == "*":
                    print("DEBUG: Handling wildcard!")
                    # Select all columns from source table
                    for column in source_table_obj.columns:
                        select_columns.append(column)
                        new_columns.append(Column(column.name, column.type, primary_key=False))
                    continue

                # Check if this is an aliased column (check both _original_column and original_column)
                original_col = getattr(col, "_original_column", None) or getattr(
                    col, "original_column", None
                )
                if original_col is not None:
                    # Use original column name for lookup, alias name for output
                    original_name = original_col.name
                    alias_name = col.name
                    try:
                        source_column = source_table_obj.c[original_name]
                        select_columns.append(source_column.label(alias_name))
                        new_columns.append(
                            Column(alias_name, source_column.type, primary_key=False)
                        )
                    except KeyError:
                        print(
                            f"Warning: Column '{original_name}' not found in table {source_table}"
                        )
                        continue
                else:
                    # Regular column (no alias)
                    try:
                        source_column = source_table_obj.c[col.name]
                        select_columns.append(source_column)
                        new_columns.append(Column(col.name, source_column.type, primary_key=False))
                    except KeyError:
                        # Column not found - raise AnalysisException
                        from mock_spark.core.exceptions import AnalysisException

                        raise AnalysisException(
                            f"Column '{col.name}' not found. Available columns: {list(source_table_obj.c.keys())}"
                        )

        # Ensure we have at least one column
        if not new_columns:
            # Add a placeholder column to avoid "Table must have at least one column!" error
            new_columns = [Column("placeholder", String, primary_key=False)]
            select_columns = [text("'placeholder' as placeholder")]

        # Create target table using SQLAlchemy Table
        target_table_obj = Table(target_table, self.metadata, *new_columns)
        target_table_obj.create(self.engine, checkfirst=True)
        self._created_tables[target_table] = target_table_obj

        # Execute select and insert results
        with Session(self.engine) as session:
            # If we have literals, we need to select from the source table to replicate them
            if any(
                hasattr(col, "text") or str(type(col)).find("TextClause") != -1
                for col in select_columns
            ):
                # Use raw SQL to ensure literals are replicated for each row
                source_table_obj = self._created_tables[source_table]
                select_clause = ", ".join(
                    [
                        (
                            str(col)
                            if hasattr(col, "text")
                            or str(type(col)).find("TextClause") != -1
                            or str(type(col)).find("Label") != -1
                            else f'"{col.name}"'
                        )
                        for col in select_columns
                    ]
                )
                sql = f"SELECT {select_clause} FROM {source_table}"
                results = session.execute(text(sql)).all()
            else:
                query = select(*select_columns)
                results = session.execute(query).all()

            for result in results:
                # Convert result to dict using column names
                result_dict = {}
                for i, column in enumerate(new_columns):
                    result_dict[column.name] = result[i]
                insert_stmt = target_table_obj.insert().values(result_dict)
                session.execute(insert_stmt)
            session.commit()

    def _apply_select_with_window_functions(
        self, source_table: str, target_table: str, columns: Tuple[Any, ...]
    ) -> None:
        """Apply select operation with window functions using raw SQL."""
        source_table_obj = self._created_tables[source_table]

        # Build the SELECT clause
        select_parts = []
        new_columns: List[Any] = []

        for col in columns:
            # print(f"DEBUG _apply_select_with_window_functions: Processing {type(col).__name__}, name={getattr(col, 'name', 'N/A')}, has_operation={hasattr(col, 'operation')}, has_function_name={hasattr(col, 'function_name')}")
            if isinstance(col, str):
                if col == "*":
                    # Select all columns
                    for column in source_table_obj.columns:
                        select_parts.append(f'"{column.name}"')
                        new_columns.append(Column(column.name, column.type, primary_key=False))
                else:
                    select_parts.append(f'"{col}"')
                    source_column = source_table_obj.c[col]
                    new_columns.append(Column(col, source_column.type, primary_key=False))
            elif (
                hasattr(col, "function_name")
                and hasattr(col, "column")
                and col.__class__.__name__ == "MockAggregateFunction"
            ):
                # Handle MockAggregateFunction objects like F.count(), F.sum(), etc.
                if col.function_name == "count":
                    if col.column is None or col.column == "*":
                        select_parts.append("COUNT(*)")
                    else:
                        column_name = col.column.name if hasattr(col.column, "name") else col.column
                        select_parts.append(f"COUNT({column_name})")
                elif col.function_name == "countDistinct":
                    column_name = col.column.name if hasattr(col.column, "name") else col.column
                    select_parts.append(f"COUNT(DISTINCT {column_name})")
                elif col.function_name == "percentile_approx":
                    column_name = col.column.name if hasattr(col.column, "name") else col.column
                    # DuckDB doesn't have percentile functions, use AVG as approximation
                    select_parts.append(f"AVG({column_name})")
                elif col.function_name == "corr":
                    # CORR function requires two columns, but we only have one
                    # This is a limitation - we'll use AVG as fallback
                    column_name = col.column.name if hasattr(col.column, "name") else col.column
                    select_parts.append(f"AVG({column_name})")
                elif col.function_name == "covar_samp":
                    # COVAR_SAMP function requires two columns, but we only have one
                    # This is a limitation - we'll use AVG as fallback
                    column_name = col.column.name if hasattr(col.column, "name") else col.column
                    select_parts.append(f"AVG({column_name})")
                elif col.function_name == "sum":
                    column_name = col.column.name if hasattr(col.column, "name") else col.column
                    select_parts.append(f"SUM({column_name})")
                elif col.function_name == "avg":
                    column_name = col.column.name if hasattr(col.column, "name") else col.column
                    select_parts.append(f"AVG({column_name})")
                elif col.function_name == "max":
                    column_name = col.column.name if hasattr(col.column, "name") else col.column
                    select_parts.append(f"MAX({column_name})")
                elif col.function_name == "min":
                    column_name = col.column.name if hasattr(col.column, "name") else col.column
                    select_parts.append(f"MIN({column_name})")
                else:
                    column_name = col.column.name if hasattr(col.column, "name") else col.column
                    select_parts.append(f"{col.function_name.upper()}({column_name})")

                # Add appropriate column type
                if col.function_name in ["count", "countDistinct"]:
                    new_columns.append(Column(col.name, Integer, primary_key=False))
                elif col.function_name == "sum":
                    # Preserve source column type for SUM
                    if col.column and hasattr(col.column, "name"):
                        column_name = col.column.name
                        source_column = source_table_obj.c[column_name]
                        source_type = str(source_column.type).upper()
                        # Use Integer for integer types, Float for floating types
                        if any(
                            int_type in source_type
                            for int_type in ["INTEGER", "BIGINT", "SMALLINT", "INT"]
                        ):
                            new_columns.append(Column(col.name, Integer, primary_key=False))
                        else:
                            new_columns.append(Column(col.name, Float, primary_key=False))
                    elif isinstance(col.column, str):
                        column_name = col.column
                        if column_name in source_table_obj.c:
                            source_column = source_table_obj.c[column_name]
                            source_type = str(source_column.type).upper()
                            if any(
                                int_type in source_type
                                for int_type in ["INTEGER", "BIGINT", "SMALLINT", "INT"]
                            ):
                                new_columns.append(Column(col.name, Integer, primary_key=False))
                            else:
                                new_columns.append(Column(col.name, Float, primary_key=False))
                        else:
                            new_columns.append(Column(col.name, Float, primary_key=False))
                    else:
                        new_columns.append(Column(col.name, Float, primary_key=False))
                elif col.function_name in ["avg", "percentile_approx", "corr", "covar_samp"]:
                    new_columns.append(Column(col.name, Float, primary_key=False))
                elif col.function_name in ["max", "min"]:
                    # For max/min, use the same type as the source column
                    if col.column and hasattr(col.column, "name"):
                        source_column = source_table_obj.c[col.column.name]
                        new_columns.append(Column(col.name, source_column.type, primary_key=False))
                    else:
                        new_columns.append(Column(col.name, Integer, primary_key=False))
                else:
                    new_columns.append(Column(col.name, String, primary_key=False))
            elif hasattr(col, "name") and col.name == "*":
                # Handle F.col("*") case - only add columns from source table
                # Check which columns are already in new_columns to avoid duplicates
                existing_col_names = {c.name for c in new_columns}
                for column in source_table_obj.columns:
                    if column.name not in existing_col_names:
                        select_parts.append(f'"{column.name}"')
                        new_columns.append(Column(column.name, column.type, primary_key=False))
            elif hasattr(col, "name") and not hasattr(col, "alias"):
                # Handle F.col("column_name") case (but not window functions)
                select_parts.append(f'"{col.name}"')
                source_column = source_table_obj.c[col.name]
                new_columns.append(Column(col.name, source_column.type, primary_key=False))
            elif hasattr(col, "operation") and hasattr(col, "column") and hasattr(col, "value"):
                # Handle MockColumnOperation objects (arithmetic and string operations)
                # Check if this is an arithmetic operation (not a function)
                if hasattr(col, "function_name") and col.function_name in ["+", "-", "*", "/", "%"]:
                    # This is an arithmetic operation, not a function
                    col_expr = self._expression_to_sql(col)
                    select_parts.append(f"({col_expr})")
                    # For arithmetic operations, handle division specially
                    if col.function_name == "/":
                        # Division always returns floating-point type
                        new_columns.append(Column(col.name, Float, primary_key=False))
                    elif hasattr(col, "column") and hasattr(col.column, "name"):
                        source_column = source_table_obj.c[col.column.name]
                        new_columns.append(Column(col.name, source_column.type, primary_key=False))
                    else:
                        new_columns.append(Column(col.name, Float, primary_key=False))
                elif hasattr(col, "function_name") and col.function_name in [
                    "upper",
                    "lower",
                    "trim",
                ]:
                    # This is a string function operation
                    col_expr = self._expression_to_sql(col)
                    select_parts.append(f"({col_expr})")
                    new_columns.append(Column(col.name, String, primary_key=False))
                elif hasattr(col, "function_name") and col.function_name in [
                    "length",
                    "abs",
                    "round",
                ]:
                    # This is a math function operation that returns numeric types
                    col_expr = self._expression_to_sql(col)
                    select_parts.append(f"({col_expr})")
                    new_columns.append(Column(col.name, Integer, primary_key=False))
                elif hasattr(col, "function_name") and col.function_name in [
                    "==",
                    "!=",
                    ">",
                    "<",
                    ">=",
                    "<=",
                ]:
                    # This is a comparison operation (returns boolean)
                    col_expr = self._expression_to_sql(col)
                    select_parts.append(f"{col_expr}")
                    new_columns.append(Column(col.name, Boolean, primary_key=False))
                elif not hasattr(col, "function_name"):
                    # This is an operation without function_name (must be arithmetic)
                    col_expr = self._expression_to_sql(col)
                    select_parts.append(f"({col_expr})")
                    # Arithmetic operation (returns numeric)
                    new_columns.append(Column(col.name, Float, primary_key=False))
            elif hasattr(col, "function_name") and hasattr(col, "window_spec"):
                # Handle MockWindowFunction objects like F.row_number().over(...).alias("rank")
                if col.function_name == "row_number":
                    # Build the window specification from the window_spec
                    window_sql = self._window_spec_to_sql(col.window_spec, source_table_obj)
                    select_parts.append(f"ROW_NUMBER() OVER ({window_sql})")
                    new_columns.append(Column(col.name, Integer, primary_key=False))
                elif col.function_name == "rank":
                    window_sql = self._window_spec_to_sql(col.window_spec, source_table_obj)
                    select_parts.append(f"RANK() OVER ({window_sql})")
                    new_columns.append(Column(col.name, Integer, primary_key=False))
                elif col.function_name == "dense_rank":
                    window_sql = self._window_spec_to_sql(col.window_spec, source_table_obj)
                    select_parts.append(f"DENSE_RANK() OVER ({window_sql})")
                    new_columns.append(Column(col.name, Integer, primary_key=False))
                else:
                    # Generic window function - handle parameters
                    window_sql = self._window_spec_to_sql(col.window_spec, source_table_obj)

                    # Build function call with parameters
                    # Get parameters from the original function stored in the MockWindowFunction
                    original_function = getattr(col, "function", None)
                    if (
                        original_function
                        and hasattr(original_function, "value")
                        and original_function.value is not None
                    ):
                        # Handle parameters for functions like NTH_VALUE, LAG, LEAD, etc.
                        if isinstance(original_function.value, tuple):
                            # Special handling for LAG and LEAD which need column name + tuple params
                            if col.function_name in ["lag", "lead"]:
                                # Get the column name from the original function
                                column_name = getattr(
                                    getattr(original_function, "column", None), "name", "unknown"
                                )
                                # Extract offset and default_value from tuple
                                params = [f'"{column_name}"']
                                for param in original_function.value:
                                    if param is not None:
                                        if isinstance(param, str):
                                            params.append(f"'{param}'")
                                        else:
                                            params.append(str(param))
                                param_str = ", ".join(params)
                                select_parts.append(
                                    f"{col.function_name.upper()}({param_str}) OVER ({window_sql})"
                                )
                            else:
                                # Extract parameters from tuple for other functions
                                params = []
                                for param in original_function.value:
                                    if param is not None:
                                        if isinstance(param, str):
                                            params.append(f"'{param}'")
                                        else:
                                            params.append(str(param))
                                param_str = ", ".join(params)
                                select_parts.append(
                                    f"{col.function_name.upper()}({param_str}) OVER ({window_sql})"
                                )
                        else:
                            # For functions like NTH_VALUE, we need both the column and the value
                            if col.function_name in ["nth_value"]:
                                # Extract column name from the original function's column attribute
                                column_name = getattr(
                                    getattr(original_function, "column", None), "name", "unknown"
                                )
                                param_str = f'"{column_name}", {original_function.value}'
                                select_parts.append(
                                    f"{col.function_name.upper()}({param_str}) OVER ({window_sql})"
                                )
                            else:
                                # Single parameter for other functions
                                if isinstance(original_function.value, str):
                                    select_parts.append(
                                        f"{col.function_name.upper()}('{original_function.value}') OVER ({window_sql})"
                                    )
                                else:
                                    select_parts.append(
                                        f"{col.function_name.upper()}({original_function.value}) OVER ({window_sql})"
                                    )
                    else:
                        # No parameters in value, but check if there's a column (for aggregate functions)
                        # Some functions like CUME_DIST, PERCENT_RANK, RANK, DENSE_RANK don't take parameters
                        if col.function_name in ["cume_dist", "percent_rank", "rank", "dense_rank"]:
                            # These functions don't take parameters
                            select_parts.append(
                                f"{col.function_name.upper()}() OVER ({window_sql})"
                            )
                        elif (
                            original_function
                            and hasattr(original_function, "column")
                            and original_function.column
                        ):
                            column_name = getattr(original_function.column, "name", "unknown")
                            # Check if column exists in table before adding to SQL
                            if column_name != "unknown" and column_name in source_table_obj.c:
                                select_parts.append(
                                    f'{col.function_name.upper()}("{column_name}") OVER ({window_sql})'
                                )
                            else:
                                # Column doesn't exist, skip this window function or use placeholder
                                # Add NULL as placeholder to maintain column position
                                select_parts.append(f"NULL AS {col.name}")
                                new_columns.append(Column(col.name, String, primary_key=False))
                                continue
                        else:
                            # Truly no parameters
                            select_parts.append(
                                f"{col.function_name.upper()}() OVER ({window_sql})"
                            )

                    new_columns.append(Column(col.name, Integer, primary_key=False))
            elif (
                hasattr(col, "operation")
                and hasattr(col, "column")
                and hasattr(col, "function_name")
            ):
                # Handle MockColumnOperation objects with function operations like F.upper()
                # Note: These need AS alias clause when col.name differs from column.name
                if col.function_name == "upper":
                    select_parts.append(f"UPPER({col.column.name}) AS {col.name}")
                elif col.function_name == "lower":
                    select_parts.append(f"LOWER({col.column.name}) AS {col.name}")
                elif col.function_name == "length":
                    select_parts.append(f"LENGTH({col.column.name}) AS {col.name}")
                elif col.function_name == "abs":
                    select_parts.append(f"ABS({col.column.name}) AS {col.name}")
                elif col.function_name == "round":
                    # For round function, check if there's a precision parameter
                    if hasattr(col, "value") and col.value is not None:
                        select_parts.append(f"ROUND({col.column.name}, {col.value}) AS {col.name}")
                    else:
                        select_parts.append(f"ROUND({col.column.name}) AS {col.name}")
                elif col.function_name == "ceil":
                    select_parts.append(f"CEIL({col.column.name}) AS {col.name}")
                elif col.function_name == "floor":
                    select_parts.append(f"FLOOR({col.column.name}) AS {col.name}")
                elif col.function_name == "sqrt":
                    select_parts.append(f"SQRT({col.column.name}) AS {col.name}")
                elif col.function_name == "months_between":
                    # For months_between, we need both column names
                    column1_name = col.column.name if hasattr(col.column, "name") else col.column
                    column2_name = col.value.name if hasattr(col.value, "name") else col.value
                    select_parts.append(
                        f"MONTHS_BETWEEN({column1_name}, {column2_name}) AS {col.name}"
                    )
                elif col.function_name == "split":
                    # For split, use DuckDB's string_split or str_split function
                    delimiter = col.value if isinstance(col.value, str) else str(col.value)
                    select_parts.append(
                        f"STRING_SPLIT({col.column.name}, '{delimiter}') AS {col.name}"
                    )
                else:
                    # Fallback to raw SQL for unknown functions
                    select_parts.append(f"{col.function_name}({col.column.name}) AS {col.name}")

                # Infer column type based on function
                if col.function_name in ["length", "abs", "ceil", "floor"]:
                    new_columns.append(Column(col.name, Integer, primary_key=False))
                elif col.function_name in ["round", "sqrt", "months_between"]:
                    new_columns.append(Column(col.name, Float, primary_key=False))
                elif col.function_name == "split":
                    # split returns an array type - use String with JSON for now (DuckDB arrays)
                    # We'll mark it but DuckDB will handle the array natively
                    from sqlalchemy import ARRAY

                    try:
                        new_columns.append(Column(col.name, ARRAY(String), primary_key=False))
                    except:  # noqa: E722
                        # Fallback to String if ARRAY not supported
                        new_columns.append(Column(col.name, String, primary_key=False))
                else:
                    new_columns.append(Column(col.name, String, primary_key=False))
            elif hasattr(col, "value") and hasattr(col, "data_type"):
                # Handle MockLiteral objects (literal values)
                if isinstance(col.value, str):
                    select_parts.append(f"'{col.value}'")
                else:
                    select_parts.append(str(col.value))
                # Use appropriate column type based on the literal value
                if isinstance(col.value, int):
                    new_columns.append(Column(col.name, Integer, primary_key=False))
                elif isinstance(col.value, float):
                    new_columns.append(Column(col.name, Float, primary_key=False))
                elif isinstance(col.value, str):
                    new_columns.append(Column(col.name, String, primary_key=False))
                else:
                    new_columns.append(Column(col.name, String, primary_key=False))
            elif (
                hasattr(col, "function_name")
                and hasattr(col, "column")
                and col.__class__.__name__ == "MockAggregateFunction"
            ):
                # Handle MockAggregateFunction objects like F.count(), F.sum(), etc.
                if col.function_name == "count":
                    if col.column is None or col.column == "*":
                        select_parts.append("COUNT(*)")
                    else:
                        column_name = col.column.name if hasattr(col.column, "name") else col.column
                        select_parts.append(f"COUNT({column_name})")
                elif col.function_name == "countDistinct":
                    column_name = col.column.name if hasattr(col.column, "name") else col.column
                    select_parts.append(f"COUNT(DISTINCT {column_name})")
                elif col.function_name == "percentile_approx":
                    column_name = col.column.name if hasattr(col.column, "name") else col.column
                    # DuckDB doesn't have percentile functions, use AVG as approximation
                    select_parts.append(f"AVG({column_name})")
                elif col.function_name == "corr":
                    # CORR function requires two columns, but we only have one
                    # This is a limitation - we'll use AVG as fallback
                    column_name = col.column.name if hasattr(col.column, "name") else col.column
                    select_parts.append(f"AVG({column_name})")
                elif col.function_name == "covar_samp":
                    # COVAR_SAMP function requires two columns, but we only have one
                    # This is a limitation - we'll use AVG as fallback
                    column_name = col.column.name if hasattr(col.column, "name") else col.column
                    select_parts.append(f"AVG({column_name})")
                elif col.function_name == "sum":
                    select_parts.append(f"SUM({col.column.name})")
                elif col.function_name == "avg":
                    select_parts.append(f"AVG({col.column.name})")
                elif col.function_name == "max":
                    select_parts.append(f"MAX({col.column.name})")
                elif col.function_name == "min":
                    select_parts.append(f"MIN({col.column.name})")
                else:
                    # Fallback for unknown aggregate functions
                    select_parts.append(
                        f"{col.function_name.upper()}({col.column.name if col.column else '*'})"
                    )

                # Add column with appropriate type
                if col.function_name == "count":
                    new_columns.append(Column(col.name, Integer, primary_key=False))
                elif col.function_name == "sum":
                    # Preserve source column type for SUM
                    column_name = col.column.name if hasattr(col.column, "name") else col.column
                    if column_name in source_table_obj.c:
                        source_type = str(source_table_obj.c[column_name].type).upper()
                        # Use Integer for integer types, Float for floating types
                        if any(
                            int_type in source_type
                            for int_type in ["INTEGER", "BIGINT", "SMALLINT", "INT"]
                        ):
                            new_columns.append(Column(col.name, Integer, primary_key=False))
                        else:
                            new_columns.append(Column(col.name, Float, primary_key=False))
                    else:
                        new_columns.append(Column(col.name, Float, primary_key=False))
                elif col.function_name == "avg":
                    new_columns.append(Column(col.name, Float, primary_key=False))
                else:
                    new_columns.append(Column(col.name, String, primary_key=False))
            else:
                pass

        # Ensure we have at least one column
        if not new_columns:
            # Add a placeholder column to avoid "Table must have at least one column!" error
            new_columns = [Column("placeholder", String, primary_key=False)]
            select_parts = ["'placeholder' as placeholder"]

        # Create target table using SQLAlchemy Table
        target_table_obj = Table(target_table, self.metadata, *new_columns)
        target_table_obj.create(self.engine, checkfirst=True)
        self._created_tables[target_table] = target_table_obj

        # Build and execute raw SQL
        select_clause = ", ".join(select_parts)
        sql = f"""
        INSERT INTO {target_table}
        SELECT {select_clause}
        FROM {source_table}
        """

        with Session(self.engine) as session:
            session.execute(text(sql))
            session.commit()

    def _apply_with_column(
        self, source_table: str, target_table: str, col_name: str, col: Any
    ) -> None:
        """Apply a withColumn operation."""
        source_table_obj = self._created_tables[source_table]

        # Copy existing columns and add new column
        new_columns: List[Any] = []

        # Copy all existing columns
        for column in source_table_obj.columns:
            new_columns.append(Column(column.name, column.type, primary_key=False))

        # Add new computed column - determine type based on operation
        if hasattr(col, "function_name") and hasattr(col, "window_spec"):
            new_columns.append(Column(col_name, Integer, primary_key=False))
        elif hasattr(col, "operation") and hasattr(col, "column") and hasattr(col, "value"):
            # Handle arithmetic operations - preserve source column type
            if hasattr(col, "function_name") and col.function_name in ["+", "-", "*", "/", "%"]:
                if col.function_name == "/":
                    # Division always returns floating-point type
                    new_columns.append(Column(col_name, Float, primary_key=False))
                elif hasattr(col.column, "name") and (
                    not hasattr(col.column, "operation") or col.column.operation is None
                ):
                    # Simple column reference - preserve its type
                    source_column = source_table_obj.c[col.column.name]
                    new_columns.append(Column(col_name, source_column.type, primary_key=False))
                else:
                    # Complex expression or nested operation - use Float for safety
                    new_columns.append(Column(col_name, Float, primary_key=False))
            else:
                new_columns.append(Column(col_name, String, primary_key=False))
        else:
            new_columns.append(Column(col_name, String, primary_key=False))

        # Handle window functions
        if hasattr(col, "function_name") and hasattr(col, "window_spec"):
            # For window functions, we need to use raw SQL
            self._apply_window_function(source_table, target_table, col_name, col, new_columns)
            return

        # Create target table using SQLAlchemy Table
        target_table_obj = Table(target_table, self.metadata, *new_columns)
        target_table_obj.create(self.engine, checkfirst=True)
        self._created_tables[target_table] = target_table_obj

        # For now, use raw SQL for complex expressions
        self._apply_with_column_sql(source_table, target_table, col_name, col)

    def _apply_window_function(
        self,
        source_table: str,
        target_table: str,
        col_name: str,
        window_func: Any,
        new_columns: List[Column],
    ) -> None:
        """Apply a window function using raw SQL."""
        source_table_obj = self._created_tables[source_table]

        # Create target table using SQLAlchemy Table
        target_table_obj = Table(target_table, self.metadata, *new_columns)
        target_table_obj.create(self.engine, checkfirst=True)
        self._created_tables[target_table] = target_table_obj

        # Build window function SQL
        window_sql = self._window_spec_to_sql(window_func.window_spec, source_table_obj)
        func_name = window_func.function_name.upper()

        # Generate function call based on type
        if func_name in ["ROW_NUMBER", "RANK", "DENSE_RANK", "CUME_DIST", "PERCENT_RANK"]:
            # These functions don't take parameters
            func_call = f"{func_name}() OVER ({window_sql})"
        else:
            # Get column from the original function if it exists
            original_function = getattr(window_func, "function", None)
            if (
                original_function
                and hasattr(original_function, "column")
                and original_function.column
            ):
                column_name = getattr(original_function.column, "name", "unknown")
                func_call = f'{func_name}("{column_name}") OVER ({window_sql})'
            else:
                func_call = f"{func_name}() OVER ({window_sql})"

        # Select all existing columns plus the window function result
        existing_cols = ", ".join([f'"{c.name}"' for c in source_table_obj.columns])

        sql = f"""
        INSERT INTO {target_table}
        SELECT {existing_cols}, {func_call} as {col_name}
        FROM {source_table}
        """

        # Execute SQL
        with Session(self.engine) as session:
            session.execute(text(sql))
            session.commit()

    def _apply_with_column_sql(
        self, source_table: str, target_table: str, col_name: str, col: Any
    ) -> None:
        """Apply withColumn using SQLAlchemy expressions for arithmetic operations."""
        # Get all existing columns from source
        source_table_obj = self._created_tables[source_table]
        existing_columns = [col.name for col in source_table_obj.columns]

        # Build the select statement using SQLAlchemy expressions
        select_columns = []
        for col_name_existing in existing_columns:
            select_columns.append(source_table_obj.c[col_name_existing])

        # Handle the new column expression using SQLAlchemy
        if hasattr(col, "operation") and hasattr(col, "column") and hasattr(col, "value"):
            # Handle arithmetic operations like MockColumnOperation
            # Check if left operand is a simple column or nested expression
            if hasattr(col.column, "name") and (
                not hasattr(col.column, "operation") or col.column.operation is None
            ):
                # Simple column
                left_col = source_table_obj.c[col.column.name]
            elif hasattr(col.column, "operation"):
                # Nested operation - convert to SQL first, then wrap in literal_column
                from sqlalchemy import literal_column

                nested_sql = self._expression_to_sql(col.column)
                left_col = literal_column(nested_sql)
            else:
                # Fallback
                left_col = source_table_obj.c[str(col.column)]

            # Convert right operand - handle MockColumn, MockLiteral, and raw values
            if hasattr(col.value, "name") and (
                not hasattr(col.value, "operation") or col.value.operation is None
            ):
                # MockColumn - convert to SQLAlchemy column reference
                right_val = source_table_obj.c[col.value.name]
            elif hasattr(col.value, "value"):
                # MockLiteral - extract the value
                right_val = col.value.value
            else:
                # Raw value (int, float, str)
                right_val = col.value

            if col.operation == "*":
                new_expr = left_col * right_val
            elif col.operation == "+":
                new_expr = left_col + right_val
            elif col.operation == "-":
                new_expr = left_col - right_val
            elif col.operation == "/":
                new_expr = left_col / right_val
            elif col.operation == "&":
                # Logical AND - recursively process both sides
                left_expr = self._expression_to_sqlalchemy(col.column, source_table_obj)
                right_expr = self._expression_to_sqlalchemy(col.value, source_table_obj)
                new_expr = and_(left_expr, right_expr)
            elif col.operation == "|":
                # Logical OR - recursively process both sides
                left_expr = self._expression_to_sqlalchemy(col.column, source_table_obj)
                right_expr = self._expression_to_sqlalchemy(col.value, source_table_obj)
                new_expr = or_(left_expr, right_expr)
            # Handle datetime functions (unary - value is None)
            elif col.value is None and col.operation in [
                "to_date",
                "to_timestamp",
                "hour",
                "minute",
                "second",
                "year",
                "month",
                "day",
                "dayofmonth",
                "dayofweek",
                "dayofyear",
                "weekofyear",
                "quarter",
            ]:
                datetime_sql = self._expression_to_sql(col)
                from sqlalchemy import literal_column

                new_expr = literal_column(datetime_sql)
            else:
                # Fallback to raw SQL for other operations
                new_expr = text(f"({left_col.name} {col.operation} {right_val})")

            # Safe label application - use .label() if available, otherwise use literal_column
            try:
                select_columns.append(new_expr.label(col_name))
            except (NotImplementedError, AttributeError):
                # For expressions that don't support .label(), use literal_column
                from sqlalchemy import literal_column

                select_columns.append(literal_column(str(new_expr)).label(col_name))
        else:
            # Fallback to raw SQL for other expressions
            from sqlalchemy import literal_column

            new_col_sql = self._expression_to_sql(col)
            # For literals, use literal() instead of text()
            if hasattr(col, "value") and isinstance(col.value, str):
                select_columns.append(literal(col.value).label(col_name))
            else:
                select_columns.append(literal_column(new_col_sql).label(col_name))

        # Create the select statement
        select_stmt = select(*select_columns).select_from(source_table_obj)

        # Create the target table with the new column
        new_columns: List[Any] = []
        for col_name_existing in existing_columns:
            col_type = source_table_obj.c[col_name_existing].type
            new_columns.append(Column(col_name_existing, col_type, primary_key=False))

        # Add the new column with appropriate type
        if hasattr(col, "operation") and hasattr(col, "column"):
            # Determine type based on operation
            if col.operation in ["to_date"]:
                from sqlalchemy import Date

                new_columns.append(Column(col_name, Date, primary_key=False))
            elif col.operation in ["to_timestamp", "current_timestamp"]:
                from sqlalchemy import DateTime

                new_columns.append(Column(col_name, DateTime, primary_key=False))
            elif col.operation in [
                "hour",
                "minute",
                "second",
                "year",
                "month",
                "day",
                "dayofmonth",
                "dayofweek",
                "dayofyear",
                "weekofyear",
                "quarter",
            ]:
                # All datetime component extractions return integers
                new_columns.append(Column(col_name, Integer, primary_key=False))
            elif hasattr(col, "value") and col.value is not None:
                # For arithmetic operations with a value, use Float type
                new_columns.append(Column(col_name, Float, primary_key=False))
            else:
                # Default to String for unknown operations
                new_columns.append(Column(col_name, String, primary_key=False))
        else:
            new_columns.append(Column(col_name, String, primary_key=False))

        target_table_obj = Table(target_table, self.metadata, *new_columns, extend_existing=True)
        target_table_obj.create(self.engine, checkfirst=True)
        self._created_tables[target_table] = target_table_obj

        # Execute the insert
        with self.engine.connect() as conn:
            conn.execute(
                insert(target_table_obj).from_select([c.name for c in new_columns], select_stmt)
            )
            conn.commit()

    def _apply_order_by(
        self, source_table: str, target_table: str, columns: Tuple[Any, ...]
    ) -> None:
        """Apply an orderBy operation using SQLAlchemy expressions."""
        source_table_obj = self._created_tables[source_table]

        # Copy table structure
        self._copy_table_structure(source_table, target_table)
        target_table_obj = self._created_tables[target_table]

        # Build SQLAlchemy order by expressions
        order_expressions = []
        [c.name for c in source_table_obj.columns]
        # print(f"DEBUG: Available columns in {source_table}: {available_columns}")
        # print(f"DEBUG: Order by columns: {[col.name if hasattr(col, 'name') else str(col) for col in columns]}")

        for col in columns:
            if isinstance(col, str):
                order_expressions.append(source_table_obj.c[col])
            elif hasattr(col, "operation") and col.operation == "desc":
                order_expressions.append(desc(source_table_obj.c[col.column.name]))
            elif hasattr(col, "operation") and col.operation == "asc":
                order_expressions.append(asc(source_table_obj.c[col.column.name]))
            elif hasattr(col, "name"):
                # Handle MockColumn objects - check if they have a desc or asc operation
                if hasattr(col, "operation") and col.operation == "desc":
                    order_expressions.append(desc(source_table_obj.c[col.name]))
                elif hasattr(col, "operation") and col.operation == "asc":
                    order_expressions.append(asc(source_table_obj.c[col.name]))
                else:
                    # Default to ascending order
                    order_expressions.append(asc(source_table_obj.c[col.name]))
            else:
                # Fallback: try to convert to string
                order_expressions.append(source_table_obj.c[str(col)])

        # Execute with ORDER BY using SQLAlchemy
        with Session(self.engine) as session:
            query = select(*source_table_obj.columns).order_by(*order_expressions)
            results: List[Any] = list(session.execute(query).all())

            # Insert into target table
            for result in results:
                # Convert result to dict using column names
                result_dict = {}
                for i, column in enumerate(source_table_obj.columns):
                    result_dict[column.name] = result[i]
                insert_stmt = target_table_obj.insert().values(result_dict)
                session.execute(insert_stmt)
            session.commit()

    def _apply_limit(self, source_table: str, target_table: str, limit_count: int) -> None:
        """Apply a limit operation using SQLAlchemy expressions."""
        source_table_obj = self._created_tables[source_table]

        # Copy table structure
        self._copy_table_structure(source_table, target_table)
        target_table_obj = self._created_tables[target_table]

        # Execute with LIMIT using SQLAlchemy
        with Session(self.engine) as session:
            query = select(*source_table_obj.columns).limit(limit_count)
            results: List[Any] = list(session.execute(query).all())

            # Insert into target table
            for result in results:
                # Convert result to dict using column names
                result_dict = {}
                for i, column in enumerate(source_table_obj.columns):
                    result_dict[column.name] = result[i]
                insert_stmt = target_table_obj.insert().values(result_dict)
                session.execute(insert_stmt)
            session.commit()

    def _build_case_when_sql(self, case_when_obj: Any, source_table_obj: Any) -> str:
        """Build SQL CASE WHEN expression from MockCaseWhen object."""
        sql_parts = ["CASE"]

        # Add WHEN conditions
        for condition, value in case_when_obj.conditions:
            # Convert condition to SQL
            condition_sql = self._condition_to_sql(condition, source_table_obj)
            # Convert value to SQL
            value_sql = self._value_to_sql(value)
            sql_parts.append(f"WHEN {condition_sql} THEN {value_sql}")

        # Add ELSE clause if default_value is set
        if case_when_obj.default_value is not None:
            else_sql = self._value_to_sql(case_when_obj.default_value)
            sql_parts.append(f"ELSE {else_sql}")

        sql_parts.append("END")
        return " ".join(sql_parts)

    def _condition_to_sql(self, condition: Any, source_table_obj: Any) -> str:
        """Convert a condition to SQL."""
        if hasattr(condition, "column") and hasattr(condition, "function_name"):
            # Handle column operations like F.col("age") > 30
            column_name = condition.column.name
            value = condition.value

            # Convert value to SQL
            value_sql = self._value_to_sql(value)

            if condition.function_name == ">" or condition.function_name == "gt":
                return f'"{column_name}" > {value_sql}'
            elif condition.function_name == "<" or condition.function_name == "lt":
                return f'"{column_name}" < {value_sql}'
            elif condition.function_name == "==" or condition.function_name == "eq":
                return f'"{column_name}" = {value_sql}'
            elif condition.function_name == "!=" or condition.function_name == "ne":
                return f'"{column_name}" != {value_sql}'
            elif condition.function_name == ">=" or condition.function_name == "ge":
                return f'"{column_name}" >= {value_sql}'
            elif condition.function_name == "<=" or condition.function_name == "le":
                return f'"{column_name}" <= {value_sql}'
        return str(condition)

    def _value_to_sql(self, value: Any) -> str:
        """Convert a value to SQL."""
        if value is None:
            return "NULL"
        elif isinstance(value, str):
            return f"'{value}'"
        elif isinstance(value, (int, float)):
            return str(value)
        elif hasattr(value, "operation") and hasattr(value, "column") and hasattr(value, "value"):
            # Handle arithmetic/comparison operations (MockColumnOperation)
            return self._expression_to_sql(value)
        elif hasattr(value, "name") and not hasattr(value, "operation"):
            # Handle MockColumn - convert to SQL column reference
            return self._column_to_sql(value)
        elif hasattr(value, "value") and hasattr(value, "data_type"):
            # Handle MockLiteral
            if value.value is None:
                return "NULL"
            elif isinstance(value.value, str):
                return f"'{value.value}'"
            else:
                return str(value.value)
        elif hasattr(value, "name"):
            # Handle column references (MockColumn)
            return f'"{value.name}"'
        else:
            return str(value)

    def _copy_table_structure(self, source_table: str, target_table: str) -> None:
        """Copy table structure from source to target."""
        source_table_obj = self._created_tables[source_table]

        # Copy all columns from source table
        new_columns: List[Any] = []
        for column in source_table_obj.columns:
            new_columns.append(Column(column.name, column.type, primary_key=False))

        # print(f"DEBUG: Copying table structure from {source_table} to {target_table}")
        # print(f"DEBUG: Source columns: {[c.name for c in source_table_obj.columns]}")
        # print(f"DEBUG: Target columns: {[c.name for c in new_columns]}")

        # Create target table using SQLAlchemy Table
        target_table_obj = Table(target_table, self.metadata, *new_columns)
        target_table_obj.create(self.engine, checkfirst=True)
        self._created_tables[target_table] = target_table_obj

    def _get_table_results(self, table_name: str) -> List[MockRow]:
        """Get all results from a table as MockRow objects."""
        table_obj = self._created_tables[table_name]

        with Session(self.engine) as session:
            # Build raw SQL query
            # Escape double quotes in column names by doubling them
            column_names = [
                f'"{col.name.replace(chr(34), chr(34)+chr(34))}"' for col in table_obj.columns
            ]
            sql = f"SELECT {', '.join(column_names)} FROM {table_name}"
            results = session.execute(text(sql)).all()

            mock_rows = []
            for result in results:
                # Convert result to dict using column names with type conversion
                result_dict = {}
                for i, column in enumerate(table_obj.columns):
                    value = result[i]
                    # Convert value to appropriate type based on column type
                    if isinstance(column.type, Integer) and value is not None:
                        try:
                            result_dict[column.name] = int(value)
                        except (ValueError, TypeError):
                            result_dict[column.name] = value
                    elif isinstance(column.type, Float) and value is not None:
                        try:
                            result_dict[column.name] = float(value)  # type: ignore[assignment]
                        except (ValueError, TypeError):
                            result_dict[column.name] = value
                    elif isinstance(column.type, Boolean) and value is not None:
                        if isinstance(value, str):
                            result_dict[column.name] = value.lower() in ("true", "1", "yes", "on")
                        else:
                            result_dict[column.name] = bool(value)
                    else:
                        result_dict[column.name] = value
                mock_rows.append(MockRow(result_dict))

            return mock_rows

    def _condition_to_sqlalchemy(self, table_obj: Any, condition: Any) -> Any:
        """Convert a condition to SQLAlchemy expression."""
        if isinstance(condition, MockColumnOperation):
            if hasattr(condition, "operation") and hasattr(condition, "column"):
                left = self._column_to_sqlalchemy(table_obj, condition.column)
                right = self._value_to_sqlalchemy(condition.value)

                if condition.operation == "==":
                    return left == right
                elif condition.operation == "!=":
                    return left != right
                elif condition.operation == ">":
                    return left > right
                elif condition.operation == "<":
                    return left < right
                elif condition.operation == ">=":
                    return left >= right
                elif condition.operation == "<=":
                    return left <= right
                elif condition.operation == "&":
                    # Logical AND operation
                    left_expr = self._condition_to_sqlalchemy(table_obj, condition.column)
                    right_expr = self._condition_to_sqlalchemy(table_obj, condition.value)
                    return and_(left_expr, right_expr)
                elif condition.operation == "|":
                    # Logical OR operation
                    left_expr = self._condition_to_sqlalchemy(table_obj, condition.column)
                    right_expr = self._condition_to_sqlalchemy(table_obj, condition.value)
                    return or_(left_expr, right_expr)
                elif condition.operation == "!":
                    # Logical NOT operation
                    expr = self._condition_to_sqlalchemy(table_obj, condition.column)
                    return ~expr
                elif condition.operation == "isnull":
                    # IS NULL operation
                    left = self._column_to_sqlalchemy(table_obj, condition.column)
                    return left.is_(None)
                elif condition.operation == "isnotnull":
                    # IS NOT NULL operation
                    left = self._column_to_sqlalchemy(table_obj, condition.column)
                    return left.isnot(None)
        elif isinstance(condition, MockColumn):
            return table_obj.c[condition.name]

        return None  # Fallback

    def _column_to_sqlalchemy(self, table_obj: Any, column: Any) -> Any:
        """Convert a MockColumn to SQLAlchemy expression."""
        if isinstance(column, MockColumn):
            column_name = column.name
        elif isinstance(column, str):
            column_name = column
        else:
            return column

        # Validate column exists
        if column_name not in table_obj.c:
            # Only raise errors if we're in strict validation mode (e.g., filters)
            # Window functions and other operations handle missing columns differently
            if getattr(self, "_strict_column_validation", False):
                from mock_spark.core.exceptions import AnalysisException

                available_columns = list(table_obj.c.keys())
                raise AnalysisException(
                    f"Column '{column_name}' not found. Available columns: {available_columns}"
                )
            else:
                # For window functions and other contexts, return literal False
                return literal(False)

        return table_obj.c[column_name]

    def _expression_to_sqlalchemy(self, expr: Any, table_obj: Any) -> Any:
        """Convert a complex expression (including AND/OR) to SQLAlchemy."""
        if isinstance(expr, MockColumnOperation):
            # Recursively process left and right sides
            if hasattr(expr, "column"):
                left = self._expression_to_sqlalchemy(expr.column, table_obj)
            else:
                left = None

            if hasattr(expr, "value") and expr.value is not None:
                if isinstance(expr.value, (MockColumn, MockColumnOperation)):
                    right = self._expression_to_sqlalchemy(expr.value, table_obj)
                elif isinstance(expr.value, MockLiteral):
                    right = expr.value.value
                else:
                    right = expr.value
            else:
                right = None

            # Apply operation
            if expr.operation == ">":
                return left > right
            elif expr.operation == "<":
                return left < right
            elif expr.operation == ">=":
                return left >= right
            elif expr.operation == "<=":
                return left <= right
            elif expr.operation == "==":
                return left == right
            elif expr.operation == "!=":
                return left != right
            elif expr.operation == "&":
                return and_(left, right)
            elif expr.operation == "|":
                return or_(left, right)
            elif expr.operation == "!":
                return ~left
            else:
                # Fallback
                return table_obj.c[str(expr)]
        elif isinstance(expr, MockColumn):
            return table_obj.c[expr.name]
        elif isinstance(expr, MockLiteral):
            return expr.value
        else:
            # Literal value
            return expr

    def _value_to_sqlalchemy(self, value: Any) -> Any:
        """Convert a value to SQLAlchemy expression."""
        if isinstance(value, MockLiteral):
            return value.value
        elif isinstance(value, MockColumn):
            # This would need the table context, but for now return the name
            return value.name
        return value

    def _column_to_orm(self, table_class: Any, column: Any) -> Any:
        """Convert a MockColumn to SQLAlchemy ORM expression."""
        if isinstance(column, MockColumn):
            return getattr(table_class, column.name)
        elif isinstance(column, str):
            return getattr(table_class, column)
        return column

    def _value_to_orm(self, value: Any) -> Any:
        """Convert a value to SQLAlchemy ORM expression."""
        if isinstance(value, MockLiteral):
            return value.value
        elif isinstance(value, MockColumn):
            # This would need the table class context, but for now return the name
            return value.name
        return value

    def _window_function_to_orm(self, table_class: Any, window_func: Any) -> Any:
        """Convert a window function to SQLAlchemy ORM expression."""
        function_name = getattr(window_func, "function_name", "window_function")

        # Get window specification
        window_spec = window_func.window_spec

        # Build partition_by and order_by
        partition_by = []
        order_by = []

        if hasattr(window_spec, "_partition_by") and window_spec._partition_by:
            for col in window_spec._partition_by:
                if isinstance(col, str):
                    partition_by.append(getattr(table_class, col))
                elif hasattr(col, "name"):
                    partition_by.append(getattr(table_class, col.name))

        if hasattr(window_spec, "_order_by") and window_spec._order_by:
            for col in window_spec._order_by:
                if isinstance(col, str):
                    order_by.append(getattr(table_class, col))
                elif hasattr(col, "operation") and col.operation == "desc":
                    order_by.append(desc(getattr(table_class, col.column.name)))
                elif hasattr(col, "name"):
                    order_by.append(getattr(table_class, col.name))

        # Build window expression
        if function_name == "rank":
            return func.rank().over(partition_by=partition_by, order_by=order_by)
        elif function_name == "row_number":
            return func.row_number().over(partition_by=partition_by, order_by=order_by)
        elif function_name == "dense_rank":
            return func.dense_rank().over(partition_by=partition_by, order_by=order_by)
        else:
            # Generic window function
            return getattr(func, function_name)().over(partition_by=partition_by, order_by=order_by)

    def _window_spec_to_sql(self, window_spec: Any, table_obj: Any = None) -> str:
        """Convert window specification to SQL."""
        parts = []

        # Get available columns if table_obj provided
        available_columns = set(table_obj.c.keys()) if table_obj is not None else None

        # Handle PARTITION BY
        if hasattr(window_spec, "_partition_by") and window_spec._partition_by:
            partition_cols = []
            for col in window_spec._partition_by:
                col_name = None
                if isinstance(col, str):
                    col_name = col
                elif hasattr(col, "name"):
                    col_name = col.name

                # Validate column exists if available_columns is set
                if available_columns is not None and col_name and col_name not in available_columns:
                    continue  # Skip non-existent columns

                if col_name:
                    partition_cols.append(f'"{col_name}"')

            if partition_cols:
                parts.append(f"PARTITION BY {', '.join(partition_cols)}")

        # Handle ORDER BY
        if hasattr(window_spec, "_order_by") and window_spec._order_by:
            order_cols = []
            for col in window_spec._order_by:
                col_name = None
                is_desc = False

                if isinstance(col, str):
                    col_name = col
                elif isinstance(col, MockColumnOperation):
                    if hasattr(col, "operation") and col.operation == "desc":
                        col_name = col.column.name
                        is_desc = True
                    else:
                        col_name = col.column.name
                elif hasattr(col, "name"):
                    col_name = col.name

                # Validate column exists if available_columns is set
                if available_columns is not None and col_name and col_name not in available_columns:
                    continue  # Skip non-existent columns

                if col_name:
                    if is_desc:
                        order_cols.append(f'"{col_name}" DESC')
                    else:
                        order_cols.append(f'"{col_name}"')

            if order_cols:
                parts.append(f"ORDER BY {', '.join(order_cols)}")

        # Handle ROWS BETWEEN
        if hasattr(window_spec, "_rows_between") and window_spec._rows_between:
            start, end = window_spec._rows_between
            # Convert to SQL ROWS BETWEEN syntax
            # Negative values are PRECEDING, positive are FOLLOWING
            if start == 0:
                start_clause = "CURRENT ROW"
            elif start < 0:
                start_clause = f"{abs(start)} PRECEDING"
            else:
                start_clause = f"{start} FOLLOWING"

            if end == 0:
                end_clause = "CURRENT ROW"
            elif end < 0:
                end_clause = f"{abs(end)} PRECEDING"
            else:
                end_clause = f"{end} FOLLOWING"

            parts.append(f"ROWS BETWEEN {start_clause} AND {end_clause}")

        # Handle RANGE BETWEEN
        if hasattr(window_spec, "_range_between") and window_spec._range_between:
            start, end = window_spec._range_between
            # Convert to SQL RANGE BETWEEN syntax
            if start == 0:
                start_clause = "CURRENT ROW"
            elif start < 0:
                start_clause = f"{abs(start)} PRECEDING"
            else:
                start_clause = f"{start} FOLLOWING"

            if end == 0:
                end_clause = "CURRENT ROW"
            elif end < 0:
                end_clause = f"{abs(end)} PRECEDING"
            else:
                end_clause = f"{end} FOLLOWING"

            parts.append(f"RANGE BETWEEN {start_clause} AND {end_clause}")

        return " ".join(parts)

    def _apply_join(
        self, source_table: str, target_table: str, join_params: Tuple[Any, ...]
    ) -> None:
        """Apply a join operation."""
        other_df, on, how = join_params

        source_table_obj = self._created_tables[source_table]

        # Materialize the other DataFrame to get its data
        other_materialized = other_df._materialize_if_lazy() if other_df.is_lazy else other_df
        other_data = other_materialized.data
        other_schema = other_materialized.schema

        # Normalize 'on' parameter to list
        if isinstance(on, str):
            on_columns = [on]
        elif isinstance(on, list):
            on_columns = on
        else:
            on_columns = [on]

        # Create target table with combined schema
        new_columns: List[Any] = []

        # Add all columns from source table
        for column in source_table_obj.columns:
            new_columns.append(Column(column.name, column.type, primary_key=False))

        # Add columns from other DataFrame (except join keys already in source)
        for field in other_schema.fields:
            if field.name not in on_columns and field.name not in [
                c.name for c in source_table_obj.columns
            ]:
                # Convert MockSpark types to SQLAlchemy types
                sql_type: Any = String  # Default, can be Integer, Float, or other types
                field_type_name = type(field.dataType).__name__
                if field_type_name in ["LongType", "IntegerType"]:
                    sql_type = Integer
                elif field_type_name in ["DoubleType", "FloatType"]:
                    sql_type = Float
                new_columns.append(Column(field.name, sql_type, primary_key=False))

        # Create target table
        target_table_obj = Table(target_table, self.metadata, *new_columns)
        target_table_obj.create(self.engine, checkfirst=True)
        self._created_tables[target_table] = target_table_obj

        # Perform the actual join operation
        with Session(self.engine) as session:
            # Get source data
            source_data = session.execute(select(*source_table_obj.columns)).all()

            # Create a lookup dictionary from other_data (key -> list of matching rows)
            other_lookup: Dict[Any, Any] = {}
            for other_row in other_data:
                # Create join key from on_columns
                join_key = tuple(other_row.get(col) for col in on_columns)
                if join_key not in other_lookup:
                    other_lookup[join_key] = []
                other_lookup[join_key].append(other_row)

            # Perform join - create one output row for each matching pair
            for row in source_data:
                row_dict = dict(row._mapping)

                # Create join key from source row
                source_join_key = tuple(row_dict.get(col) for col in on_columns)

                # Look up all matching rows from other DataFrame
                if source_join_key in other_lookup:
                    for other_row in other_lookup[source_join_key]:
                        # Create a new combined row for each match
                        combined_row = row_dict.copy()

                        # Add columns from other DataFrame
                        for field in other_schema.fields:
                            if field.name not in on_columns and field.name not in combined_row:
                                combined_row[field.name] = other_row.get(field.name)

                        # Ensure all target columns have values
                        target_column_names = [col.name for col in target_table_obj.columns]
                        complete_row = {}
                        for col_name in target_column_names:
                            complete_row[col_name] = combined_row.get(col_name, None)

                        # Insert into target table
                        insert_stmt = target_table_obj.insert().values(complete_row)
                        session.execute(insert_stmt)

            session.commit()

    def _apply_union(self, source_table: str, target_table: str, other_df: Any) -> None:
        """Apply a union operation."""
        # Get source table structure
        source_table_obj = self._created_tables[source_table]
        new_columns: List[Any] = []
        for column in source_table_obj.columns:
            new_columns.append(Column(column.name, column.type, primary_key=False))

        # Create target table with same structure
        target_table_obj = Table(target_table, self.metadata, *new_columns)
        target_table_obj.create(self.engine, checkfirst=True)
        self._created_tables[target_table] = target_table_obj

        # Combine data from both dataframes
        with Session(self.engine) as session:
            # Get source data
            source_data = session.execute(select(*source_table_obj.columns)).all()
            for row in source_data:
                row_dict = dict(row._mapping)
                insert_stmt = target_table_obj.insert().values(row_dict)
                session.execute(insert_stmt)

            # Get other dataframe data by materializing it
            other_data = other_df.collect()
            for row in other_data:
                row_dict = dict(row.asDict())
                insert_stmt = target_table_obj.insert().values(row_dict)
                session.execute(insert_stmt)

            session.commit()

    def _expression_to_sql(self, expr: Any) -> str:
        """Convert an expression to SQL."""
        if isinstance(expr, str):
            return f'"{expr}"'
        elif hasattr(expr, "conditions") and hasattr(expr, "default_value"):
            # Handle MockCaseWhen objects
            return self._build_case_when_sql(expr, None)
        elif hasattr(expr, "operation") and hasattr(expr, "column") and hasattr(expr, "value"):
            # Handle string/math functions like upper, lower, abs, etc.
            if expr.operation in ["upper", "lower", "length", "trim", "abs", "round"]:
                column_name = self._column_to_sql(expr.column)
                return f"{expr.operation.upper()}({column_name})"

            # Handle unary operations (value is None)
            if expr.value is None:
                left = self._column_to_sql(expr.column)
                if expr.operation == "-":
                    return f"(-{left})"
                elif expr.operation == "+":
                    return f"(+{left})"
                # Handle datetime functions
                elif expr.operation in ["to_date", "to_timestamp"]:
                    # DuckDB: CAST(column AS DATE/TIMESTAMP)
                    target_type = "DATE" if expr.operation == "to_date" else "TIMESTAMP"
                    return f"CAST({left} AS {target_type})"
                elif expr.operation in ["hour", "minute", "second"]:
                    # DuckDB: extract(part from timestamp)
                    return f"extract({expr.operation} from CAST({left} AS TIMESTAMP))"
                elif expr.operation in ["year", "month", "day", "dayofmonth"]:
                    # DuckDB: extract(part from date)
                    part = "day" if expr.operation == "dayofmonth" else expr.operation
                    return f"extract({part} from CAST({left} AS DATE))"
                elif expr.operation in ["dayofweek", "dayofyear", "weekofyear", "quarter"]:
                    # DuckDB date part extraction
                    part_map = {
                        "dayofweek": "dow",
                        "dayofyear": "doy",
                        "weekofyear": "week",
                        "quarter": "quarter",
                    }
                    part = part_map.get(expr.operation, expr.operation)
                    return f"extract({part} from CAST({left} AS DATE))"
                else:
                    # For other unary operations, treat as function
                    return f"{expr.operation.upper()}({left})"

            # Handle arithmetic operations like MockColumnOperation
            # For column references in expressions, don't quote them
            left = self._column_to_sql(expr.column)
            right = self._value_to_sql(expr.value)

            # Handle comparison operations
            if expr.operation == "==":
                # Handle NULL comparisons specially
                if right == "NULL":
                    return f"({left} IS NULL)"
                return f"({left} = {right})"
            elif expr.operation == "!=":
                # Handle NULL comparisons specially
                if right == "NULL":
                    return f"({left} IS NOT NULL)"
                return f"({left} <> {right})"
            elif expr.operation == ">":
                return f"({left} > {right})"
            elif expr.operation == "<":
                return f"({left} < {right})"
            elif expr.operation == ">=":
                return f"({left} >= {right})"
            elif expr.operation == "<=":
                return f"({left} <= {right})"
            # Handle arithmetic operations
            elif expr.operation == "*":
                return f"({left} * {right})"
            elif expr.operation == "+":
                return f"({left} + {right})"
            elif expr.operation == "-":
                return f"({left} - {right})"
            elif expr.operation == "/":
                return f"({left} / {right})"
            else:
                return f"({left} {expr.operation} {right})"
        elif hasattr(expr, "name"):
            return f'"{expr.name}"'
        elif hasattr(expr, "value"):
            # Handle literals
            if isinstance(expr.value, str):
                return f"'{expr.value}'"
            else:
                return str(expr.value)
        else:
            return str(expr)

    def _column_to_sql(self, expr: Any) -> str:
        """Convert a column reference to SQL without quotes for expressions."""
        if isinstance(expr, str):
            return expr
        elif hasattr(expr, "name"):
            return expr.name
        else:
            return str(expr)

    def close(self) -> None:
        """Close the SQLAlchemy engine."""
        try:
            if hasattr(self, "engine") and self.engine:
                self.engine.dispose()
                self.engine = None  # type: ignore[assignment]
        except Exception:
            pass  # Ignore errors during cleanup

    def __del__(self) -> None:
        """Cleanup on deletion to prevent resource leaks."""
        try:
            self.close()
        except:  # noqa: E722
            pass
