{% macro glue__py_write_table(compiled_code, target_relation) %}
{{ compiled_code }}

# --- Autogenerated dbt materialization code. --- #
from pyspark.sql import SparkSession
from pyspark.sql.functions import *

# Get or create SparkSession
spark = SparkSession.builder.getOrCreate()

# Set up the schema and model name
schema = "{{ target_relation.schema }}"
model_name = "{{ target_relation.identifier }}"
print("DEBUG: Setting up model in schema: " + schema)

class dbtObj:
    def __init__(self, table_function, schema, model_name):
        self.table_function = table_function
        self.schema = schema
        self.model_name = model_name
        self.this = schema + "." + model_name
        
    def config(self, **config_args):
        # This is a placeholder for dbt config in Python models
        print("DEBUG: dbt.config called with:", config_args)
        pass
        
    def ref(self, name):
        return self.table_function(name)
        
    def source(self, source_name, table_name):
        return self.table_function(source_name + "." + table_name)
        
    def is_incremental(self):
        # Check if the target table exists by trying to query it
        try:
            spark.sql("SELECT 1 FROM " + schema + "." + model_name + " LIMIT 1")
            print("DEBUG: Table exists, this is an incremental run")
            return True
        except Exception as e:
            print("DEBUG: Table does not exist, this is a full refresh run:", str(e))
            return False

# Initialize dbt object
dbt = dbtObj(spark.table, schema, model_name)

# Execute the model function
print("DEBUG: About to execute model function")
df = None

try:
    print("DEBUG: Calling model function...")
    df = model(dbt, spark)
    print("DEBUG: Model function executed successfully, got DataFrame: " + str(type(df)))
except NameError as e:
    print("DEBUG: NameError when calling model:", e)
    try:
        print("DEBUG: Trying main function as fallback...")
        df = main(dbt, spark)
        print("DEBUG: Main function executed successfully, got DataFrame: " + str(type(df)))
    except NameError as e2:
        print("DEBUG: NameError when calling main:", e2)
        print("DEBUG: Available functions:")
        import types
        for name, obj in globals().items():
            if isinstance(obj, types.FunctionType) and not name.startswith('_'):
                print("  -", name, ":", type(obj))
        raise Exception("Neither 'model' nor 'main' function found in the Python code")
except Exception as e:
    print("DEBUG: Exception when calling model function:", e)
    import traceback
    traceback.print_exc()
    raise e

# Validate that we got a DataFrame
import pyspark.sql.dataframe
if not isinstance(df, pyspark.sql.dataframe.DataFrame):
    raise Exception("Model function must return a Spark DataFrame, got " + str(type(df)))

# Write the DataFrame to the target table
print("DEBUG: Writing DataFrame to table {{ target_relation.schema }}.{{ target_relation.identifier }}")
writer = df.write.mode("overwrite").option("overwriteSchema", "true")

# Apply file format and other options from config
{%- set file_format = config.get('file_format', 'parquet') -%}
{%- set partition_by = config.get('partition_by', none) -%}
{%- set custom_location = config.get('custom_location', '') -%}

# Determine the correct relation to use (with catalog prefix for Iceberg)
{%- if file_format == 'iceberg' -%}
    {%- set full_relation = glue__make_target_relation(this, file_format) -%}
{%- else -%}
    {%- set full_relation = this -%}
{%- endif -%}

{%- if file_format %}
writer = writer.format("{{ file_format }}")
{%- endif %}

{%- if partition_by is not none -%}
    {%- if partition_by is string -%}
        {%- set partition_by = [partition_by] -%}
    {%- endif %}
writer = writer.partitionBy({{ partition_by | tojson }})
{%- endif %}

{%- if custom_location %}
writer = writer.option("path", "{{ custom_location }}")
{%- endif %}

# For Iceberg tables, use SQL approach exclusively (saveAsTable doesn't support Iceberg)
{%- if file_format == 'iceberg' -%}
# Create temp view first
df.createOrReplaceTempView("temp_python_df")
print("DEBUG: Created temp view temp_python_df")

# Check if temp view was created successfully
temp_count = spark.sql("SELECT COUNT(*) FROM temp_python_df").collect()[0][0]
print("DEBUG: Temp view has", temp_count, "rows")

# Use the catalog-aware relation for Iceberg table creation
table_name = "{{ full_relation }}"
print("DEBUG: Creating Iceberg table:", table_name)

# Check if this is an incremental model with merge strategy
materialized = "{{ config.get('materialized', 'python_model') }}"
incremental_strategy = "{{ config.get('incremental_strategy', 'append') }}"
unique_key = "{{ config.get('unique_key', none) }}"

print("DEBUG: materialized =", materialized)
print("DEBUG: incremental_strategy =", incremental_strategy)
print("DEBUG: unique_key =", unique_key)

is_incremental_merge = (materialized == "incremental" and 
                       incremental_strategy == "merge" and 
                       unique_key != "None" and 
                       unique_key != "none" and 
                       unique_key != "")

print("DEBUG: is_incremental_merge =", is_incremental_merge)

# Check if target table already exists (for incremental logic)
table_exists = False
try:
    spark.sql(f"DESCRIBE TABLE {table_name}")
    table_exists = True
    print("DEBUG: Target table exists - this is an incremental run")
except Exception as e:
    print("DEBUG: Target table does not exist - this is a full refresh run")

try:
    if is_incremental_merge and table_exists:
        # For incremental merge, use MERGE INTO statement
        print("DEBUG: Using MERGE INTO for incremental update")
        
        # Parse unique_key (handle both string and list formats)
        if unique_key.startswith('[') and unique_key.endswith(']'):
            # List format: ['id'] or ['id', 'name']
            import ast
            unique_key_list = ast.literal_eval(unique_key)
        else:
            # String format: 'id'
            unique_key_list = [unique_key]
        
        print("DEBUG: unique_key_list =", unique_key_list)
        
        # Create the merge conditions
        merge_conditions = []
        for key in unique_key_list:
            merge_conditions.append(f"target.{key} = source.{key}")
        merge_condition = " AND ".join(merge_conditions)
        
        merge_sql = f"""
        MERGE INTO {table_name} AS target
        USING temp_python_df AS source
        ON {merge_condition}
        WHEN MATCHED THEN UPDATE SET *
        WHEN NOT MATCHED THEN INSERT *
        """
        
        print("DEBUG: Executing MERGE SQL:", merge_sql)
        spark.sql(merge_sql)
        print("DEBUG: MERGE completed successfully")
        
    else:
        # For first run or non-merge strategies, use CREATE OR REPLACE
        print("DEBUG: Using CREATE OR REPLACE TABLE for full refresh")
        create_sql = "CREATE OR REPLACE TABLE " + table_name + " USING ICEBERG AS SELECT * FROM temp_python_df"
        print("DEBUG: Executing SQL:", create_sql)
        spark.sql(create_sql)
        print("DEBUG: Iceberg table created successfully")
    
    # Clean up temp view to avoid conflicts with subsequent models
    spark.sql("DROP VIEW IF EXISTS temp_python_df")
    print("DEBUG: Cleaned up temp view")
    
except Exception as e:
    print("DEBUG: Error creating Iceberg table:", str(e))
    print("DEBUG: Trying with CREATE TABLE IF NOT EXISTS...")
    try:
        create_sql_fallback = "CREATE TABLE IF NOT EXISTS " + table_name + " USING ICEBERG AS SELECT * FROM temp_python_df"
        spark.sql(create_sql_fallback)
        print("DEBUG: Iceberg table created with IF NOT EXISTS")
        
        # Clean up temp view
        spark.sql("DROP VIEW IF EXISTS temp_python_df")
        print("DEBUG: Cleaned up temp view")
        
    except Exception as e2:
        print("DEBUG: Both Iceberg approaches failed:", str(e2))
        # For Iceberg, we must use SQL - no fallback to saveAsTable
        raise Exception("Failed to create Iceberg table: " + str(e2))
{%- else -%}
# For non-Iceberg tables, use the standard saveAsTable approach
writer.saveAsTable("{{ target_relation.schema }}.{{ target_relation.identifier }}")
{%- endif %}

# Refresh the table to make it available - use catalog-qualified name for Iceberg
{%- if file_format == 'iceberg' -%}
spark.sql("REFRESH TABLE {{ full_relation }}")
print("DEBUG: Successfully wrote table {{ full_relation }}")
{%- else -%}
spark.sql("REFRESH TABLE {{ target_relation.schema }}.{{ target_relation.identifier }}")
print("DEBUG: Successfully wrote table {{ target_relation.schema }}.{{ target_relation.identifier }}")
{%- endif %}
print("DEBUG: Python model execution completed successfully")
{% endmacro %}

{% macro glue__py_get_writer_options() %}
    {%- set file_format = config.get('file_format', 'parquet') -%}
    {%- set location = adapter.get_location(this) -%}
    {%- set partition_by = config.get('partition_by', none) -%}
    {%- set table_properties = config.get('table_properties', {}) -%}

    {%- set options = {
        'file_format': file_format,
        'location': location,
        'partition_by': partition_by,
        'table_properties': table_properties
    } -%}

    {{ return(options) }}
{% endmacro %}

{% macro glue__create_python_merge_table(model, df_name, unique_key) %}
    {%- set target_relation = this.incorporate(type='table') -%}
    {%- set temp_view = model['name'] + '_temp_view' -%}

    -- Create a temporary view of the new data
    {{ df_name }}.createOrReplaceTempView('{{ temp_view }}')

    -- Merge the data using the unique key
    MERGE INTO {{ target_relation }} AS target
    USING {{ temp_view }} AS source
    ON {% for key in unique_key %}
        target.{{ key }} = source.{{ key }}
        {% if not loop.last %} AND {% endif %}
    {% endfor %}
    WHEN MATCHED THEN
        UPDATE SET
        {% for column in adapter.get_columns_in_relation(target_relation) %}
            {% if column.name not in unique_key %}
                target.{{ column.name }} = source.{{ column.name }}
                {% if not loop.last %},{% endif %}
            {% endif %}
        {% endfor %}
    WHEN NOT MATCHED THEN
        INSERT *
{% endmacro %}

{% macro glue__create_python_intermediate_table(model, df_name) %}
    {%- set relation = this.incorporate(type='table') -%}
    {%- set dest_columns = adapter.get_columns_in_relation(this) -%}

    {%- set write_mode = 'overwrite' if should_full_refresh() else 'append' -%}
    {%- set location = adapter.get_location(this) -%}

    {{ df_name }}.createOrReplaceTempView('{{ relation.identifier }}_intermediate')

    CREATE TABLE {{ relation }} AS
    SELECT * FROM {{ relation.identifier }}_intermediate
{% endmacro %}
