        
class Pipelines:

    __iceberg_table_properties = [
        {"read.split.target-size": "134217728"},  # 128 MB
        {"read.split.metadata-target-size": "33554432"},  # 32 MB
        {"read.split.planning-lookback": "10"},
        {"read.split.open-file-cost": "4194304"},  # 4 MB
        {"read.parquet.vectorization.enabled": "true"},
        {"read.parquet.vectorization.batch-size": "5000"},
        {"write.format.default": "parquet"},
        {"write.parquet.row-group-size-bytes": "134217728"},  # 128 MB
        {"write.parquet.page-size-bytes": "1048576"},  # 1 MB
        {"write.parquet.page-row-limit": "20000"},
        {"write.parquet.dict-size-bytes": "2097152"},  # 2 MB
        {"write.parquet.compression-codec": "zstd"},
        {"write.parquet.bloom-filter-max-bytes": "1048576"},  # 1 MB
        {"write.parquet.bloom-filter-fpp.column.col1": "0.01"},
        {"write.avro.compression-codec": "gzip"},
        {"write.orc.stripe-size-bytes": "67108864"},  # 64 MB
        {"write.orc.block-size-bytes": "268435456"},  # 256 MB
        {"write.orc.compression-codec": "zlib"},
        {"write.orc.compression-strategy": "speed"},
        {"write.orc.bloom.filter.fpp": "0.05"},
        {"write.metadata.metrics.max-inferred-column-defaults": "100"},
        {"write.metadata.metrics.default": "truncate(16)"},
        {"write.target-file-size-bytes": "536870912"},  # 512 MB
        {"write.delete.target-file-size-bytes": "67108864"},  # 64 MB
        {"write.wap.enabled": "false"},
        {"write.summary.partition-limit": "0"},
        {"write.metadata.delete-after-commit.enabled": "false"},
        {"write.metadata.previous-versions-max": "100"},
        {"write.spark.fanout.enabled": "false"},
        {"write.object-storage.enabled": "false"},
        {"write.object-storage.partitioned-paths": "true"},
        {"write.data.path": "table location + /data"},
        {"write.metadata.path": "table location + /metadata"},
        {"write.delete.mode": "copy-on-write"},
        {"write.delete.isolation-level": "serializable"},
        {"write.update.mode": "copy-on-write"},
        {"write.update.isolation-level": "serializable"},
        {"write.merge.mode": "copy-on-write"},
        {"write.merge.isolation-level": "serializable"},
        {"write.delete.granularity": "partition"}]

    __delta_table_properties = [
        {"delta.enableChangeDataFeed": "true"},
        {"delta.enableDeletionVectors": "true"},
        {"delta.enableRowTracking": "true"},
        {"delta.feature.appendOnly": "supported"},
        {"delta.feature.changeDataFeed": "supported"},
        {"delta.feature.deletionVectors": "supported"}] 

    from pyspark.sql import SparkSession, DataFrame
    from typing import List, Dict, Optional, Callable
    
    def __init__(
        self, 
        spark: SparkSession,
        current_catalog: Optional[str]= None,
        current_schema: Optional[str]= None,
        checkpoint_location: Optional[str]= None
            ):
        self.spark = spark
        self.current_catalog = current_catalog
        self.current_schema = current_schema
        self.checkpoint_location = checkpoint_location

    def __to_table(
        self,
        df: DataFrame,
        name: Optional[str] = None,
        comment: Optional[str]= None,
        table_properties: Optional[Dict[str, str]]= None,
        class_table_properties: Optional[Dict[str, str]]= None,
        path: Optional[str] = None,
        partition_cols: Optional[List[str]]= None,
        schema: Optional[str]= None,
        func_name: Optional[str]= None,
        table_format: str = "delta"
            ):
        if self.current_catalog:
            if self.current_schema:
                if name:
                    writer = df.writeTo(f"{self.current_catalog}.{self.current_schema}.{name}").using(table_format)
                else:
                    writer = df.writeTo(f"{self.current_catalog}.{self.current_schema}.{func_name}").using(table_format)
            elif name:
                writer = df.writeTo(f"{self.current_catalog}.{name}").using(table_format)
            else:
                writer = df.writeTo(f"{self.current_catalog}.{func_name}").using(table_format)
        elif name:
            writer = df.writeTo(name).using(table_format)
        else:
            writer = df.writeTo(func_name).using(table_format)

        if table_properties:
            for k, v in table_properties.items():
                writer = writer.tableProperty(k, v)

        if class_table_properties:
            for item in class_table_properties:
                for k, v in item.items():
                    writer = writer.tableProperty(k, v)

        if partition_cols:
            writer = writer.partitionedBy(*partition_cols)
        
        if path:
            writer = writer.option("path", path)
        
        if comment:
            writer = writer.tableProperty("comment", comment)
        
        if schema:
            writer = writer.tableProperty("custom.schema", schema)
        
        if table_format != "iceberg" and table_format != "delta":
            writer.create()
        else:
            writer.createOrReplace()

    def create_streaming_table(
        self,
        name: Optional[str] = None,
        comment: Optional[str] = None,
        table_properties: Optional[Dict[str, str]] = None,
        path: Optional[str] = None,
        partition_cols: Optional[List[str]] = None,
        schema: Optional[str] = None,
        table_format: str = "delta"
    ):
        # Build full table name
        if self.current_catalog and self.current_schema:
            table_name = f"{self.current_catalog}.{self.current_schema}.{name}"
        elif self.current_catalog:
            table_name = f"{self.current_catalog}.{name}"
        else:
            table_name = name  

        # Check if table already exists
        if self.spark.catalog.tableExists(table_name):
            print(f"Table already exists: {table_name}")
            return
        
        schema_def=None
        if schema:
            schema_def = f" ({schema})"
        # Start building SQL
        ddl = f"CREATE TABLE {table_name}{schema_def} USING {table_format}"

        if path:
            ddl += f" LOCATION '{path}'"

        if partition_cols:
            ddl += f" PARTITIONED BY ({', '.join(partition_cols)})"

        if comment:
            ddl += f" COMMENT '{comment}'"

        class_table_properties = None
        if table_format == "delta":
            class_table_properties = self.__delta_table_properties
        elif table_format == "iceberg":
            class_table_properties = self.__iceberg_table_properties

        # Add table properties
        if table_properties or class_table_properties:
            props = []
            if table_properties:
                props.extend([f"'{k}' = '{v}'" for k, v in table_properties.items()])
            if class_table_properties:
                for item in class_table_properties:
                    for k, v in item.items():
                        props.append(f"'{k}' = '{v}'")
            if props:
                ddl += f" TBLPROPERTIES ({', '.join(props)})"

        # Run the SQL
        print(f"Executing DDL:\n{ddl}")
        self.spark.sql(ddl)


    def __to_streaming_table(
        self,
        df: DataFrame,
        name: Optional[str] = None,
        func_name: Optional[str]= None,
        streaming_output_mode: Optional[str]= 'append',
        availableNow: Optional[bool]= True,
        processingTime: Optional[str]= None,
        table_format: str = "delta"
            ):
        if self.current_catalog and self.current_schema:
            table_name = f"{self.current_catalog}.{self.current_schema}.{name or func_name}"
        elif self.current_catalog:
            table_name = f"{self.current_catalog}.{name or func_name}"
        else:
            table_name = name or func_name
        
        if availableNow:
            df.writeStream.format(table_format).outputMode(streaming_output_mode).option("checkpointLocation", self.checkpoint_location+"_"+table_name).trigger(availableNow=True).toTable(table_name)
        else:
            df.writeStream.format(table_format).outputMode(streaming_output_mode).option("checkpointLocation", self.checkpoint_location+"_"+table_name).trigger(processingTime=processingTime).toTable(table_name)

    def table(
        self,
        name: Optional[str] = None,
        comment: Optional[str]= None,
        table_properties: Optional[Dict[str, str]]= None,
        path: Optional[str] = None,
        partition_cols: Optional[List[str]]= None,
        schema: Optional[str]= None,
        table_format: str = "delta",
        streaming_output_mode: Optional[str]= 'append',
        availableNow: Optional[bool]= True,
        processingTime: Optional[str]= None,
            ):
        
        from pyspark.sql import DataFrame
        from typing import Callable
        
        def decorator(func: Callable[..., DataFrame]) -> Callable[..., DataFrame]:
            def wrapper(*args, **kwargs) -> DataFrame:
                df: DataFrame = func(*args, **kwargs)
                if df.isStreaming:
                    self.__to_streaming_table(
                        df= df,
                        name= name,
                        func_name= func.__name__,
                        streaming_output_mode= streaming_output_mode,
                        availableNow= availableNow,
                        processingTime= processingTime,
                        table_format= table_format)
                    
                else:  
                    if table_format == "delta":
                        self.__to_table(
                            df= df,
                            name=name,
                            comment=comment,
                            table_properties=table_properties,
                            class_table_properties=Pipelines.__delta_table_properties,
                            path=path,
                            partition_cols=partition_cols,
                            schema=schema,
                            func_name=func.__name__,
                            table_format=table_format
                            )
                        
                    if table_format == "iceberg":
                        self.__to_table(
                            df= df,
                            name=name,
                            comment=comment,
                            table_properties=table_properties,
                            class_table_properties=Pipelines.__iceberg_table_properties,
                            path=path,
                            partition_cols=partition_cols,
                            schema=schema,
                            func_name=func.__name__,
                            table_format=table_format
                            )

                    if table_format != "iceberg" and table_format != "delta":
                        self.__to_table(
                            df= df,
                            name=name,
                            comment=comment,
                            table_properties=table_properties,
                            class_table_properties=Pipelines.__iceberg_table_properties,
                            path=path,
                            partition_cols=partition_cols,
                            schema=schema,
                            func_name=func.__name__,
                            table_format=table_format
                            )
                return df
            return wrapper
        return decorator
