import sys
import os

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

import tdsbrondata
from pyspark.sql import functions as F
from pyspark.sql.column import Column
from pyspark.sql.types import *

def getSecret(secretName):
    return tdsbrondata._notebookutils.credentials.getSecret(tdsbrondata.keyvaultUrl, secretName)

def getCurrentData(workspaceName, lakehouseName, schemaName, tableName, usesScd=False, scdMoment=None):
    
    abfss = f"abfss://{workspaceName}@onelake.dfs.fabric.microsoft.com/{lakehouseName}.Lakehouse/Tables/{schemaName}/{tableName}"

    data = tdsbrondata._spark.read.format("delta").load(abfss)
    
    if usesScd:
        if scdMoment is not None:
            data = data.filter(
                (F.col("ScdStartDate") <= scdMoment) &
                (
                    F.col("ScdEndDate").isNull() |
                    (F.col("ScdEndDate") > scdMoment)
                )
            )
        else:
            data = data.filter(
                (F.col("CurrentFlag") == 1) &
                (F.col("ScdEndDate").isNull())
            )

    return data

def getCurrentDataSQL(schemaName, tableName, usesScd=False, scdMoment=None):

    table = f"{schemaName}.{tableName}"
    whereClause = ""

    if isinstance(scdMoment, Column):
        ts_value = (
            tdsbrondata._spark
                .range(1)
                .select(scdMoment.alias("ts"))
                .collect()[0]["ts"]
        )

        scdMomentLiteral = f"'{ts_value}'"
    else:
        scdMomentLiteral = None

    if usesScd:
        if scdMomentLiteral is not None:
            whereClause = f"""
            WHERE ScdStartDate <= {scdMomentLiteral}
              AND (ScdEndDate IS NULL OR ScdEndDate > {scdMomentLiteral})
            """
        else:
            whereClause = """
            WHERE CurrentFlag = 1
              AND ScdEndDate IS NULL
            """

    query = f"(SELECT * FROM {table} {whereClause}) AS t"

    return (
        tdsbrondata._spark.read
            .format("jdbc")
            .option("url", tdsbrondata.jdbcUrl)
            .option("dbtable", query)
            .option("accessToken", tdsbrondata.token)
            .load()
    )

def getAvailableSourceFiles(schemaName, tableName, sourceDataMode, start, directory="", lakehouseName="B_Diensten"):

    availableSourceFiles = []

    if sourceDataMode == 'automatic' and directory == "":
        sourceDataPathRoot = f"abfss://Tosch-Data@onelake.dfs.fabric.microsoft.com/{lakehouseName}.Lakehouse/Files/AutomaticData/{schemaName}/"
        for item in tdsbrondata._notebookutils.fs.ls(sourceDataPathRoot):
            if item.isDir:
                folder = item.name.rstrip("/")
                if folder.isdigit() and int(folder) >= int(start):
                    folderPath = f"{sourceDataPathRoot}{folder}/"
                    try:
                        files = tdsbrondata._notebookutils.fs.ls(folderPath)
                        for f in files:
                            fileName = f.name.rstrip("/")
                            if fileName == f"{tableName}_full.parquet":
                                filePath = f"{folderPath}{fileName}"
                                ScdMoment = folder[:4] + "-" + folder[4:6] + "-" + folder[6:8] + " 00:00:00"
                                availableSourceFiles.append({"filePath": filePath, "ScdMoment": ScdMoment})
                    except Exception as e:
                        print(f"Cannot list files in folder: {folderPath}, {e}")
    elif sourceDataMode == 'automatic':
        sourceDataPathRoot = f"abfss://Tosch-Data@onelake.dfs.fabric.microsoft.com/{lakehouseName}.Lakehouse/Files/AutomaticData/{schemaName}/{directory}/"
        files = tdsbrondata._notebookutils.fs.ls(sourceDataPathRoot)
        for f in files:
            fileName = f.name.rstrip("/")
            prefix = fileName.split("_")[0]
            if prefix.isdigit() and int(prefix) >= int(start) and fileName.endswith(".csv"):
                filePath = f"{sourceDataPathRoot}{fileName}"
                ScdMoment = prefix[:4] + "-" + prefix[4:6] + "-01 00:00:00"
                availableSourceFiles.append({"filePath": filePath, "ScdMoment": ScdMoment})
    else:
        sourceDataPathRoot = f"abfss://Tosch-Data@onelake.dfs.fabric.microsoft.com/{lakehouseName}.Lakehouse/Files/ManualData/{schemaName}/"
        files = tdsbrondata._notebookutils.fs.ls(sourceDataPathRoot)
        for f in files:
            fileName = f.name.rstrip("/")
            prefix = fileName.split("_")[0]
            if prefix.isdigit() and int(prefix) >= int(start) and tableName in fileName and fileName.endswith(".csv"):
                filePath = f"{sourceDataPathRoot}{fileName}"
                ScdMoment = prefix[:4] + "-" + prefix[4:6] + "-01 00:00:00"
                availableSourceFiles.append({"filePath": filePath, "ScdMoment": ScdMoment})

    return availableSourceFiles

def addEventLogEntry(lakehouseName, schemaName, tableName, eventType, eventResult):

    abfss = 'abfss://Tosch-Data@onelake.dfs.fabric.microsoft.com/M_Metadata.Lakehouse/Tables/dbo'
    
    schema = StructType([
        StructField("LakehouseName", StringType(), True),
        StructField("SchemaName", StringType(), True),
        StructField("TableName", StringType(), True),
        StructField("EventType", StringType(), True),
        StructField("EventResult", StringType(), True)
    ])

    row = {"LakehouseName": lakehouseName, "SchemaName": schemaName, "TableName": tableName, "EventType": eventType, "EventResult": eventResult}
    df = tdsbrondata._spark.createDataFrame([row], schema=schema)

    try:
        dfExisting = tdsbrondata._spark.read.format("delta").load(f"{abfss}/eventlog")
    except Exception as e:
        dfExisting = tdsbrondata._spark.createDataFrame([], schema=schema)

    isDuplicate = (
        dfExisting.filter(
            (F.col("LakehouseName") == row["LakehouseName"]) &
            (F.col("SchemaName") == row["SchemaName"]) &
            (F.col("TableName") == row["TableName"]) &
            (F.col("EventType") == row["EventType"]) &
            (F.col("EventResult") == row["EventResult"])
        )
        .limit(1)
        .count() > 0
    )

    if not isDuplicate:
        df.write.mode("append").format("delta").save(f"{abfss}/eventlog")

def fileToBeProcessed(lakehouseName, schemaName, tableName, filePattern):

    shouldBeProcessed = True

    eventLog = tdsbrondata._spark.read.format("delta").load("abfss://Tosch-Data@onelake.dfs.fabric.microsoft.com/M_Metadata.Lakehouse/Tables/dbo/eventlog")

    fileAdded = eventLog.filter(
        (F.col("LakehouseName") == lakehouseName) & 
        (F.col("SchemaName") == schemaName) & 
        (F.col("TableName") == tableName) & 
        (F.col("EventType") == "fileAdded") & 
        (F.col("EventResult").startswith(filePattern))
    ).limit(1)

    fileAdded = fileAdded.collect()
    if fileAdded:
        fileName = fileAdded[0]["EventResult"]
        fileProcessed = eventLog.filter(
            (F.col("LakehouseName") == lakehouseName) & 
            (F.col("SchemaName") == schemaName) & 
            (F.col("TableName") == tableName) & 
            (F.col("EventType") == "fileProcessed") & 
            (F.col("EventResult").startswith(filePattern))
        ).limit(1)

        fileProcessed = fileProcessed.collect()
        if fileProcessed:
            shouldBeProcessed = False
    else:
        shouldBeProcessed = False

    if shouldBeProcessed:
        return fileName
    else:
        return None