import bribrain1
#import importlib
#importlib.reload(bribrain1)

spark = bribrain1.sparkSession()

df = spark.read.table("dataanl.bribrain_mikro_phd_fact_all_score").limit(1000)
df.cache()


bribrain1.set_timer()
partitions_brimo = bribrain1.get_list_partition(spark, "datamart", "bribrain_brimo_raw_list_fiturid")[:5]
bribrain1.get_timer()


bribrain1.set_timer()
partitions_lnhist = bribrain1.get_list_partition(spark, "datalake", "8000_rc_trx_as4_lnhist")[:5]
bribrain1.get_timer()



bribrain1.set_timer()
partitions_ddhist = bribrain1.get_list_partition(spark, "datalake", "8000_rc_trx_as4_ddhist")[:5]
bribrain1.get_timer()


bribrain1.set_timer()
partitions = spark.catalog.listPartitions("datamart.bribrain_brimo_raw_list_fiturid")
bribrain1.get_timer()


schema = "datalake"
table = "8000_rc_trx_as4_ddhist"

def get_partition_column_name(partition_list):
    if not partition_list:
        return None
        
    # Get all keys from the first record
    sample_entry = partition_list[0]
    
    # Filter out keys that are internal metadata
    # (usually starting with underscores or known static names)
    potential_cols = [k for k in sample_entry.keys() if not k.startswith("__")]
    
    return potential_cols if potential_cols else None
  

get_partition_column_name(partitions)

import re

def check_partition_exists(spark, target_value, schema, table, mode=None):
    """Mengecek keberadaan partisi pada tabel HDFS dan mengembalikan metadata partisi terkait

    Args:
        spark (pyspark.sql.session.SparkSession): objek spark session yang digunakan
        target_value (str): nilai partisi yang ingin dicek (contoh: '2023|03|07')
        schema (str): nama database/schema di Hive/HDFS
        table (str): nama table yang akan dicek partisinya
        mode (str): mode pencarian jika target_value tidak ditemukan 
            (optional is None, 'forward', 'backward')
            (default is None)

    Returns:
        dict: dictionary berisi detail kolom partisi (year, month, day, dsb.)
            tanpa kolom __formatted_partition, atau None jika tidak ditemukan
    """
    partition_list = get_list_partition(spark, schema, table)
    
    if not partition_list:
        return None

    # 1. Get a sample format from the metadata
    sample_format = partition_list[0].get('__formatted_partition', '')
    
    # 2. STRICT VALIDATION: Compare structure
    # We build the regex pattern outside the f-string to avoid SyntaxError
    digit_pattern = re.sub(r'\d', r'\\d', sample_format)
    escaped_pattern = digit_pattern.replace('|', r'\|')
    regex_final = "^" + escaped_pattern + "$"
    
    if not re.match(regex_final, target_value):
        raise ValueError(
            "Format Mismatch!\n"
            f"Input target_value: '{target_value}'\n"
            f"HDFS Expected Format: '{sample_format}'\n"
            "Please ensure separators (|, -, /) and padding (01 vs 1) match exactly."
        )

    # 3. Build lookup
    lookup = {}
    for p in partition_list:
        fmt_key = p.get('__formatted_partition')
        if fmt_key:
            # Clean dict for output by removing the helper key
            clean_dict = {k: v for k, v in p.items() if k != '__formatted_partition'}
            lookup[fmt_key] = clean_dict

    # 4. Logic Execution
    sorted_keys = sorted(lookup.keys())

    if target_value in lookup:
        return lookup[target_value]

    if mode == "forward":
        for k in sorted_keys:
            if k > target_value:
                return lookup[k]
                
    elif mode == "backward":
        for k in reversed(sorted_keys):
            if k < target_value:
                return lookup[k]

    return None
  
  
  
  
  
bribrain1.load_to_hive(
    spark,
    df,
    schema="sharingvision",
    table="hapus_bribrain_mikro_phd_fact_all_score",
    partition = "ds",
    mode = 'overwrite',
    repartition_number = None,
    blocksize = 64,
    modified_date = False,
    schema_temp = 'sharingvision',
    count = False,
)


bribrain1.load_to_file(
    df.drop("modified_date"),
    file_name = "test_excel",
    file_type = 'excel',
    repartition_number = 1,
    delimiter = '|',
    download = True,
    delete_hdfs_file = True,
)

bribrain1.load_to_file(
    df.drop("modified_date"),
    file_name = "test",
    file_type = 'json',
    repartition_number = 1,
    delimiter = '|',
    download = True,
    delete_hdfs_file = True,
)


import subprocess
import os

hadoop_user = os.getenv("HADOOP_USER_NAME")

hdfs_dir = f"/user/{hadoop_user}/{file_name}"
subprocess.run(
    ["hdfs", "dfs", "-mkdir", "-p", hdfs_dir],
    check=True
)

