## DEPENDENCIES         
#=============================================================#

import bribrain


## PANDAS CONFIG
#=============================================================#

import pandas as pd
pd.options.display.html.table_schema=True
pd.options.display.max_columns=999
pd.options.display.max_rows=999

## SPARK SESSION
#=============================================================#

spark = bribrain.sparkSession(
  appname="ETL BRImo List Feature", 
  executor="high")


## MAIN PROCESS
#=============================================================#

#=>PARAMS

bribrain.set_timer()

hive_schema = "temp"
hive_table = "tbl_trhistmaster"

ds = (datetime.now(timezone("Asia/Jakarta")) - relativedelta(months=1)).strftime("%Y%m")

print("Params `ds` =", ds)

#=>END PARAMS




def load_to_hive(spark, df, schema, table, partition=None, mode="append", repartition_number=None, blocksize=256):
  """Melakukan penulisan pyspark dataframe sebagai tabel hive di hdfs
  
  Args:
      spark (pyspark.sql.session.SparkSession): spark session
      df (pyspark.sql.dataframe.DataFrame): pyspark dataframe yang akan di tulis sebagai tabel hive
      schema (str): target schema di hdfs
      table (str): target table di hdfs
      partition (str): kolom yang akan dijadikan partisi (dipisahkan ',' jika lebih dari satu kolom)
          (default is None)
      mode (str): cara yang dilakukan untuk penulisan tabel
          (optional is append, overwrite, error, ignore)
          (default is append)
      repartition_number (int): jumlah file parquet yang diinginkan dalam satu partisi
          (default is None)      
      blocksize (int): ukuran per file dalam satuan megabyte
          (default is 256)
    
  Returns:
      -
  """
  
  if schema.startswith("dev"):
    hdfs_path = "/dev/"
  else:
    hdfs_path = "/user/hive/warehouse/"
  
  print("Write table to Hive: {0}.{1}".format(schema, table))
    
  # pengecekan partitioned table
  if partition:
    
    # kondisi ketika tabel berpartisi
    partition = partition.split(",")
    list_partition = df.select(partition).distinct().collect()
    
    unit = len(str(len(list_partition)))
    
    first = True
    row_total = 0
    
    # loop untuk penulisan data per partisi sesuai standard blocksize
    for i, row in enumerate(list_partition):
      
      condition = ' AND '.join(["{0}='{1}'".format(col, row[col]) for col in partition])
      df_write = df.filter(condition)


      if not repartition_number:
        tag = '_'.join([row[col].strip() for col in partition]).lower()

        spark.sql("DROP TABLE IF EXISTS project_sv.{0}_{1}_allow_deleted".format(table, tag))

        # penulisan sample data untuk estimasi ukuran data
        df_write\
          .sample(False,0.1,None)\
          .write\
          .format("parquet")\
          .mode("overwrite")\
          .saveAsTable("project_sv.{0}_{1}_allow_deleted".format(table, tag))

        # mendapatkan metadata content summary dari sample table
        path = lambda p: spark._jvm.org.apache.hadoop.fs.Path(p)
        fs = spark._jvm.org.apache.hadoop.fs.FileSystem.get(spark._jsc.hadoopConfiguration())
        hdfs_folder = fs.getContentSummary(path(hdfs_path+"project_sv.db/{0}_{1}_allow_deleted/".format(table, tag)))

        # kalkulasi jumlah file yang optimal sesuai standard blocksize
        blocksize_number = float(hdfs_folder.getLength()*10)/float(blocksize*1024*1024)
        repartition_number = int(1 if blocksize_number < 1 else blocksize_number)

        spark.sql("DROP TABLE project_sv.{0}_{1}_allow_deleted".format(table, tag))

      # set jumlah file sesuai hasil kalkulasi
      df_write = df_write.repartition(repartition_number)
    
      # penghapusan data partisi jika sudah ada di target tabel
      if spark.sql("SHOW TABLES IN {0} LIKE '{1}'".format(schema, table)).count() != 0:
        query_drop_partition = "ALTER TABLE {0}.{1} DROP IF EXISTS PARTITION({2})".format(schema, table, condition.replace(" AND ",","))
        spark.sql(query_drop_partition) # menghapus partisi di spark

      # penulisan pyspark dataframe ke hdfs
      df_write\
        .write\
        .format("parquet")\
        .partitionBy(partition)\
        .mode(mode)\
        .saveAsTable("{0}.{1}".format(schema,table))

      spark.sql("MSCK REPAIR TABLE {0}.{1}".format(schema, table)) # memperbaiki tabel
      spark.sql("ANALYZE TABLE {0}.{1} PARTITION ({2}) COMPUTE STATISTICS".format(schema, table, condition.replace(" AND ",","))) # kalkulasi data
      spark.sql("REFRESH TABLE {0}.{1}".format(schema, table)) # refresh tabel di spark
      

      first = False
      if not first: mode="append"
        
      row_count = df_write.count()
      row_total += row_count
      print("{0}) Count Record ({1}) = {2}".format(str(i+1).ljust(unit, " "), condition.replace(" AND ",","), row_count))

  else:
    
    if not repartition_number:
      # kondisi ketika tabel berpartisi    
      spark.sql("DROP TABLE IF EXISTS project_sv.{0}_allow_deleted".format(table))

      # penulisan sample data untuk estimasi ukuran data
      df\
        .sample(False,0.1,None)\
        .write\
        .format("parquet")\
        .mode("overwrite")\
        .saveAsTable("project_sv.{0}_allow_deleted".format(table))

      # mendapatkan metadata content summary dari sample table
      path = lambda p: spark._jvm.org.apache.hadoop.fs.Path(p)
      fs = spark._jvm.org.apache.hadoop.fs.FileSystem.get(spark._jsc.hadoopConfiguration())
      hdfs_folder = fs.getContentSummary(path(hdfs_path+"project_sv.db/{0}_allow_deleted/".format(table)))

      # kalkulasi jumlah file yang optimal sesuai standard blocksize
      blocksize_number = float(hdfs_folder.getLength()*10)/float(blocksize*1024*1024)
      repartition_number = int(1 if blocksize_number < 1 else blocksize_number)

      spark.sql("DROP TABLE project_sv.{0}_allow_deleted".format(table))

    # set jumlah file sesuai hasil kalkulasi
    df = df.repartition(repartition_number)

    # penulisan pyspark dataframe ke hdfs
    df\
      .write\
      .format("parquet")\
      .mode(mode)\
      .saveAsTable("{0}.{1}".format(schema,table))

    spark.sql("ANALYZE TABLE {0}.{1} COMPUTE STATISTICS".format(schema, table)) # kalkulasi data
    spark.sql("REFRESH TABLE {0}.{1}".format(schema, table)) # refresh tabel di spark
    

  print("\n====>> Grand Total Record =", df.count())
