#=============================================================
###  CREATED AT     : 12 MARET 2024                         
###  UPDATED AT     : 12 FEBRUARI 2024                          
###  COPYRIGHT      : BRIBRAIN DATA ENGINEER TEAM           
###  DESCRIPTION    : Module untuk kumpulan function        
#=============================================================
import os
import re
import time
from pytz import timezone
from functools import reduce
from datetime import datetime, timedelta
from pyspark.sql import functions as F
from pyspark.sql import types as T

import pandas as pd
from typing import Optional, List

from pandas import DatetimeTZDtype
from pandas.api.types import (
    is_datetime64_any_dtype,
    is_datetime64tz_dtype,
    is_integer_dtype,
    is_float_dtype,
    is_bool_dtype,
    is_string_dtype,
    is_categorical_dtype,
)


#=============================================================#

def try_or(func, default=None, expected_exc=(Exception,)):
  """Menangkap error dan memberikan keluaran sesuai parameter

  Args:
      func (function): python function
        (notes is ditambahkan lambda pada sebelum nama function, ex: try_or(lambda:func))
      default (object): keluaran yang diharapkan ketika terjadi error
      expected_exc (Exception): Exception yang diharapkan
    
  Returns:
      object: keluaran dari function atau mengembalikan keluaran sesuai input jika terdapat error
  """
  
  try:
    return func()
  except expected_exc:
    return default
  
#=============================================================# 

START_TIME = None

def set_timer():
  """Menetapkan waktu awal untuk penghitungan durasi proses

  Args:
      -
    
  Returns:
      float: waktu dalam format float
  """
  global START_TIME
  START_TIME = time.time()
  # We return it just in case, but usually, we just want to 'set' it.
  return START_TIME

#=============================================================#

def get_timer(start_time=None):
  """Memperoleh durasi proses berdasarkan waktu awal dikurangi waktu sekarang

  Args:
      start_time (float) : timestamp yang menunjukan time
    
  Returns:
      string: durasi proses dengan format HH:MM:SS
  """
    
  if START_TIME is None:
        return "Timer was not started. Call set_timer() first."
    
  if start_time:
    return (datetime(1,1,1)+timedelta(seconds=int(time.time()-start_time))).strftime("%H:%M:%S")
  
  return (datetime(1,1,1)+timedelta(seconds=int(time.time()-START_TIME))).strftime("%H:%M:%S")

#=============================================================#

def show_time(format_time="%Y-%m-%d %H:%M:%S"):
  """Menampilkan waktu saat ini

  Args:
      format_time (str) : format waktu yang ingin ditampilkan 
          (default is %Y-%m-%d %H:%M:%S)

  Returns:
      string: waktu saat ini sesuai format input
  """
      
  # Get the current time as a UNIX timestamp
  timestamp = time.time()

  # Convert the timestamp to a timezone-aware datetime object
  local_time = datetime.fromtimestamp(timestamp, timezone('Asia/Jakarta'))

  # Format the time as Year-Month-Day Hour:Minute:Second with timezone
  formatted_time = local_time.strftime(format_time)

  return formatted_time

#=============================================================#

def remove_blank_space(x):
  """Menghapus spasi kosong pada kolom

  Args:
      x (str): nama kolom
    
  Returns:
      pyspark.sql.functions.Column: kolom dengan spasi kosong dihapus
  """


  return F.when(F.trim(F.col(x)) != '', F.trim(F.col(x))).otherwise(F.lit(None))

def cleanse_blank_space(df_to_clean):
  """Membersihkan spasi kosong pada kolom string

  Args:
      df_to_clean (pyspark.sql.DataFrame): DataFrame yang akan dibersihkan
    
  Returns:
      pyspark.sql.DataFrame: DataFrame dengan spasi kosong dihapus
  """
  col_to_clean = []
  for val in df_to_clean.dtypes:
    if val[1] == "string":
      col_to_clean.append(val[0])
  return reduce(lambda df, x: df.withColumn(x, remove_blank_space(x)), set(col_to_clean), df_to_clean)

#=============================================================#  

def lower_columns_name(df):
  """Mengubah nama kolom menjadi huruf kecil

  Args:
      df (pyspark.sql.DataFrame): DataFrame yang akan diubah
    
  Returns:
      pyspark.sql.DataFrame: DataFrame dengan nama kolom huruf kecil
  """
  list_column = []
  for col in df.columns:
    list_column.append(col.lower())
  return df.select(list_column)

#=============================================================#  

def spark_type_from_pandas_dtype(dtype, series: Optional[pd.Series] = None) -> T.DataType:
  """Menentukan tipe data Spark berdasarkan dtype dari pandas

  Args:
      dtype : tipe data kolom pandas (mis. numpy.dtype atau pandas ExtensionDtype)
      series (pandas.Series, optional): data kolom asli, dapat digunakan untuk
          logika tambahan (mis. cek min/max), saat ini opsional dan belum
          dimanfaatkan secara spesifik

  Returns:
      pyspark.sql.types.DataType: tipe data Spark yang paling sesuai dengan dtype pandas
  """

  # datetime (naive atau timezone-aware)
  if is_datetime64_any_dtype(dtype) or is_datetime64tz_dtype(dtype):
      return T.TimestampType()

  # boolean (termasuk pandas BooleanDtype)
  if is_bool_dtype(dtype):
      return T.BooleanType()

  # integer (termasuk nullable Int64/Int32)
  if is_integer_dtype(dtype):
      # LongType paling aman untuk range nilai besar
      return T.LongType()

  # float (termasuk nullable Float64)
  if is_float_dtype(dtype):
      return T.DoubleType()

  # string / kategori → StringType
  if is_string_dtype(dtype) or is_categorical_dtype(dtype):
      return T.StringType()

  # fallback untuk object / mixed type
  return T.StringType()

def pandas_to_spark(pdf: pd.DataFrame, tz_to_utc: bool = True):
  """Mengonversi pandas DataFrame menjadi Spark DataFrame dengan schema otomatis

  Args:
      pdf (pandas.DataFrame): DataFrame pandas yang akan dikonversi ke Spark
      tz_to_utc (bool): jika True, seluruh kolom datetime yang memiliki timezone
          akan dikonversi ke zona waktu UTC dan dibuat naive (tanpa informasi TZ)
          untuk menjaga konsistensi di Spark
          (default is True)

  Returns:
      pyspark.sql.DataFrame: DataFrame Spark dengan schema yang diturunkan dari
          dtype masing-masing kolom di pandas
  """

  # Bekerja pada copy agar tidak mengubah pdf asli di pemanggil
  pdf = pdf.copy()

  # Normalisasi datetime yang timezone-aware → naive UTC
  if tz_to_utc:
      for col in pdf.columns:
          dtype = pdf[col].dtype
          if isinstance(dtype, DatetimeTZDtype) or is_datetime64tz_dtype(dtype):
              pdf[col] = (
                  pdf[col]
                  .dt.tz_convert("UTC")   # konversi ke UTC
                  .dt.tz_localize(None)   # hilangkan informasi timezone
              )

  # Bangun schema Spark berdasarkan dtype tiap kolom
  fields: List[T.StructField] = []
  for col in pdf.columns:
      dtype = pdf[col].dtype
      spark_type = spark_type_from_pandas_dtype(dtype, pdf[col])
      fields.append(T.StructField(col, spark_type, nullable=True))

  schema = T.StructType(fields)

  # Buat Spark DataFrame dengan schema eksplisit
  return spark.createDataFrame(pdf, schema=schema)

#=============================================================#

def get_formatted_bytes(size):
    """Mengonversi ukuran file dari bytes ke format yang mudah dibaca manusia

    Args:
        size (int): ukuran file dalam bytes

    Returns:
        str: ukuran file dalam format yang mudah dibaca manusia (PB, TB, GB, MB, KB, B)
    """
    size = float(size)
    
    # Define the powers
    power_pb = 2**50  # Petabyte
    power_tb = 2**40  # Terabyte
    power_gb = 2**30  # Gigabyte
    power_mb = 2**20  # Megabyte
    power_kb = 2**10  # Kilobyte

    if size >= power_pb:
        return f"{size / power_pb:.2f} PB"
    elif size >= power_tb:
        return f"{size / power_tb:.2f} TB"
    elif size >= power_gb:
        return f"{size / power_gb:.2f} GB"
    elif size >= power_mb:
        return f"{size / power_mb:.2f} MB"
    elif size >= power_kb:
        return f"{size / power_kb:.2f} KB"
    else:
        return f"{size:.0f} B" # Just Bytes

#=============================================================#

def get_list_table(spark, schema, limit = None, sort = True):
  """Memperoleh list table dari schema di hive

  Args:
      spark (pyspark.sql.session.SparkSession): spark session
      schema (str): nama schema dari table di hive
      limit (int): jumlah maksimum table yang ingin diambil
          (default is None)
      sort (bool | None): mengurutkan list table
          - True  : mengurutkan berdasarkan abjad (A-Z)
          - False : mengurutkan berdasarkan abjad terbalik (Z-A)
          - None  : tanpa pengurutan
          (default is True)
    
  Returns:
      list | None: list table, atau None jika terjadi error
  """
  
  try:
    tables = spark.sql("""
     SHOW TABLES IN {}
    """.format(schema))
    
    tables = tables.collect()
    tables = [row.tableName for row in tables]
    
    if sort is True:
      tables.sort()
    elif sort is False:
      tables.sort(reverse=True)
    
    if limit:
      tables = tables[:limit]
    
    return tables
  except Exception as e:
    print(e)
    return None 

#=============================================================#  

def get_list_tables_with_metadata(spark, schema, owner=None, sort="oldest"):
    """Memperoleh list table beserta metadata dari schema di HDFS

    Args:
        spark (pyspark.sql.session.SparkSession): spark session
        schema (str): nama schema/database di Hive
        owner (str | None): filter berdasarkan owner table
            - Jika diisi, hanya table milik owner tersebut yang dikembalikan
            - Jika None, semua table dikembalikan
            (default is None)
        sort (str | None): mengurutkan list table
            - "oldest"  : berdasarkan tanggal terlama
            - "newest"  : berdasarkan tanggal terbaru
            - "biggest" : berdasarkan ukuran terbesar
            - "smallest": berdasarkan ukuran terkecil
            - None      : tanpa pengurutan
            (default is "oldest")

    Returns:
        list[dict]: list dictionary berisi metadata setiap table
    """
  
    Path = lambda p: spark._jvm.org.apache.hadoop.fs.Path(p)
    fs = spark._jvm.org.apache.hadoop.fs.FileSystem.get(spark._jsc.hadoopConfiguration())

    # 1. Determine the hdfs_path
    if schema.startswith("dev_") or schema == "dgv":
        hdfs_schema_path = f"/dev/{schema}.db"
    else:
        hdfs_schema_path = f"/user/hive/warehouse/{schema}.db"
        
    # 2. Get list of FileStatus objects
    try:
        status_array = fs.listStatus(Path(hdfs_schema_path))
    except Exception as e:
        print(f"Error accessing path {hdfs_schema_path}: {e}")
        return []

    file_list = []

    for status in status_array:
        # 3. Extract metadata directly from the Java object
        full_path = status.getPath().toString()
        filename = status.getPath().getName()
        table_owner = status.getOwner()
        
        # Filter by owner early to skip expensive getContentSummary
        if owner is not None and table_owner != owner:
            continue
        
        # Hadoop returns modification time in milliseconds (Long)
        m_time_ms = status.getModificationTime()
        dt_obj = datetime.fromtimestamp(m_time_ms / 1000.0)
        
        content_summary = fs.getContentSummary(Path(full_path))

        size_in_bytes = content_summary.getLength()
        size_in_formatted_bytes = get_formatted_bytes(content_summary.getLength())
        space_consumed_in_bytes = content_summary.getSpaceConsumed()
        space_consumed_in_formatted_bytes = get_formatted_bytes(content_summary.getSpaceConsumed())

        directory_count = content_summary.getDirectoryCount() - 1
        file_count = content_summary.getFileCount() - 1

        file_list.append({
            'table': filename,
            'owner': table_owner,
            'size': size_in_bytes,
            'size_formatted': size_in_formatted_bytes,
            'space_consumed': space_consumed_in_bytes,
            'space_consumed_formatted': space_consumed_in_formatted_bytes,
            'directory_count': directory_count,
            'file_count': file_count,
            'modified_date': dt_obj,
            'modified_date_formatted': dt_obj.strftime("%Y-%m-%d %H:%M:%S"),
            'path': full_path,
        })

    # 4. Sort
    sort_config = {
        "oldest":   ("modified_date", False),
        "newest":   ("modified_date", True),
        "biggest":  ("space_consumed", True),
        "smallest": ("space_consumed", False),
    }

    if sort is not None:
        if sort not in sort_config:
            raise ValueError(
                f"sort must be one of {list(sort_config.keys())} or None, got '{sort}'"
            )
        key, reverse = sort_config[sort]
        file_list.sort(key=lambda x: x[key], reverse=reverse)
    
    return file_list

#=============================================================#  

def get_list_partition(spark, schema, table):
  """Memperoleh list partisi dari hive table yang diurutkan dari partisi terbaru

  Args:
      spark (pyspark.sql.session.SparkSession): spark session
      schema (str): nama schema dari table di hive
      table (str): nama table di hive
    
  Returns:
      list: list partisi diurutkan dari partisi yang terbaru
  """
  
  try:
    partitions = spark.sql("""
     SHOW PARTITIONS {}.{}
     """.format(schema, table)).sort("partition", ascending=False).collect() # ambil partisi sesuai format
    if len(partitions) != 0: # jika ada partisi
      list_partition = []
      for row in partitions:
        if "__HIVE_DEFAULT_PARTITION__" not in row[0]:
          arrange = []
          dict_partition = {}
          for partition in row[0].split("/"):
            value = partition.split("=")
            arrange.append(value[1].zfill(2))
            dict_partition[value[0]] = value[1]
          dict_partition["__formatted_partition"] = "|".join(arrange)
          list_partition.append(dict_partition)
      list_partition = sorted(list_partition, key=lambda row: row['__formatted_partition'], reverse=True)
      return list_partition
    else: # selain itu
      return None # tidak ada partisi
  except:
    print("is not a partitioned table")
    return None
  
#=============================================================#

def get_first_partition(spark, schema, table):
  """Memperoleh partisi pertama dari hive table

  Args:
      spark (pyspark.sql.session.SparkSession): spark session
      schema (str): nama schema dari table di hive
      table (str): nama table di hive
    
  Returns:
      dict: partisi pertama
  """
    
  partitions = get_list_partition(spark, schema, table)
  if partitions == None:
    return None
  
  return partitions[-1]

#=============================================================#  

def get_last_partition(spark, schema, table):
  """Memperoleh partisi terakhir dari hive table

  Args:
      spark (pyspark.sql.session.SparkSession): spark session
      schema (str): nama schema dari table di hive
      table (str): nama table di hive
    
  Returns:
      dict: partisi terakhir
  """
    
  partitions = get_list_partition(spark, schema, table)
  if partitions == None:
    return None
  
  return partitions[0]

#=============================================================#

def check_partition_exists(spark, target_value, schema, table, mode=None):
    """Mengecek keberadaan partisi pada tabel HDFS dan mengembalikan metadata partisi terkait

    Args:
        spark (pyspark.sql.session.SparkSession): objek spark session yang digunakan
        target_value (str): nilai partisi yang ingin dicek (contoh: '20230307')
            *gunakan pemisah '|' jika partiti kolom lebih dari satu, (contoh: '2023|03|07')
        schema (str): nama database/schema di Hive/HDFS
        table (str): nama table yang akan dicek partisinya
        mode (str): mode pencarian jika target_value tidak ditemukan 
            (optional is None, 'forward', 'backward')
            (default is None)

    Returns:
        dict: dictionary berisi detail kolom partisi (year, month, day, dsb.)
            tanpa kolom __formatted_partition, atau None jika tidak ditemukan
    """
    partition_list = get_list_partition(spark, schema, table)
    
    if not partition_list:
        return None

    # 1. Get a sample format from the metadata
    sample_format = partition_list[0].get('__formatted_partition', '')
    
    # 2. STRICT VALIDATION: Compare structure
    # We build the regex pattern outside the f-string to avoid SyntaxError
    digit_pattern = re.sub(r'\d', r'\\d', sample_format)
    escaped_pattern = digit_pattern.replace('|', r'\|')
    regex_final = "^" + escaped_pattern + "$"
    
    if not re.match(regex_final, target_value):
        raise ValueError(
            "Format Mismatch!\n"
            f"Input target_value: '{target_value}'\n"
            f"HDFS Expected Format: '{sample_format}'\n"
            "Please ensure separators (|, -, /) and padding (01 vs 1) match exactly."
        )

    # 3. Build lookup
    lookup = {}
    for p in partition_list:
        fmt_key = p.get('__formatted_partition')
        if fmt_key:
            # Clean dict for output by removing the helper key
            clean_dict = {k: v for k, v in p.items() if k != '__formatted_partition'}
            lookup[fmt_key] = clean_dict

    # 4. Logic Execution
    sorted_keys = sorted(lookup.keys())

    if target_value in lookup:
        return lookup[target_value]

    if mode == "forward":
        for k in sorted_keys:
            if k > target_value:
                return lookup[k]
                
    elif mode == "backward":
        for k in reversed(sorted_keys):
            if k < target_value:
                return lookup[k]

    return None

#=============================================================#

def drop_partition_exists(spark, target_value, schema, table, mode=None, limit=None, verbose=True):
    """Menghapus partisi dari tabel HDFS

    Args:
        spark (pyspark.sql.session.SparkSession): objek spark session yang digunakan
        target_value (str | list[str]): nilai partisi yang ingin dihapus (contoh: '20230307')
            *gunakan pemisah '|' jika partisi kolom lebih dari satu, (contoh: '2023|03|07')
            *dapat berupa list untuk menghapus beberapa partisi sekaligus,
             (contoh: ['20230307', '20230308'])
        schema (str): nama database/schema di Hive/HDFS
        table (str): nama table yang akan dihapus partisinya
        mode (str): mode pencarian jika target_value tidak ditemukan 
            (optional is None, 'forward', 'backward')
            (default is None)
        limit (int | None): jumlah maksimum partisi yang dihapus saat menggunakan
            mode 'forward' atau 'backward'
            - Jika None → semua partisi yang cocok akan dihapus
            - Jika int  → maksimum sebanyak limit partisi
            (default is None)
        verbose (bool): jika True, menampilkan informasi setiap partisi yang dihapus
            (default is True)

    Returns:
        list[dict] | None: list dictionary berisi detail kolom partisi yang berhasil
            dihapus, atau None jika tidak ada partisi yang dihapus
    """
    partition_list = get_list_partition(spark, schema, table)

    if not partition_list:
        return None

    # Build lookup (same logic as check_partition_exists)
    sample_format = partition_list[0].get('__formatted_partition', '')

    digit_pattern = re.sub(r'\d', r'\\d', sample_format)
    escaped_pattern = digit_pattern.replace('|', r'\|')
    regex_final = "^" + escaped_pattern + "$"

    lookup = {}
    for p in partition_list:
        fmt_key = p.get('__formatted_partition')
        if fmt_key:
            clean_dict = {k: v for k, v in p.items() if k != '__formatted_partition'}
            lookup[fmt_key] = clean_dict

    sorted_keys = sorted(lookup.keys())

    # Normalize target_value to list
    if isinstance(target_value, str):
        target_values = [target_value]
    elif isinstance(target_value, list):
        target_values = target_value
    else:
        raise TypeError(
            f"target_value must be str or list[str], got {type(target_value).__name__}"
        )

    # Resolve each target to partition dict(s)
    dropped = []

    for tv in target_values:
        # Validate format
        if not re.match(regex_final, tv):
            raise ValueError(
                "Format Mismatch!\n"
                f"Input target_value: '{tv}'\n"
                f"HDFS Expected Format: '{sample_format}'\n"
                "Please ensure separators (|, -, /) and padding (01 vs 1) match exactly."
            )

        # Find matching partition(s)
        matches = []

        if tv in lookup:
            matches.append(lookup[tv])
        elif mode == "forward":
            for k in sorted_keys:
                if k > tv:
                    matches.append(lookup[k])
                    if limit is not None and len(matches) >= limit:
                        break
        elif mode == "backward":
            for k in reversed(sorted_keys):
                if k < tv:
                    matches.append(lookup[k])
                    if limit is not None and len(matches) >= limit:
                        break

        # Drop each matched partition
        for matched in matches:
            partition_spec = ", ".join(
                f"{col}='{val}'" for col, val in matched.items()
            )
            drop_query = (
                f"ALTER TABLE {schema}.{table} "
                f"DROP IF EXISTS PARTITION ({partition_spec})"
            )
            spark.sql(drop_query)
            dropped.append(matched)

            if verbose:
                print(f"dropped partition: {matched} from `{schema}.{table}`")

    return dropped if dropped else None

#=============================================================#

def limit_partition_exists(spark, schema, table, limit, keep="newest", verbose=True):
    """Membatasi jumlah partisi yang ada di tabel

    Partisi yang melebihi limit akan dihapus berdasarkan parameter keep.

    Args:
        spark (pyspark.sql.session.SparkSession): objek spark session yang digunakan
        schema (str): nama database/schema di Hive/HDFS
        table (str): nama table yang akan dibatasi partisinya
        limit (int): jumlah maksimum partisi yang diizinkan
        keep (str): menentukan partisi mana yang dipertahankan
            - "newest" : mempertahankan partisi terbaru, menghapus yang terlama
            - "oldest" : mempertahankan partisi terlama, menghapus yang terbaru
            (default is "newest")
        verbose (bool): jika True, menampilkan informasi setiap partisi yang dihapus
            (default is True)

    Returns:
        list[dict] | None: list dictionary berisi detail kolom partisi yang berhasil
            dihapus, atau None jika tidak ada partisi yang dihapus
    """
    if keep not in {"newest", "oldest"}:
        raise ValueError("keep must be 'newest' or 'oldest'")

    if not isinstance(limit, int) or limit < 1:
        raise ValueError("limit must be a positive integer")

    partition_list = get_list_partition(spark, schema, table)

    if not partition_list:
        return None

    total = len(partition_list)

    if total <= limit:
        if verbose:
            print(f"Partition count ({total}) is within limit ({limit}). Nothing to drop.")
        return None

    # partition_list is sorted newest-first (descending)
    if keep == "newest":
        # Keep the first `limit` (newest), drop the rest (oldest)
        to_drop = partition_list[limit:]
    else:
        # Keep the last `limit` (oldest), drop the rest (newest)
        to_drop = partition_list[:total - limit]

    # Extract formatted partition values for drop_partition
    target_values = [p['__formatted_partition'] for p in to_drop]

    if verbose:
        print(f"Dropping {len(target_values)} partition(s), keeping {keep} {limit}...")

    return drop_partition_exists(spark, target_values, schema, table, verbose=verbose)

#=============================================================#

def drop_hive_table(spark, schema, table, verbose=True):
  """Menghapus table dari Hive

  Args:
      spark (pyspark.sql.session.SparkSession): objek spark session yang digunakan
      schema (str): nama database/schema di Hive/HDFS
      table (str | list[str]): nama table atau list nama table yang akan dihapus
      verbose (bool): jika True, menampilkan informasi setiap table yang dihapus
          (default is True)
    
  Returns:
      list[str] | None: list nama table yang berhasil dihapus,
          atau None jika tidak ada table yang dihapus
  """
  # Normalize table to list
  if isinstance(table, str):
      tables = [table]
  elif isinstance(table, list):
      tables = table
  else:
      raise TypeError(
          f"table must be str or list[str], got {type(table).__name__}"
      )

  dropped = []

  for t in tables:
      try:
          spark.sql(f"DROP TABLE IF EXISTS {schema}.{t}")
          dropped.append(t)
          if verbose:
              print(f"dropped table: `{schema}.{t}`")
      except Exception as e:
          print(f"failed to drop `{schema}.{t}`: {e}")

  return dropped if dropped else None

#=============================================================#

def put_to_bridrive(file_path, upload_url):
  """Mengirimkan file dari CDSW ke BRIDrive

  Args:
      upload_url (str): upload link dari Seafile (BRIDrive) untuk upload file
      file_path (str): path file yang akan di upload dari CDSW ke BRIDrive
    
  Returns:
      -
  """

  res = os.system("hdfs dfs -get /user/bribrain_sac/packages/seaf-share.py")

  script = "PYTHONHTTPSVERIFY=0 python seaf-share.py put {}".format(" ".join([upload_url, file_path]))
  os.system(script)
  
  os.remove("/home/cdsw/seaf-share.py")

#=============================================================# 

def put_to_hdfs(file_path, hdfs_path = "/tmp/development/bribrain"):
  """Mengirimkan file dari CDSW ke HDFS

  Args:
      file_path (str): path file yang akan di upload dari CDSW ke HDFS
      hdfs_path (str): path lokasi penyimpanan data di HDFS
          (default is /tmp/development/bribrain)
    
  Returns:
      -
  """
  command = "hdfs dfs -put {0} {1}".format(file_path, hdfs_path)
  res = os.system(command)
  if res == 0:
    print("success - put: `{0}` to `{1}`".format(file_path, hdfs_path))
    
#=============================================================#

def get_from_hdfs(file_path, cml_path = "/home/cdsw/"):
  """Mengambil file dari HDFS ke CML

  Args:
      file_path (str): path file yang akan dipindahkan dari HDFS ke CML
      cdsw_path (str): path penyimpanan file di HDFS yang ingin dipindahkan
          (default is "/home/cdsw/")
    
  Returns:
      -
  """
  command = "hdfs dfs -get {0} {1}".format(file_path, cml_path)
  res = os.system(command)
  if res == 0:
    print("success - get: `{0}` to `{1}`".format(file_path, cml_path))

#=============================================================#

def list_file_hdfs(hdfs_path = "/user/{}/".format(os.environ["HADOOP_USER_NAME"])):
  """Memperlihatkan list file yang ada pada lokasi HDFS

  Args:
      hdfs_path (str): path HDFS yang ingin di show
          (default is "/user/{username}/")
    
  Returns:
      -
  """
  command = "hdfs dfs -ls {}".format(hdfs_path)
  res = os.system(command)

#=============================================================#

