#=========================================
###  POSISI   : 01 JULI 2024
###  PIC      : DINDA PARWITA AULIA NOFRI
###  CASE     : QA QC
###  TASK     : SCRIPT FUNCTION QA (PART.1)
#=========================================

#=========================================
##           Import Dependencies           
#=========================================
from time import time, sleep
from pytz import timezone
from pyspark.sql.types import *
from pyspark.sql.functions import *
from functools import reduce
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from pyspark.sql import SparkSession, functions as F, types as T, Window as W
from pyspark.sql.window import Window
from pyspark.sql import SparkSession, functions as F, types as T, window as W
from math import radians, cos, sin, asin, sqrt
from pyspark.sql.types import StructType, StructField, StringType, IntegerType

import bribrain
import re
from dateutil.parser import parse

##              Pandas Config              
#=========================================
import pandas as pd
pd.options.display.html.table_schema=True
pd.options.display.max_columns=999
pd.options.display.max_rows=999

##              Spark Session              
#=========================================
spark = SparkSession\
    .builder\
    .appName("Func QA part1 ")\
    .config("spark.sql.crossJoin.enabled", "true")\
    .config("spark.dynamicAllocation.enabled", "false")\
    .config("spark.executor.instances", "8")\
    .config("spark.executor.cores", "8")\
    .config("spark.executor.memory", "8g")\
    .config("spark.yarn.executor.memoryOverhead", "8g")\
    .config("spark.network.timeout", 3600)\
    .config("spark.ui.showConsoleProgress","false")\
    .config("spark.sql.legacy.allowCreatingManagedTableUsingNonemptyLocation", "true")\
    .enableHiveSupport()\
    .getOrCreate()

##               Functions                 
#=========================================

### List Function
#1. remove_blank_space
#2. cleanse_blank_space
#3. udf_qc_date
#4. udf_cleansing_date
#5. udf_qc_phonenum
#6. udf_cleansing_non_numeric
#7. udf_formatted_phonenum
#8. udf_clean_phonenum_length
#9. udf_cleansing_phonenum
#10. udf_qc_ktp
#11. udf_clean_ktp_length
#12. udf_cleansing_ktp
#13. udf_qc_acctno
#14. udf_clean_acctno_length
#15. udf_cleansing_acctno

def remove_blank_space(x):
  return F.when((F.trim(F.col(x))!='')&(F.trim(F.upper(F.col(x)))!='NAN'), F.trim(F.col(x))).otherwise(F.lit(None))

def cleanse_blank_space(df_to_clean, *exclude):
  col_to_clean = []
  for val in df_to_clean.dtypes:
    if val[1] == "string" and val[0] not in (exclude):
      col_to_clean.append(val[0])
  return reduce(lambda df, x: df.withColumn(x, remove_blank_space(x)), set(col_to_clean), df_to_clean)

def udf_qc_date():
    def get_qc_date(col):
        try:
            result = datetime.strptime(col, "%Y-%m-%d")
            return "True"
        except:
            return "False"
    
    return F.udf(lambda col: get_qc_date(col), T.StringType())

from dateutil import parser

def udf_cleansing_date(default=True):
    def get_clean_date(col):
        try:
            str_col = str(col)
            try:
                parsed_date = parser.parse(str_col)
                return parsed_date.strftime('%Y-%m-%d')
            except ValueError:
                date_formats = ["%Y%m%d", "%d%m%Y"]
                for date_format in date_formats:
                    try:
                        clean_date = datetime.strptime(col, date_format)
                        return clean_date.strftime("%Y-%m-%d")
                    except ValueError:
                        continue
                if default:
                    return col
                else:
                    return None
        except:
            return None

    return F.udf(lambda col: get_clean_date(col), T.StringType())

def udf_qc_phonenum():
  def get_qc_phonenum(col):
    try:
      if (col.isdigit()) & (10<=len(col)<=13) & (col.startswith('0')):
        return "True"
      else:
        return "False"
    except:
      return None

  return F.udf(lambda col: get_qc_phonenum(col), T.StringType())

def udf_cleansing_non_numeric():
    def cleansing_non_numeric(col):
        try:
            remove_char = re.sub(r'[^0-9]', '', col)
            return remove_char
        except:
            return None
    
    return F.udf(lambda col: cleansing_non_numeric(col), T.StringType())

def udf_formatted_phonenum(default=True):
    def get_formatted_phonenum(col):
        try:
            if col.startswith('620'):
                formatted_phonenum = '0' + col[3:]
            elif col.startswith('62'):
                formatted_phonenum = '0' + col[2:]
            elif col.startswith('8'):
                formatted_phonenum = '0' + col
            elif col.startswith('0'):
                formatted_phonenum = col
            else:
                if default:
                    formatted_phonenum = col
                else:
                    formatted_phonenum = None
                    
            if len(set(formatted_phonenum)) in (1, 2):
                if default:
                    formatted_phonenum = col
                else:
                    formatted_phonenum = None
                    
            return formatted_phonenum
        except:
            return None
        
    return F.udf(lambda col: get_formatted_phonenum(col), T.StringType())

def udf_clean_phonenum_length(default=True):
    def clean_phonenum_length(col):
        try:
            if 10 <= len(col) <= 13:
                clean_phonenum = col
                return clean_phonenum
            else:
                if default:
                    return col
                else:
                    return None
        except:
            return None
        
    return F.udf(lambda col: clean_phonenum_length(col), T.StringType())

def udf_cleansing_phonenum(default=True):
    def get_clean_phonenum(col):
        try:
            # Menghapus karakter non-digit
            remove_char = re.sub(r'[^0-9]', '', col)
            
            # Format nomor telepon berdasarkan awalan
            if remove_char.startswith('620'):
                formatted_phonenum = '0' + remove_char[3:]
            elif remove_char.startswith('62'):
                formatted_phonenum = '0' + remove_char[2:]
            elif remove_char.startswith('8'):
                formatted_phonenum = '0' + remove_char
            elif remove_char.startswith('0'):
                formatted_phonenum = remove_char
            else:
                if default:
                    formatted_phonenum = col
                else:
                    formatted_phonenum = None
            
            if len(set(formatted_phonenum)) in (1, 2):
                if default:
                    new_formatted_phonenum = formatted_phonenum
                else:
                    return None
            else:
                new_formatted_phonenum = formatted_phonenum
                    
            # Memastikan panjang nomor telepon antara 10 hingga 13 digit
            if 10 <= len(new_formatted_phonenum) <= 13:
                return new_formatted_phonenum
            else:
                if default:
                    return col
                else:
                    return None
        except:
            return None
    
    return F.udf(lambda col: get_clean_phonenum(col), T.StringType())

def udf_qc_ktp():
    def get_qc_ktp(col):
        try:
            if (col.isdigit()) & (len(col)==16):
                return "True"
            else:
                return "False"
        except:
            return None

    return F.udf(lambda col: get_qc_ktp(col), T.StringType())

def udf_clean_ktp_length(default=True):
    def clean_ktp_length(col):
        try:
            if len(col) == 16:
                clean_ktp = col
                return clean_ktp
            else:
                if default:
                    return col
                else:
                    return None
        except:
            return None
        
    return F.udf(lambda col: clean_ktp_length(col), T.StringType())

def udf_cleansing_ktp(default=True):
    def get_clean_ktp(col):
        try:
            # Menghapus karakter non-digit
            remove_char = re.sub(r'[^0-9]', '', col)
            
            # Memastikan panjang ktp 16 digit
            if len(remove_char) == 16:
                clean_ktp = remove_char
                return clean_ktp
            else:
                if default:
                    return col
                else:
                    return None
        except:
            return None
    
    return F.udf(lambda col: get_clean_ktp(col), T.StringType())

def udf_qc_acctno():
    def get_qc_acctno(col):
        try:
            str_col = str(col).strip()
            
            if (str_col.isdigit()) & (12<=len(str_col)<=15):
                return "True"
            else:
                return "False"
        except:
            return None

    return F.udf(lambda col: get_qc_acctno(col), T.StringType())

def udf_clean_acctno_length(default=True):
    def clean_acctno_length(col):
        try:
            str_col = str(col).strip()
            if 12 <= len(str_col) <= 15:
                clean_acctno = str_col
                return clean_acctno
            else:
                if default:
                    return str(col).strip()
                else:
                    return None
        except:
            return None
        
    return F.udf(lambda col: clean_acctno_length(col), T.StringType())

def udf_cleansing_acctno(default=True):
    def get_clean_acctno(col):
        try:
            str_col = str(col).strip()
            # Menghapus karakter non-digit
            remove_char = re.sub(r'[^0-9]', '', str_col)
            
            # Memastikan panjang ktp 16 digit
            if 12 <= len(remove_char) <= 15:
                clean_acctno = remove_char
                return clean_acctno
            else:
                if default:
                    return col
                else:
                    return None
        except:
            return None
    
    return F.udf(lambda col: get_clean_acctno(col), T.StringType())
