#%%

import os, sys
import numpy as np
import pandas
import pandas as pd
import datetime 
import time 
import logging
from pytz import timezone
from pandas import json_normalize
import cryptography
import base64
# from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
# from cryptography.fernet import Fernet
import math
from pathlib import Path
#import adal
import threading
import concurrent.futures
import requests
import json
import yaml
import argparse
import string
import pickle
import urllib
import pytz
import ldap3
import xml.etree.ElementTree as ET
from urllib.parse import urlparse
import socket
import re
import hashlib
import subprocess
import random
import xml.dom.minidom
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import smtplib
from email import encoders
from ast import literal_eval
import glob
import shutil

pd.options.display.max_columns = 20
pd.options.display.width = 300


visibilityTimeWindow = 10 #seconds


logger = logging.getLogger(__name__)


import warnings
from requests.packages.urllib3.exceptions import InsecureRequestWarning
warnings.simplefilter('ignore',InsecureRequestWarning)





today_date = datetime.datetime.now().strftime('%d-%b-%Y')




#%%


# chardet.charsetpropber takes a looong time to detect the charset of response.text in requests.
# Charset detection in text() can be pathologically slow #2359
# https://github.com/psf/requests/issues/2359







# Convert epoch value to date format
def epochToDate(value):
	return pd.to_datetime(value).date()




# Covert timestamp into the correct datetime format
def standardizeTimestampString(value, existingstringformat=["%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%dT%H:%M:%S", "%m/%d/%Y %I:%M:%S %p", "%d/%m/%Y %I:%M:%S %p", \
    "%Y/%m/%d %H:%M:%S", "%d/%m/%Y %H.%M.%S"], outputstringformat="%Y-%m-%dT%H:%M:%S%z", timezoneValue='UTC'):
    dateTimeValue = None
    if value == '':
        return value
    logger = logging.getLogger(__name__ + '.standardizeTimestampString')
    for formatstring in existingstringformat:
        try:
            dateTimeValue = datetime.datetime.strptime(value, formatstring)
            break
        except KeyboardInterrupt:
            sys.exit(1)
        except Exception:
            pass
    if dateTimeValue == None:
        logger.warning(f'Value: {value}, could not be converted to DateTime in python.')
        return value
    dateTimeValue = dateTimeValue.astimezone(timezone('UTC'))
    try:
        standardizedDateTimeString = datetime.datetime.strftime(dateTimeValue, outputstringformat)
    except KeyboardInterrupt:
        sys.exit(1)
    except Exception:
        logger.exception(f'Value: {value}, dateTimeValue: {dateTimeValue}, outputstringformat: {outputstringformat}')
        return value
    return standardizedDateTimeString



# Covert timestamp into the correct datetime format
def getDateStringFromTimestampString(value, existingstringformat=["%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%dT%H:%M:%S", "%m/%d/%Y %I:%M:%S %p", "%d/%m/%Y %I:%M:%S %p", \
    "%Y/%m/%d %H:%M:%S", "%d/%m/%Y %H.%M.%S"], outputstringformat="%Y-%m-%d", timezoneValue='UTC'):
    dateTimeValue = None
    if value == '':
        return value
    logger = logging.getLogger(__name__ + '.getDateStringFromTimestampString')
    for formatstring in existingstringformat:
        try:
            dateTimeValue = datetime.datetime.strptime(value, formatstring)
            break
        except KeyboardInterrupt:
            sys.exit(1)
        except Exception:
            pass
    if dateTimeValue == None:
        logger.warning(f'Value: {value}, could not be converted to DateTime in python.')
        return value
    dateTimeValue = dateTimeValue.astimezone(timezone('UTC'))
    try:
        standardizedDateString = datetime.datetime.strftime(dateTimeValue.date(), outputstringformat)
    except KeyboardInterrupt:
        sys.exit(1)
    except Exception:
        logger.exception(f'Value: {value}, dateTimeValue: {dateTimeValue}, outputstringformat: {outputstringformat}')
        return value
    return standardizedDateString




def getDateTime(value, existingstringformat=["%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%dT%H:%M:%S", "%m/%d/%Y %I:%M:%S %p", "%d/%m/%Y %I:%M:%S %p", \
    "%Y/%m/%d %H:%M:%S", "%d/%m/%Y %H.%M.%S"], timezoneValue='UTC'):
    logger = logging.getLogger(__name__ + '.getDateTime')
    dateTimeValue = None
    if value in ['', None]:
        return dateTimeValue
    for formatstring in existingstringformat:
        try:
            dateTimeValue = datetime.datetime.strptime(value, formatstring)
            break
        except KeyboardInterrupt:
            sys.exit(1)
        except Exception:
            pass
    if dateTimeValue == None:
        logger.warning(f'Value: {value}, could not be converted to DateTime in python.')
        return dateTimeValue
    dateTimeValue = dateTimeValue.astimezone(timezone('UTC'))
    return dateTimeValue






'''
def getTimeStampString(value, standardstringformat="%Y-%m-%dT%H:%M:%S%z", timezoneValue='UTC'):
    logger = logging.getLogger(__name__ + '.getTimeStampString')
    timestampString = ''
    if type(value) == datetime.datetime:
        #value = value.astimezone(timezone('UTC'))
        try:
            timestampString = datetime.datetime.strftime(value, standardstringformat)
        except KeyboardInterrupt:
            sys.exit(1)
        except Exception:
            logger.exception(f'value: {value}, standardstringformat: {standardstringformat}')
            return timestampString
    elif type(value) ==  pd._libs.tslibs.timestamps.Timestamp:
        timestampString = value.strftime(standardstringformat)
        return timestampString
    else:
        return value
'''

def getTimeStampString(value, standardstringformat="%Y-%m-%dT%H:%M:%S%z"):
    locallogger = logging.getLogger(__name__ + '.getTimeStampString')
    
    if type(value) == datetime.datetime:
        timestampString = datetime.datetime.strftime(value, standardstringformat)
        # locallogger.debug(f"datetime.datetime value; timestampString: {timestampString}")
        return timestampString
    else:
        return value


def removeTimeZone(value):
    if (type(value) == datetime.datetime) or (type(value) == pd._libs.tslibs.timestamps.Timestamp):
        value = value.replace(tzinfo=None)
    return value




def timeZoneConversionUTCToLocal(utc_dt, pytz_timeZoneString='Asia/Singapore'):
    if type(utc_dt) == datetime.datetime:
        local_tz = pytz.timezone(pytz_timeZoneString)
        local_dt = utc_dt.replace(tzinfo=pytz.utc).astimezone(local_tz)
        return local_tz.normalize(local_dt)
    else:
        return utc_dt


def dateFromDateTime(value):
    if type(value) == datetime.datetime:
        value = value.date()
    return value


# Convert date values from origin to standardized date representation
def formatDate(value, stringformat="%Y-%m-%dT%H:%M:%S%z"):
    if type(value) == pd.Timestamp:
        return value.to_pydatetime().date()
    elif type(value) == datetime.datetime:
        return value.date()
    elif type(value) == datetime.date:
        return value
    elif type(value) == int:
        return pd.to_datetime(value).date()
    elif type(value) == str:
        if value == '':
            return value
        else:
            return datetime.datetime.strptime(value, stringformat).date()
    else:
        return ''

#%% ===================================================================================================


def getUniqueTimeReferenceString(unit='this_week'):
    value = ""
    today = datetime.date.today()
    
    latestMonday = today - datetime.timedelta(days = today.weekday() )
    sundayBeforeLatestMonday = latestMonday - datetime.timedelta(days = 1)
    mondayBeforeLatestMonday = latestMonday - datetime.timedelta(days = 7)
    sundayAfterLatestMonday = latestMonday + datetime.timedelta(days = 6)
    
    if unit == 'last_week':
        value = f"{mondayBeforeLatestMonday.strftime('%d-%b-%Y')}_to_{sundayBeforeLatestMonday.strftime('%d-%b-%Y')}"
        
    if unit == 'this_week':
        value = f"{latestMonday.strftime('%d-%b-%Y')}_to_{sundayAfterLatestMonday.strftime('%d-%b-%Y')}"
        
    if unit == 'this_month':
        value = f"{today.strftime('%Y_%b')}"
    
    return value

#%% ===================================================================================================





# startTime = datetime.datetime(year=2022, month=1, day=1)
# timeIntervalUnit = datetime.timedelta(days=1)
# timeInterval = datetime.timedelta(days=6)
# endTime = datetime.datetime.now()
# outputStringformat = "%Y-%m-%d"

# timeIntervalGenerator = TimeIntervalGenerator(startTime=startTime, timeIntervalUnit=timeIntervalUnit, timeInterval=timeInterval, endTime=endTime)

# while True:
#     outputDict = timeIntervalGenerator.nextIteration(outputStringformat=outputStringformat)
#     print(outputDict)
#     if outputDict == False:
#         break







#%% ===================================================================================================

# Convert from Timestamp to String Date
def fromTimestampToStringDate(value, stringformat='%d-%b-%Y'):
    if type(value) == pd.Timestamp:
        return value.to_pydatetime().strftime(stringformat)
    elif type(value) == datetime.datetime:
        return value.strftime(stringformat)
    else:
        return value
         



#%% ======================================================================================================


def stripOuterSpacesInStringColumnInDataframe(df, columnList, loggingPrefix=None):
    locallogger = getLocalLogger(localloggername='stripOuterSpacesInStringColumnInDataframe', rootFileName=__name__, loggingPrefix=loggingPrefix)
    df = df.copy()
    def stripString(objectValue):
        if type(objectValue) != str:
            objectValue = str(objectValue)
        objectValue = objectValue.strip()
        return objectValue
    for column in columnList:
        df[column] = df.apply(lambda x: stripString(x[column]), axis=1)
    locallogger.debug(f"Stripped outer spaces in the string columns.  Column List {columnList}")
    return df



#%% ======================================================================================================


# Get a Cross Tabulation based on two columns in the dataframe
def crossTab(df, col1, col2, sortOrder=True):
    # t = pd.crosstab(df[col1].astype(str), df[col2].astype(str))  # This is causing issues as the columns which are supposed to be 'date' values are being returned as 'str'
    t = pd.crosstab(df[col1], df[col2]) 
    t = t.assign(Total = t.sum(axis=1))
    #t['Total'] = t.sum(axis=1)
    t['Percentage'] = t['Total'] * 100 / t['Total'].sum(axis=0)
    t['Percentage'] = t['Percentage'].round(2)
    if(sortOrder):
        t = t.sort_values('Total', ascending=False)
    t[col1] = t.index
    t = moveColumns(df=t, columnsList=[col1], refColumn=None)
    t = t.rename_axis(index=None, columns= t.index.name)
    return t




def dataframeExtractColumnContainingList(do_df, column_name):
    do_df = do_df.copy()
    do_df = do_df.reset_index(drop=True)
    locallogger = logging.getLogger(f"{__name__}.dataframeExtractColumnContainingList")
    
    locallogger.debug(f"Starting Operation.  Shape of do_df: {do_df.shape}")
    column_flat = pd.DataFrame([[i, c_flattened] for i, y in do_df[column_name].apply(list).items() for c_flattened in y], columns=['I', column_name])
    column_flat = column_flat.set_index('I')
    do_df = do_df.drop(column_name, axis=1).merge(column_flat, left_index=True, right_index=True)
    do_df.reset_index(drop=True, inplace=True)
    locallogger.debug(f"Finished Operation. Shape of do_df: {do_df.shape}")
    return do_df






def dataframeExtractColumnContainingDict(do_df, column_name):
    do_df = do_df.copy()
    do_df = do_df.reset_index(drop=True)
    locallogger = logging.getLogger(f"{__name__}.dataframeExtractColumnContainingDict")
    
    locallogger.debug(f"Starting Operation.  Shape of do_df: {do_df.shape}")
    progressTracker = ProgressTracker(len(do_df))
    df = pd.DataFrame()
    for index, row in do_df.iterrows():
        element = row[column_name]
        #locallogger.debug(f"index: {index}, element: {element}")
        elementDf = json_normalize(element)
        #locallogger.debug(f"elementDf: {elementDf}")
        df = pd.concat([df, elementDf], axis=0, ignore_index=True, sort=False)
        #locallogger.debug(f"df: {df}")
        progressTracker.progressReport(locallogger)
    #locallogger.debug(f"Concatenation finished.  Columns of the df: {df.columns}")
    do_df = do_df.drop(columns=column_name, inplace=False)
    df = do_df.merge(df, left_index=True, right_index=True)
    locallogger.debug(f"Finished Operation. Shape of df: {df.shape}")
    return df






def dataframeExtractColumnWithListAndDict(do_df, column_name):
    do_df = do_df.copy()
    locallogger = logging.getLogger(f"{__name__}.dataframeExtractColumnWithListAndDict")
    
    locallogger.debug(f"Starting Operation.  Shape of do_df: {do_df.shape}")
    do_df = dataframeExtractColumnContainingList(do_df, column_name)
    df = dataframeExtractColumnContainingDict(do_df, column_name)
    locallogger.debug(f"Finished Operation. Shape of df: {df.shape}")
    return df




def cryptOperation(passphrase, salt):
    # Convert to type bytes
    passphraseEncoded = passphrase.encode() 
    saltEncoded = salt.encode()
    # Key Derivation Function
    kdf = PBKDF2HMAC(
            algorithm=cryptography.hazmat.primitives.hashes.SHA256(),
            length=32,
            salt=saltEncoded,
            iterations=100000,
            backend=cryptography.hazmat.backends.default_backend(),
        )
    key = base64.urlsafe_b64encode(kdf.derive(passphraseEncoded))
    fernetSymmetric = Fernet(key)
    return fernetSymmetric






def decryptFile(encrypted_file_full_path, decrypted_file_full_path, passphrase, salt):
    locallogger = logging.getLogger(f"{__name__}.DataframeExtractColumnContainingList")
    
    fernetSymmetric = cryptOperation(passphrase, salt)
    # Decrypt the encrypted file
    with open(encrypted_file_full_path, 'rb') as file:
        encryptedData = file.read()
        decryptedData = fernetSymmetric.decrypt(encryptedData)
        # Write decrypted data to excel file
        with open(decrypted_file_full_path, 'wb') as file:
            file.write(decryptedData)





def encryptFile(decrypted_file_full_path, encrypted_file_full_path,  passphrase, salt):
    fernetSymmetric = cryptOperation(passphrase, salt)
    # Decrypt the encrypted file
    with open(decrypted_file_full_path, 'rb') as file:
        decryptedData = file.read()
        encryptedData = fernetSymmetric.encrypt(decryptedData)
        # Write decrypted data to excel file
        with open(encrypted_file_full_path, 'wb') as file:
            file.write(encryptedData)



def decryptData(encryptedData, passphrase, salt):
    encryptedData = encryptedData.encode()
    fernetSymmetric = cryptOperation(passphrase, salt)
    decryptedData = fernetSymmetric.decrypt(encryptedData)
    decryptedData = decryptedData.decode()
    return decryptedData


def encryptData(decryptedData, passphrase, salt):
    decryptedData = decryptedData.encode()
    fernetSymmetric = cryptOperation(passphrase, salt)
    encryptedData = fernetSymmetric.encrypt(decryptedData)
    encryptedData = encryptedData.decode()
    return encryptedData




def encryptDataFrame(df, columnsToEncryptList, passphrase, salt):
    for col in columnsToEncryptList:
        df[col] = df[col].apply(lambda x: encryptData(decryptedData=x, passphrase=passphrase, salt=salt))
    return df


def decryptDataFrame(df, columnsToDecryptList, passphrase, salt):
    for col in columnsToDecryptList:
        df[col] = df[col].apply(lambda x: decryptData(encryptedData=x, passphrase=passphrase, salt=salt))
    return df










def getAppendedListOfUniqueElements(series, Unique_ID):
    locallogger = logging.getLogger(f"{__name__}.getAppendedListOfUniqueElements")
    
    list1 = []
    for item in series:
        #locallogger.debug(f"Unique_ID: {Unique_ID}, item: {item}, type:{type(item)}")
        if type(item) == str:
            if item.strip() != '':
                list1.append(item)
        elif type(item) == list:
            list1 = list1 + item
        elif pd.isnull(item):
            pass
        else:
           locallogger.warning(f"Unique_ID: {Unique_ID}, item: {item}, type: {type(item)}, Type is not string, list or 'np.nan'.") 
    list1 = list(set(list1)) 
    return list1 





def stringUpperAndStrip(stringValue):
    stringValue = str(stringValue)
    stringValue = stringValue.strip()
    stringValue = stringValue.upper()
    return stringValue


def stringLowerAndStrip(stringValue):
    stringValue = str(stringValue)
    stringValue = stringValue.strip()
    stringValue = stringValue.lower()
    return stringValue



def flattenList(list_of_lists):
    flattened = []
    for sublist in list_of_lists:
        for val in sublist:
            flattened.append(val)
    return flattened  





def checkSubStringPresence(subStringList, mainString):
    locallogger = logging.getLogger(f"{__name__}.checkSubStringPresence")
    
    if type(mainString) != str:
        locallogger.error(f"mainString: {mainString} is not in String format.")
        sys.exit(1)
    mainString = mainString.lower()
    for subString in subStringList:
        if subString in mainString:
            return True
    return False




def dropRowsWithStringColumnHavingEmptyOrNullValues(df, stringColumnName):
    locallogger = logging.getLogger(f"{__name__}.dropRowsWithStringColumnHavingEmptyOrNullValues")
    
    df = df.copy()
    locallogger.debug(f"Before operation: df.shape: {df.shape}")
    df[stringColumnName] = df.apply(lambda x: x[stringColumnName].strip(), axis=1)
    df[stringColumnName] = df[stringColumnName].replace('', np.nan, inplace=False)
    df = df.dropna(subset=[stringColumnName], inplace=False)
    locallogger.debug(f"After operation: df.shape: {df.shape}")
    return df


















def dropNonNeededColumns(df, neededColumnsList):
    locallogger = logging.getLogger(f"{__name__}.dropNonNeededColumns")
    
    allColumnsList = list(df.columns)
    nonNeededColumnsList = [x for x in allColumnsList if x in neededColumnsList]
    locallogger.debug(f"Needed Columns List: {neededColumnsList}")
    locallogger.debug(f"Non-Needed Columns List: {nonNeededColumnsList}")
    locallogger.debug(f"Dropping Non-Needed Columns.")
    df = df.drop(columns=nonNeededColumnsList)
    return df




def getDefinedColumns(columnDefExcelFullPath, defColumnName):
    locallogger = logging.getLogger(f"{__name__}.getDefinedColumns")
    
    filePathObject = Path(columnDefExcelFullPath)
    if filePathObject.is_file():
        tempDf = readExcel(filePath=columnDefExcelFullPath, sheetName='data')
    else:
        locallogger.error(f"File does not exist: {columnDefExcelFullPath}")
        sys.exit(1)
    definedColumnsList = tempDf[defColumnName].tolist()
    definedColumnsList = [x for x in definedColumnsList if x != '']
    if not len(definedColumnsList):
        locallogger.error(f"No column values defined under columns name '{defColumnName}' in the excel '{columnDefExcelFullPath}'")
        sys.exit(1)
    return definedColumnsList



def renameDataframeColumns(df, columnDefExcelFullPath, defColumnNameFrom, defColumnNameTo):
    locallogger = logging.getLogger(f".{__name__}.renameDataframeColumns")
    
    error = False
    nonDefinedColumnList = []
    definedColumnDoesNotExistList = []
    definedColumnsFromList = getDefinedColumns(columnDefExcelFullPath, defColumnNameFrom)
    definedColumnsToList = getDefinedColumns(columnDefExcelFullPath, defColumnNameTo)
    existingColumnsList = df.columns
    if not len(existingColumnsList):
        locallogger.error(f"Dataframe does not have any columns")
        sys.exit(1)  
    for column in definedColumnsFromList:
        if column not in existingColumnsList:
            error = True
            definedColumnDoesNotExistList.append(column) 
    if error:
        locallogger.error(f"Column '{definedColumnDoesNotExistList}' defined under columns name '{defColumnNameFrom}' in the excel '{columnDefExcelFullPath}' not present in existing columns in the dataframe.")
        sys.exit(1)
    for column in existingColumnsList:
        if column not in definedColumnsFromList:
            nonDefinedColumnList.append(column)
    for i in range(0, len(definedColumnsFromList)):
        df = df.rename(columns = {definedColumnsFromList[i] : definedColumnsToList[i]})
    arrangedColumnsList = definedColumnsToList + nonDefinedColumnList
    df = df[arrangedColumnsList]
    locallogger.info(f"-----------------------------------------------------------------------")
    locallogger.info(f"Renamed dataframe columns.  Full Column list: {df.columns}")
    if len(nonDefinedColumnList):
        locallogger.info(f"-----------------------------------------------------------------------")
        locallogger.warning(f"Non defined Columns: {nonDefinedColumnList}")
        locallogger.info(f"-----------------------------------------------------------------------")
    return df



def arrangeDataframeByColumns(df, columnDefExcelFullPath, defColumnName, ):
    locallogger = logging.getLogger(f"{__name__}.arrangeDataframeByColumns")
    
    error = False
    nonDefinedColumnList = []
    definedColumnDoesNotExistList = []
    existingColumnsList = df.columns
    if not len(existingColumnsList):
        locallogger.error(f"Dataframe does not have any columns")
        sys.exit(1)        
    definedColumnsList = getDefinedColumns(columnDefExcelFullPath, defColumnName)
    for column in definedColumnsList:
        if column not in existingColumnsList:
            error = True
            definedColumnDoesNotExistList.append(column) 
    if error:
        locallogger.error(f"Columns '{definedColumnDoesNotExistList}' defined under columns name '{defColumnName}' in the excel '{columnDefExcelFullPath}' not present in existing columns in the dataframe.")
        sys.exit(1)
    for column in existingColumnsList:
        if column not in definedColumnsList:
            nonDefinedColumnList.append(column)
    arrangedColumnsList = definedColumnsList + nonDefinedColumnList
    df = df[arrangedColumnsList]
    locallogger.info(f"-----------------------------------------------------------------------")
    locallogger.info(f"Arranged dataframe by columns: {df.columns}")
    if len(nonDefinedColumnList):
        locallogger.info(f"-----------------------------------------------------------------------")
        locallogger.warning(f"Non defined Columns: {nonDefinedColumnList}")
        locallogger.info(f"-----------------------------------------------------------------------")
    return df


#%%


#%%
    
def replaceValue(objectValue, checkValue, replacementValue):
    if objectValue == checkValue:
        objectValue = replacementValue
    return objectValue

#%%

def replaceValueInDataFrame(df, excludedColumnsList, checkValue, replacementValue):
    locallogger = logging.getLogger(f"{__name__}.replaceValueInDataFrame")
    if excludedColumnsList != None:
        includedColumnsList = columnListExceptCertainOnes(df, exceptList=excludedColumnsList)
    else:
        includedColumnsList = list(df.columns)
        excludedColumnsList = []
    for column in includedColumnsList:  
        df[column] = df[column].apply(lambda x: replaceValue(x, checkValue, replacementValue))
    locallogger.info("Replacement completed.")
    return df



#%%

def writeListtoExcel(valueList, folderPath, fileNamePrefix):
    tempDf = pd.DataFrame({'Values' : valueList})
    writeExcel(dfList=[tempDf], folderPath=folderPath, fileNamePrefix=fileNamePrefix)



# Function to get authentication token from Azure
def getAzureAccessToken(client_id, client_secret, tenant_id, subscription_name,  proxies):
    locallogger = logging.getLogger(f'{__name__}.getAzureAccessToken')
    
    token = ''
    errorCount = 0
    authorityURL = f'https://login.microsoftonline.com/{tenant_id}'
    authContext = adal.AuthenticationContext(authorityURL, verify_ssl=False, proxies=proxies)
    azureResource = f'https://management.azure.com'
    while token == '':
        try:
            token = authContext.acquire_token_with_client_credentials(azureResource, client_id, client_secret)
        except KeyboardInterrupt:
            sys.exit(1)
        except Exception:
            locallogger.exception(f'Exception happened while acquring token for subscription_name:{subscription_name}')
            errorCount = errorCount + 1
            locallogger.warning(f"errorCount: {errorCount}")
            if errorCount >=5:
                locallogger.error(f"errorCount reached value of 5.  Skipping subscription.")
                return token
            pass
    authHeader = {
        'Authorization': f"Bearer {token['accessToken']}",
        'Content-Type': 'application/json',
    }
    locallogger.debug(f'Access token was received successfully for subscription_name:{subscription_name}')
    #locallogger.debug(f'authHeader: {authHeader}')
    return authHeader





def getAzureAccessTokenADAL(adalTokenDict, tenantID, clientID, clientSecret, proxies):
    locallogger = logging.getLogger(f'{__name__}.getAzureAccessTokenADAL')
    
    errorDetails = ''
    tenantID = str(tenantID)
    clientID = str(clientID)
    clientSecret = str(clientSecret)
    
    tokenID = f"{tenantID}_{clientID}_{clientSecret}"
    
    if tokenID in adalTokenDict.keys():
        existingValueDict = adalTokenDict[tokenID]
        nowTime = datetime.datetime.now()
        deltaTime = datetime.timedelta(minutes=30)
        
        tokenGenerationTime = existingValueDict['Generation_Time']
        thresholdTime = tokenGenerationTime + deltaTime
        
        if nowTime < thresholdTime: # It has not been 30 minutes since the token was generated
            authHeader = existingValueDict['Auth_Header']
            locallogger.info(f"Returning token from memory...")
            return (authHeader, errorDetails, adalTokenDict)
        
    
    
    token = ''
    authHeader = None
    errorCount = 0
    authorityURL = f'https://login.microsoftonline.com/{tenantID}'
    authContext = adal.AuthenticationContext(authorityURL, verify_ssl=False, proxies=proxies)
    azureResource = f'https://management.azure.com'
    while token == '':
        try:
            token = authContext.acquire_token_with_client_credentials(azureResource, clientID, clientSecret)
        except Exception as e:
            locallogger.exception(f'Exception happened while acquring token for tenantID:{tenantID}, clientID:{clientID}')
            errorCount = errorCount + 1
            locallogger.warning(f"errorCount: {errorCount}.  ")
            time.sleep(5 * errorCount)
            if errorCount >=5:
                locallogger.error(f"errorCount reached value of 5.")
                errorDetails = str(e)
                return (authHeader, errorDetails, adalTokenDict)
            pass
    authHeader = {
        'Authorization': f"Bearer {token['accessToken']}",
        'Content-Type': 'application/json',
    }
    
    valueDict = {
            'Generation_Time' : datetime.datetime.now(),
            'Auth_Header' : authHeader,
        }
    
    adalTokenDict[tokenID] = valueDict
    
    
    locallogger.debug(f'Access token was received successfully for tenantID:{tenantID}')
    #locallogger.debug(f'authHeader: {authHeader}')
    return (authHeader, errorDetails, adalTokenDict)



# The below uses MSAL instead of ADAL.  Yet to understand MSAL fully.

def getAzureAccessTokenAPI(tenantID, clientID, clientSecret):
    locallogger = logging.getLogger(f'{__name__}.getAzureAccessTokenAPI')
    token = None
    
    url = f"https://login.microsoftonline.com/{tenantID}/oauth2/v2.0/token"
    method = "POST"
    
    #scopeString = "https://graph.microsoft.com/.default"
    scopeString = "https://management.azure.com"
    # Making modification as per instructions at: https://stackoverflow.com/questions/56060462/what-are-proper-scopes-for-api-requests
    #scopeString = "https://management.azure.com/.default"
    
    data = {
                "grant_type" : "client_credentials",
                "client_id" : clientID,
                "client_secret" : clientSecret,
                "scope" : scopeString
                
            }
    
    
    response, _ = getResponseFromURL(url, method, headers=None, auth=None,  proxies=None, data=data, json=None, params=None, verify=False, timeout=120, session=None, loggingPrefix=None, responseEncoding=None, maxRetry=3)
    
    if response.status_code == 200:
        jresponse = response.json()
        
        token = jresponse['access_token']
    else:
        locallogger.error(f"Failed to get Azure Access Token for Tenant ID: {tenantID}")
        sys.exit(1)
    
    
    authHeader = {
        'Authorization': f"Bearer {token}",
        'Content-Type': 'application/json',
    }
    
    return (token, authHeader)


def stringToBase64(string):
    return base64.b64encode(string.encode())


def base64ToString(value):
    return base64.b64decode(value).decode()


def getSecretsFromHashicorpVault(hashicorpLoginURL, keyVaultURL, userID, password, proxies):
    
    locallogger = logging.getLogger(f'{__name__}.getSecretsFromHashicorpVault')
    

    locallogger.info(f"Getting Secrets from Hashicorp Vault")


    apiURL = f"{hashicorpLoginURL}/{userID}"
    headers = {'Accept': 'application/json'}
    data = {"password": password}
    # response = requests.post(apiURL, headers=headers, json=data, proxies=proxies, verify=False, timeout=120)
    response, dataPullStatus = getResponseFromURL(url=apiURL, method='POST', headers=headers,  proxies=proxies, json=data, verify=False, timeout=120, loggingPrefix=locallogger.name)
    if response.status_code != 200:
        locallogger.error(f"Login to Hashicorp Vault Failed!")
        sys.exit(1)
    jresponse = response.json()
    vaultToken = jresponse['auth']['client_token']
    
    



    apiURL = f"{keyVaultURL}"
    headers = {'x-vault-token': f'{vaultToken}'}
    #response = requests.get(apiURL, headers=headers, proxies=proxies, verify=False, timeout=120)
    response, dataPullStatus = getResponseFromURL(url=apiURL, method='GET', headers=headers,  proxies=proxies, verify=False, timeout=120, loggingPrefix=locallogger.name)
    if response.status_code != 200:
        locallogger.error(f"Retrieval of secrets from Hashicorp Vault failed!")
        sys.exit(1)    
    jresponse = response.json()
    return jresponse['data']['data']
    






def getHashicorpUserToken(hashicorpLoginURL, userID, password):
    # Logging into Vault using AD Account
    locallogger = logging.getLogger(f'{__name__}.getHashicorpVaultAppRoleToken')
    apiURL = f"{hashicorpLoginURL}/{userID}"
    headers = {'Accept': 'application/json'}
    data = {"password": password}

    response, _ = getResponseFromURL(url=apiURL, method='POST', headers=headers,  proxies=None, json=data, verify=False, timeout=120, loggingPrefix=locallogger.name)
    if response.status_code != 200:
        locallogger.error(f"Login to Hashicorp Vault Failed!")
        sys.exit(1)
    jresponse = response.json()
    vaultToken = jresponse['auth']['client_token']
    return vaultToken
    




def getHashicorpAppRoleID(roleURL, vaultToken):
    # Getting App role details - role_id
    locallogger = logging.getLogger(f'{__name__}.getHashicorpAppRoleID')
    apiURL = f'{roleURL}'
    headers = {'x-vault-token': f'{vaultToken}'}

    response, _ = getResponseFromURL(url=apiURL, method='GET', headers=headers,  proxies=None, verify=False, timeout=120, loggingPrefix=locallogger.name)
    if response.status_code != 200:
        locallogger.error(f"Retrieval of app role ID from Hashicorp Vault failed!")
        sys.exit(1)    
    jresponse = response.json()
    roleID = jresponse['data']['role_id']
    return roleID





def getHashicorpAppRoleSecretID(secretIDURL, vaultToken):
    # Getting App role details - secret_id
    locallogger = logging.getLogger(f'{__name__}.getHashicorpAppRoleSecretID')
    apiURL = f'{secretIDURL}'
    headers = {'x-vault-token': f'{vaultToken}'}
    #response = requests.get(apiURL, headers=headers, proxies=proxies, verify=False, timeout=120)
    response, _ = getResponseFromURL(url=apiURL, method='POST', headers=headers,  proxies=None, verify=False, timeout=120, loggingPrefix=locallogger.name)
    if response.status_code != 200:
        locallogger.error(f"Retrieval of app role secret from Hashicorp Vault failed!")
        sys.exit(1)    
    jresponse = response.json()
    secretID = jresponse['data']['secret_id']
    return secretID





def getHashicorpAppRoleToken(appRoleLoginURL, roleID, secretID):
    
    # Logging into Vault using App Role Credentials
    locallogger = logging.getLogger(f'{__name__}.getHashicorpAppRoleToken')
    apiURL = f"{appRoleLoginURL}"
    headers = {'Accept': 'application/json'}
    data = {"role_id": roleID, "secret_id": secretID}
    # response = requests.post(apiURL, headers=headers, json=data, proxies=proxies, verify=False, timeout=120)
    response, dataPullStatus = getResponseFromURL(url=apiURL, method='POST', headers=headers,  proxies=None, json=data, verify=False, timeout=120, loggingPrefix=locallogger.name)
    if response.status_code != 200:
        locallogger.error(f"Login to Hashicorp Vault Failed!")
        sys.exit(1)
    jresponse = response.json()
    appRoleToken = jresponse['auth']['client_token']
    return appRoleToken






def getHashicorpSysMounts(hashicorpSysMountsURL, appRoleToken, pathDict=None):
    locallogger = logging.getLogger(f'{__name__}.getHashicorpSysMounts')
    apiURL = f"{hashicorpSysMountsURL}"
    headers = {'x-vault-token': f'{appRoleToken}'}

    response, _ = getResponseFromURL(url=apiURL, method='GET', headers=headers,  proxies=None, verify=False, timeout=120, loggingPrefix=locallogger.name)
    if response.status_code != 200:
        locallogger.error(f"Unable to get mount details from Hashicorp Vault!")
        sys.exit(1)
    jresponse = response.json()
    itemsList = []
    for key, value in jresponse.items():
        dictionary = {}
        dictionary['Hashicorp_Vault_Path'] = key
        if type(value) == dict:
            if 'description' in value.keys():
                dictionary['Description'] = value['description']
            if 'type' in value.keys():
                dictionary['Type'] = value['type']
        dictionary['Details'] = value
        itemsList.append(dictionary.copy())
    df = pd.DataFrame(data=itemsList)
    if pathDict != None:
        if 'Output_Folder' in pathDict.keys():
            jsonPretty = json.dumps(jresponse, indent=4, sort_keys=True)
            fileName = f"vault_mounts.json"
            filePath = os.path.join(os.path.normpath(pathDict['Output_Folder']), fileName)
            with open(filePath, 'w') as outfile:
                outfile.write(jsonPretty)
            locallogger.info(f"File written: {filePath}")
            writeExcel(dfList=[df], folderPath=pathDict['Output_Folder'], fileNamePrefix='Hashicorp_Sys_Mounts') 
    return (jresponse, df)






def getGCPSecretsFromHashicorp(hashcorpMainURL, gcpVaultPath, appRoleToken):
    locallogger = logging.getLogger(f'{__name__}.getGCPSecretsFromHashicorp')
    headers = {'x-vault-token': f'{appRoleToken}'}

    apiURL = f"{hashcorpMainURL}/{gcpVaultPath}token/folder_viewer"

    response, _ = getResponseFromURL(url=apiURL, method='GET', headers=headers,  proxies=None, verify=False, timeout=360, session=None, loggingPrefix=locallogger.name)

    if response.status_code != 200:
        locallogger.error(f"Unable to get GCP token for path: '{gcpVaultPath}'; response.status_code: {response.status_code}; response.text: {response.text}")
        sys.exit(1)
    jresponse = response.json()
    gcpToken = jresponse['data']['token']
    return gcpToken





def getAzureSecretsFromHashicorp(hashcorpMainURL, vaultPath, appRoleToken):
    locallogger = logging.getLogger(f'{__name__}.getAzureSecretsFromHashicorp')
    headers = {'x-vault-token': f'{appRoleToken}'}

    
    apiURL = f"{hashcorpMainURL}/{vaultPath}creds/rol-dynamic-24h-subscription-assuranceanalytics"


    response, _ = getResponseFromURL(url=apiURL, method='GET', headers=headers,  proxies=None, verify=False, timeout=360, session=None, loggingPrefix=locallogger.name)

    if response.status_code != 200:
        locallogger.error(f"Unable to get Azure token for path: '{vaultPath}'; response.status_code: {response.status_code}; response.text: {response.text}")
        sys.exit(1)
    jresponse = response.json()

    clientID = jresponse['data']['client_id']
    clientSecret = jresponse['data']['client_secret']
    
    return (clientID, clientSecret)









def getAzureServicePrincipalsFromHashicorpVault(hashicorpURL, hashicorpUserLoginURL, hashicorpRoleIDURL, hashicorpSecretIDURL, hashicorpAppRoleLoginURL, hashicorpSysMountURL, userID, password, pathDict, proxies, programName, senderEmail, recipientEmail):

    locallogger = logging.getLogger(f'{__name__}.getAzureServicePrincipalsFromHashicorpVault')
    

    timer = Timer()


    locallogger.info(f"Pulling Azure Service Principals from Hashicorp Vault")


    # Logging into Vault using AD Account
    apiURL = f"{hashicorpUserLoginURL}/{userID}"
    headers = {'Accept': 'application/json'}
    data = {"password": password}
    # response = requests.post(apiURL, headers=headers, json=data, proxies=proxies, verify=False, timeout=120)
    response, dataPullStatus = getResponseFromURL(url=apiURL, method='POST', headers=headers,  proxies=None, json=data, verify=False, timeout=120, loggingPrefix=locallogger.name)
    if response.status_code != 200:
        locallogger.error(f"Login to Hashicorp Vault Failed!")
        sys.exit(1)
    jresponse = response.json()
    vaultToken = jresponse['auth']['client_token']
    
    
    # Getting App role details - role_id
    apiURL = f'{hashicorpRoleIDURL}'
    headers = {'x-vault-token': f'{vaultToken}'}
    #response = requests.get(apiURL, headers=headers, proxies=proxies, verify=False, timeout=120)
    response, dataPullStatus = getResponseFromURL(url=apiURL, method='GET', headers=headers,  proxies=None, verify=False, timeout=120, loggingPrefix=locallogger.name)
    if response.status_code != 200:
        locallogger.error(f"Retrieval of app role details from Hashicorp Vault failed!")
        sys.exit(1)    
    jresponse = response.json()
    roleID = jresponse['data']['role_id']
    
    
    # Getting App role details - secret_id
    apiURL = f'{hashicorpSecretIDURL}'
    headers = {'x-vault-token': f'{vaultToken}'}
    #response = requests.get(apiURL, headers=headers, proxies=proxies, verify=False, timeout=120)
    response, dataPullStatus = getResponseFromURL(url=apiURL, method='POST', headers=headers,  proxies=None, verify=False, timeout=120, loggingPrefix=locallogger.name)
    if response.status_code != 200:
        locallogger.error(f"Retrieval of app role details from Hashicorp Vault failed!")
        sys.exit(1)    
    jresponse = response.json()
    secretID = jresponse['data']['secret_id']
    
    
    # Logging into Vault using App Role Credentials
    apiURL = f"{hashicorpAppRoleLoginURL}"
    headers = {'Accept': 'application/json'}
    data = {"role_id": roleID, "secret_id": secretID}
    # response = requests.post(apiURL, headers=headers, json=data, proxies=proxies, verify=False, timeout=120)
    response, dataPullStatus = getResponseFromURL(url=apiURL, method='POST', headers=headers,  proxies=None, json=data, verify=False, timeout=120, loggingPrefix=locallogger.name)
    if response.status_code != 200:
        locallogger.error(f"Login to Hashicorp Vault Failed!")
        sys.exit(1)
    jresponse = response.json()
    approleToken = jresponse['auth']['client_token']
    
    
    
    
    apiURL = f"{hashicorpSysMountURL}"
    headers = {'x-vault-token': f'{approleToken}'}
    # response = requests.get(apiURL, headers=headers, proxies=proxies, verify=False, timeout=120)
    response, dataPullStatus = getResponseFromURL(url=apiURL, method='GET', headers=headers,  proxies=None, verify=False, timeout=120, loggingPrefix=locallogger.name)
    if response.status_code != 200:
        locallogger.error(f"Unable to get mount details from Hashicorp Vault!")
        sys.exit(1)
    jresponse = response.json()
    jsonPretty = json.dumps(jresponse, indent=4, sort_keys=True)
    fileName = f"vault_mounts.json"
    filePath = os.path.join(os.path.normpath(pathDict['Output_Folder']), fileName)
    with open(filePath, 'w') as outfile:
        outfile.write(jsonPretty)
    vaultDf = pd.DataFrame()
    vaultDf['Hashicorp_Vault_Path'] = list(x for x in jresponse.keys() if 'azuresp' in x) 
    vaultDf['Description'] = vaultDf['Hashicorp_Vault_Path'].apply(lambda x: jresponse[x]['description'])

    azureSubscriptionDf = readExcel(filePath=pathDict['Azure_Subscription_Details'], sheetName='data')

    vaultDf = mergeDataframes(vaultDf, azureSubscriptionDf, onColumns_Left='Hashicorp_Vault_Path', \
        onColumns_Right='Hashicorp_Vault_Path', howMerge='outer', )
        
    
        
    vaultDf = replaceNullValuesInDataframe(df=vaultDf)

    # Remove the Out-of-Scope Subscrptions
    
    filterframe = (vaultDf['Subscription_ID'] == 'excluded-from-scope')
    vaultDf = vaultDf[~filterframe]



    
    writeExcel(dfList=[vaultDf], folderPath=pathDict['Output_Folder'], fileNamePrefix='Azure_Subscription_Details_In_Vault')
    
    


    if ('' in list(vaultDf['Subscription_ID'])) or ('' in list(vaultDf['Hashicorp_Vault_Path'])):
        locallogger.error(f"Empty Subscription_ID or Hashicorp_Vault_Path exists in Azure_Subscription_Details_In_Vault.xlsx file.  Please add correct details to Azure_Subscription_Details.xlsx file.")
        
        
        # Send Email Informing rows without Subscription_ID
        filterframe = (( vaultDf['Subscription_ID'] == '' ) | ( vaultDf['Hashicorp_Vault_Path'] == '' ))
        vaultDf2 = vaultDf[filterframe]
        filePath = os.path.join(pathDict['Output_Folder'], f'Azure_Subscription_Paths_With_No_Subscription_ID_Empty_Vault_Path.xlsx')
        writeExcel(dfList=[vaultDf2], filePath=filePath)

        attachmentPathList = [filePath]

        subject = f"Azure Subscription Paths With No Subscription ID on File or Empty Vault Path"

        body = f"""
            <html>
            <body>
            Hi,<br><br>
            There were Azure Subscription Paths With no Subscription ID on File.  Please refer attached excel.
            <br><br>
            Program Name: <b>{programName}</b>
            <br><br>
            Thank you.
            <br><br>
            <i>
            Best Regards <br>
            </i>
            </body>
            </html>
        """

        senderEmail = senderEmail

        recipientEmail = recipientEmail


        sendEmail(subject=subject, body=body, senderEmail=senderEmail, recipientEmail=recipientEmail, bccRecipientEmail=None, ccRecipientEmail=None, attachmentPathList=attachmentPathList, loggingPrefix=locallogger.name, testing=False)
        
        
        
        #sys.exit(1)
        
        vaultDf = vaultDf[~filterframe]



    thread_local = threading.local()
    vaultDf['Client_ID'] = ''
    vaultDf['Client_Secret'] = ''
    vaultDf['Successfully_Pulled_Credentials'] = ''
    vaultDf['Error_Details'] = ''

    def get_session():
        if not hasattr(thread_local, "session"):
            thread_local.session = requests.Session()
        return thread_local.session


    def getHashicorpAzureServicePrincipals(hashicorpURL, hashicorp_vault_path):
        locallogger = logging.getLogger(f'{__name__}.getHashicorpAzureServicePrincipals')
        
        session = get_session()
        headers = {'x-vault-token': f'{approleToken}'}
        apiURL = f"{hashicorpURL}/{hashicorp_vault_path}creds/rol-dynamic-24h-subscription-assuranceanalytics"
        # response = session.get(apiURL, headers=headers, proxies=proxies, verify=False, timeout=120)
        response, dataPullStatus = getResponseFromURL(url=apiURL, method='GET', headers=headers,  proxies=None, verify=False, timeout=120, session=session, loggingPrefix=locallogger.name)
        # response, dataPullStatus = getResponseFromURL(url=apiURL, method='GET', headers=headers,  proxies=None, verify=False, timeout=120, session=session, loggingPrefix=locallogger.name)
        if response.status_code != 200:
            vaultDf.loc[vaultDf['Hashicorp_Vault_Path'] == hashicorp_vault_path, ['Successfully_Pulled_Credentials']] = False
            vaultDf.loc[vaultDf['Hashicorp_Vault_Path'] == hashicorp_vault_path, ['Error_Details']] = f"response.status_code: {response.status_code}; response.text: {response.text}"
            locallogger.error(f"hashicorp_vault_path: {hashicorp_vault_path}; response.status_code: {response.status_code}; response.text: {response.text}")
        else:
            vaultDf.loc[vaultDf['Hashicorp_Vault_Path'] == hashicorp_vault_path, ['Successfully_Pulled_Credentials']] = True
            jresponse = response.json()
            clientID = jresponse['data']['client_id']
            clientSecret = jresponse['data']['client_secret']
            #locallogger.info(f"hashicorp_vault_path: {hashicorp_vault_path}; clientID: {clientID}; clientSecret: {clientSecret}")
            vaultDf.loc[vaultDf['Hashicorp_Vault_Path'] == hashicorp_vault_path, ['Client_ID']] = clientID
            vaultDf.loc[vaultDf['Hashicorp_Vault_Path'] == hashicorp_vault_path, ['Client_Secret']] = clientSecret



    # Removing subscriptions which are excluded from scope
    filterframe = ( vaultDf['Subscription_ID'] != 'excluded-from-scope' )
    vaultDf = vaultDf[filterframe]


    with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
        executor.map(getHashicorpAzureServicePrincipals, list(vaultDf['Hashicorp_Vault_Path']))


    selectedColumns = list(x for x in vaultDf.columns if x not in ['Client_ID', 'Client_Secret'])


    writeExcel(dfList=[vaultDf[selectedColumns]], folderPath=pathDict['Output_Folder'], fileNamePrefix='Azure_Subscription_Details_In_Vault') 

    locallogger.info(f"----------------------------------------------------------------------------------")
    locallogger.info(f"Got Azure Service Principals from Hashicorp.")
    locallogger.info(f"Successfully got credentials for {len(vaultDf[(vaultDf['Successfully_Pulled_Credentials'] == True)])} subscriptions.")
    locallogger.info(f"Could not get credentials for {len(vaultDf[(vaultDf['Successfully_Pulled_Credentials'] == False)])} Azure Subscriptions.")
    processingTime = timer.getTimerTime()
    locallogger.info(f"Total processing time : {processingTime}")
    locallogger.info(f"----------------------------------------------------------------------------------")
    time.sleep(visibilityTimeWindow)

    return vaultDf







# Function definition for the collection of Azure SPNs

def getAzureSPNs(appRoleToken, pathDict, factor1, factor2, programName, senderEmail, recipientEmail):
    locallogger = logging.getLogger(f'{__name__}.getAzureSPNs')
    
    today_date = datetime.datetime.now().strftime('%d-%b-%Y')
    
    stateDict = getStageState(pathDict=pathDict, stageUniqueName='getAzureSPNs', locallogger=locallogger)
    
    if 'timeOfSPNCollection' in stateDict.keys():
        timeOfSPNCollection = stateDict['timeOfSPNCollection']
        timeNow = datetime.datetime.now()
        timeDelta = datetime.timedelta(hours=20)
        ceilingTime = timeOfSPNCollection + timeDelta
        
        if timeNow < ceilingTime:
            vaultDf2 = stateDict['vaultDf2']
            vaultDf = decryptDataFrame(df=vaultDf2, columnsToDecryptList=['Client_ID', 'Client_Secret',], passphrase=factor1, salt=factor2)
            locallogger.info(f"Retriving vaultDf from local storage....")
            return vaultDf
    
    (sysMountsJSON, sysMountsDf) = getHashicorpSysMounts(appRoleToken=appRoleToken, pathDict=pathDict)
    
    filterframe = ( sysMountsDf['Type'] == 'azure' )
    df1 = sysMountsDf[filterframe]
    

    df2 = df1

    def getAzureSubscriptionName(vaultPath):
        stringSplitList = vaultPath.split('/')
        subscriptionName = f"{stringSplitList[1]}-{stringSplitList[2]}-{stringSplitList[3]}"
        return subscriptionName
    
    df2['Azure_Subscription_Name'] = df2['Hashicorp_Vault_Path'].apply(lambda x: getAzureSubscriptionName(x))
    
    writePickle(df2, folderPath=pathDict['Output_Folder'], fileNamePrefix='Azure_Subscription_Details_In_Hashicorp')
    writeExcel(dfList=[df2], folderPath=pathDict['Output_Folder'], fileNamePrefix=f'Azure_Subscription_Details_In_Hashicorp_{today_date}')  
        
    df1 = readPickle(folderPath=pathDict['Output_Folder'], fileNamePrefix='Azure_Subscription_Details_In_Hashicorp')
    
    df2 = readExcel(folderPath=pathDict['Common_Folder'], fileNamePrefix='Azure_Subscription_Details')
    
    selectedColumns = [
        'Hashicorp_Vault_Path', 
        'Subscription_ID', 
        'Tenant_ID',
        'Subscription_In_Scope',
        ]
    
    df2 = df2[selectedColumns]
    
    df3 = mergeDataframes(dataframe1=df1, dataframe2=df2, onColumns_Left=['Hashicorp_Vault_Path'], onColumns_Right=['Hashicorp_Vault_Path'], howMerge='left', ensureNoDuplicatesOnRightForLeftMerge=True, replaceNullValuesInOutput=True)
    
    
    # Remove the Out-of-Scope Subscrptions
    

    
    
    
    
    writePickle(df3, folderPath=pathDict['Output_Folder'], fileNamePrefix='Azure_Subscription_Details_Reconciled')
    writeExcel(dfList=[df3], folderPath=pathDict['Output_Folder'], fileNamePrefix=f'Azure_Subscription_Details_Reconciled_{today_date}')
    
    filterframe = (df3['Subscription_In_Scope'] == 'No')
    df3 = df3[~filterframe]
    
    if '' in list(df3['Subscription_ID']):
        locallogger.error(f"There are Hashicorp Vault Azure Paths with no associated Subscription ID on file!")
        
        
        # Send Email Informing rows without Subscription_ID
        filterframe = ( df3['Subscription_ID'] == '' )
        df4 = df3[filterframe]
        filePath = os.path.join(pathDict['Output_Folder'], f'Azure_Subscription_Paths_With_No_Subscription_ID_{today_date}.xlsx')
        writeExcel(dfList=[df4], filePath=filePath)

        attachmentPathList = [filePath]

        subject = f"Azure Subscription Paths With No Subscription ID on File"

        body = f"""
            <html>
            <body>
            Hi,<br><br>
            There were Azure Subscription Paths With no Subscription ID on File.  Please refer attached excel.
            <br><br>
            Program Name: <b>{programName}</b>
            <br><br>
            Thank you.
            <br><br>
            <i>
            Best Regards<br>
            </i>
            </body>
            </html>
        """

        senderEmail = senderEmail

        recipientEmail = recipientEmail


        sendEmail(subject=subject, body=body, senderEmail=senderEmail, recipientEmail=recipientEmail, bccRecipientEmail=None, ccRecipientEmail=None, attachmentPathList=attachmentPathList, loggingPrefix=locallogger.name, testing=False)
        #sys.exit(1)
        
    locallogger.info(f"Pulling Azure Service Principals from Hashicorp Vault")
    
    vaultDf = df3[~filterframe]

    thread_local = threading.local()
    vaultDf['Client_ID'] = ''
    vaultDf['Client_Secret'] = ''
    vaultDf['Successfully_Pulled_Credentials_from_Hashicorp'] = ''
    vaultDf['Error_Details_Hashicorp'] = ''

    def get_session():
        if not hasattr(thread_local, "session"):
            thread_local.session = requests.Session()
        return thread_local.session


    def getHashicorpAzureServicePrincipals(hashicorpURL, hashicorp_vault_path):
        locallogger = logging.getLogger(f'{__name__}.getHashicorpAzureServicePrincipals')
        
        session = get_session()
        headers = {'x-vault-token': f'{appRoleToken}'}

        apiURL = f"{hashicorpURL}/{hashicorp_vault_path}creds/rol-dynamic-24h-subscription-assuranceanalytics"

        response, dataPullStatus = getResponseFromURL(url=apiURL, method='GET', headers=headers,  proxies=None, verify=False, timeout=120, session=session, loggingPrefix=locallogger.name)

        if response.status_code != 200:
            vaultDf.loc[vaultDf['Hashicorp_Vault_Path'] == hashicorp_vault_path, ['Successfully_Pulled_Credentials_from_Hashicorp']] = False
            vaultDf.loc[vaultDf['Hashicorp_Vault_Path'] == hashicorp_vault_path, ['Error_Details_Hashicorp']] = f"response.status_code: {response.status_code}; response.text: {response.text}"
            locallogger.error(f"hashicorp_vault_path: {hashicorp_vault_path}; response.status_code: {response.status_code}; response.text: {response.text}")
        else:
            vaultDf.loc[vaultDf['Hashicorp_Vault_Path'] == hashicorp_vault_path, ['Successfully_Pulled_Credentials_from_Hashicorp']] = True
            jresponse = response.json()
            clientID = jresponse['data']['client_id']
            clientSecret = jresponse['data']['client_secret']
            #locallogger.info(f"hashicorp_vault_path: {hashicorp_vault_path}; clientID: {clientID}; clientSecret: {clientSecret}")
            vaultDf.loc[vaultDf['Hashicorp_Vault_Path'] == hashicorp_vault_path, ['Client_ID']] = clientID
            vaultDf.loc[vaultDf['Hashicorp_Vault_Path'] == hashicorp_vault_path, ['Client_Secret']] = clientSecret


    with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
        executor.map(getHashicorpAzureServicePrincipals, list(vaultDf['Hashicorp_Vault_Path']))


    timeOfSPNCollection = datetime.datetime.now()
    
    vaultDf2 = vaultDf.copy()

    vaultDf2 = encryptDataFrame(df=vaultDf2, columnsToEncryptList=['Client_ID', 'Client_Secret',], passphrase=factor1, salt=factor2)


    writePickle(vaultDf2, folderPath=pathDict['Output_Folder'], fileNamePrefix='Azure_Subscription_Details_In_Vault')
    writeExcel(dfList=[vaultDf2], folderPath=pathDict['Output_Folder'], fileNamePrefix=f'Azure_Subscription_Details_In_Vault_{today_date}') 

    


    locallogger.info(f"----------------------------------------------------------------------------------")
    locallogger.info(f"Got Azure Service Principals from Hashicorp.")
    numberOfSuccessfulRetrival = len(vaultDf[(vaultDf['Successfully_Pulled_Credentials_from_Hashicorp'] == True)])
    locallogger.info(f"Successfully got credentials for '{numberOfSuccessfulRetrival}' subscriptions.")
    numberOfFailedRetrival = len(vaultDf[(vaultDf['Successfully_Pulled_Credentials_from_Hashicorp'] == False)])
    if numberOfFailedRetrival > 0:
        locallogger.error(f"Could not get credentials for {numberOfFailedRetrival} Azure Subscriptions.")
        
        # Send Email with failure details
        filterframe = ( vaultDf2['Successfully_Pulled_Credentials_from_Hashicorp'] == False )
        vaultDf3 = vaultDf2[filterframe]
        filePath = os.path.join(pathDict['Output_Folder'], f'Azure_Subscriptions_In_Vault_Error_{today_date}.xlsx')
        writeExcel(dfList=[vaultDf3], filePath=filePath)

        attachmentPathList = [filePath]

        subject = f"Azure Subscription Details Pull from Hashicorp Vault - Errors were encountered"

        body = f"""
            <html>
            <body>
            Hi,<br><br>
            Errors were encountered in pulling Azure Subscription Details from Hashicorp Vault.  Please refer attached excel.
            <br><br>
            Program Name: <b>{programName}</b>
            <br><br>
            Thank you.
            <br><br>
            <i>
            Best Regards<br>
            </i>
            </body>
            </html>
        """

        senderEmail = senderEmail

        recipientEmail = recipientEmail


        sendEmail(subject=subject, body=body, senderEmail=senderEmail, recipientEmail=recipientEmail, bccRecipientEmail=None, ccRecipientEmail=None, attachmentPathList=attachmentPathList, loggingPrefix=locallogger.name, testing=False)

        
        
        
        #sys.exit(1)
    locallogger.info(f"----------------------------------------------------------------------------------")
    
    stateDict = {
            'vaultDf2' : vaultDf2,
            'timeOfSPNCollection' : timeOfSPNCollection,
        }
    
    setStageState(stateDict=stateDict, pathDict=pathDict, stageUniqueName='getAzureSPNs', locallogger=locallogger)

    filterframe = ( vaultDf['Successfully_Pulled_Credentials_from_Hashicorp'] == True )
    vaultDf4 = vaultDf[filterframe]

    return vaultDf4



















def getAzureServicePrincipalFromKeyFile(pathDict, passphrase, salt):
    
    decryptFile(pathDict['Azure_Keys_Encrypted'], pathDict['Azure_Keys_Decrypted'], passphrase, salt)
    
    #encryptFile(pathDict['Azure_Keys_Decrypted'], pathDict['Azure_Keys_Encrypted'], passphrase, salt)
    

    # Create dataframe from azureKeysDecryptedFile
    azureKeyDf = readExcel(filePath=pathDict['Azure_Keys_Decrypted'], sheetName='Sheet1')

    
    os.remove(pathDict['Azure_Keys_Decrypted'])

    return azureKeyDf



# ---------------------------------------------------------------------------------------------------------------- 
#%%

# Function definition for the collection of GCP SPNs

def getGcpSPNs(appRoleToken, pathDict, factor1, factor2, programName, senderEmail, recipientEmail):
    locallogger = logging.getLogger(f'{__name__}.getGcpSPNs')
    
    today_date = datetime.datetime.now().strftime('%d-%b-%Y')
    
    stateDict = getStageState(pathDict=pathDict, stageUniqueName='getGcpSPNs', locallogger=locallogger)
    
    if 'timeOfSPNCollection' in stateDict.keys():
        timeOfSPNCollection = stateDict['timeOfSPNCollection']
        timeNow = datetime.datetime.now()
        timeDelta = datetime.timedelta(hours=1)
        ceilingTime = timeOfSPNCollection + timeDelta
        
        if timeNow < ceilingTime:
            vaultDf2 = stateDict['vaultDf2']
            vaultDf = decryptDataFrame(df=vaultDf2, columnsToDecryptList=['GCP_Token',], passphrase=factor1, salt=factor2)
            locallogger.info(f"Retriving vaultDf from local storage....")
            return vaultDf
    
    (sysMountsJSON, sysMountsDf) = getHashicorpSysMounts(appRoleToken=appRoleToken, pathDict=pathDict)
    
    filterframe = ( sysMountsDf['Type'] == 'gcp' )
    
    df1 = sysMountsDf[filterframe]
    
    filterframe = ( df1['Hashicorp_Vault_Path'].str.endswith('/master/') )
    
    df2 = df1[filterframe]
    
    
    def getGCPEnvironment(gcpVaultPath):
        stringSplitList = gcpVaultPath.split('/')
        environmentName = f"{stringSplitList[1]}-{stringSplitList[2]}"
        return environmentName
    
    
    df2['GCP_Environment'] = df2['Hashicorp_Vault_Path'].apply(lambda x: getGCPEnvironment(x))
    
    
    
    writePickle(df2, folderPath=pathDict['Output_Folder'], fileNamePrefix='GCP_Environment_Details_In_Hashicorp')
    writeExcel(dfList=[df2], folderPath=pathDict['Output_Folder'], fileNamePrefix=f'GCP_Environment_Details_In_Hashicorp_{today_date}')  

    
    # ---------- 

    
    
    vaultDf = df2
      

    thread_local = threading.local()

    vaultDf['GCP_Token'] = ''
    vaultDf['Successfully_Pulled_Credentials_from_Hashicorp'] = ''
    vaultDf['Error_Details_Hashicorp'] = ''

    def get_session():
        if not hasattr(thread_local, "session"):
            thread_local.session = requests.Session()
        return thread_local.session


    def getHashicorpGCPServicePrincipalsInternalFunction(hashicorpURL, hashicorp_vault_path):
        locallogger = logging.getLogger(f'{__name__}.getHashicorpGCPServicePrincipalsInternalFunction')
        
        # ------------
        
        session = get_session()
        headers = {'x-vault-token': f'{appRoleToken}'}

        apiURL = f"{hashicorpURL}/{hashicorp_vault_path}token/folder_viewer"
        


        response, dataPullStatus = getResponseFromURL(url=apiURL, method='GET', headers=headers,  proxies=None, verify=False, timeout=360, session=session, loggingPrefix=locallogger.name)

        if response.status_code != 200:
            vaultDf.loc[vaultDf['Hashicorp_Vault_Path'] == hashicorp_vault_path, ['Successfully_Pulled_Credentials_from_Hashicorp']] = False
            vaultDf.loc[vaultDf['Hashicorp_Vault_Path'] == hashicorp_vault_path, ['Error_Details_Hashicorp']] = f"response.status_code: {response.status_code}; response.text: {response.text}"
            locallogger.error(f"hashicorp_vault_path: {hashicorp_vault_path}; response.status_code: {response.status_code}; response.text: {response.text}")
        else:
            vaultDf.loc[vaultDf['Hashicorp_Vault_Path'] == hashicorp_vault_path, ['Successfully_Pulled_Credentials_from_Hashicorp']] = True
            jresponse = response.json()
            gcpToken = jresponse['data']['token']
            #locallogger.info(f"hashicorp_vault_path: {hashicorp_vault_path}; gcpToken: {gcpToken}")
            vaultDf.loc[vaultDf['Hashicorp_Vault_Path'] == hashicorp_vault_path, ['GCP_Token']] = gcpToken



    with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
        executor.map(getHashicorpGCPServicePrincipalsInternalFunction, list(vaultDf['Hashicorp_Vault_Path']))


    timeOfSPNCollection = datetime.datetime.now()
    
    vaultDf2 = vaultDf.copy()

    vaultDf2 = encryptDataFrame(df=vaultDf2, columnsToEncryptList=['GCP_Token',], passphrase=factor1, salt=factor2)


    writePickle(vaultDf2, folderPath=pathDict['Output_Folder'], fileNamePrefix='GCP_Project_Details_In_Vault')
    writeExcel(dfList=[vaultDf2], folderPath=pathDict['Output_Folder'], fileNamePrefix=f'GCP_Project_Details_In_Vault_{today_date}') 



    locallogger.info(f"----------------------------------------------------------------------------------")
    locallogger.info(f"GCP Service Principals from Hashicorp.")
    numberOfSuccessfulRetrival = len(vaultDf[(vaultDf['Successfully_Pulled_Credentials_from_Hashicorp'] == True)])
    locallogger.info(f"Successfully got credentials for '{numberOfSuccessfulRetrival}' GCP Environments.")
    numberOfFailedRetrival = len(vaultDf[(vaultDf['Successfully_Pulled_Credentials_from_Hashicorp'] == False)])
    if numberOfFailedRetrival > 0:
        locallogger.error(f"Could not get credentials for {numberOfFailedRetrival} GCP Environments.")
        
        # Send Email with failure details
        filterframe = ( vaultDf2['Successfully_Pulled_Credentials_from_Hashicorp'] == False )
        vaultDf3 = vaultDf2[filterframe]
        filePath = os.path.join(pathDict['Output_Folder'], f'GCP_Environments_In_Vault_Error_{today_date}.xlsx')
        writeExcel(dfList=[vaultDf3], filePath=filePath)

        attachmentPathList = [filePath]

        subject = f"GCP Environment Details Pull from Hashicorp Vault - Errors were encountered"

        body = f"""
            <html>
            <body>
            Hi,<br><br>
            Errors were encountered in pulling GCP Environment Details from Hashicorp Vault.  Please refer attached excel.
            <br><br>
            Program Name: <b>{programName}</b>
            <br><br>
            Thank you.
            <br><br>
            <i>
            Best Regards, <br>
            E2EMA Program<br>
            </i>
            </body>
            </html>
        """

        senderEmail = senderEmail

        recipientEmail = recipientEmail


        sendEmail(subject=subject, body=body, senderEmail=senderEmail, recipientEmail=recipientEmail, bccRecipientEmail=None, ccRecipientEmail=None, attachmentPathList=attachmentPathList, loggingPrefix=locallogger.name, testing=False)

        
        
        
        #sys.exit(1)
    locallogger.info(f"----------------------------------------------------------------------------------")
    
    stateDict = {
            'vaultDf2' : vaultDf2,
            'timeOfSPNCollection' : timeOfSPNCollection,
        }
    
    setStageState(stateDict=stateDict, pathDict=pathDict, stageUniqueName='getGcpSPNs', locallogger=locallogger)

    filterframe = ( vaultDf['Successfully_Pulled_Credentials_from_Hashicorp'] == True )
    vaultDf4 = vaultDf[filterframe]

    return vaultDf4





#%%


def formatFilename(inputString):
    """Take a string and return a valid filename constructed from the string.
Uses a whitelist approach: any characters not present in valid_chars are
removed. Also spaces are replaced with underscores.
 
Note: this method may produce invalid filenames such as ``, `.` or `..`
When I use this method I prepend a date string like '2009_01_15_19_46_32_'
and append a file extension like '.txt', so I avoid the potential of using
an invalid filename.
 
"""
    valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
    filename = ''.join(c for c in inputString if c in valid_chars)
    filename = filename.replace(' ','_') # I don't like spaces in filenames.
    return filename


def confirmWholeNumber(s):
    try:
        int(s)
    except ValueError:
        return False
    return True


# Function to pull details of Firewall Rules from a particular Azure Subscription
def pullRulesFromSubscription(subscription_name, subscription_id, authHeader, proxies, programName, pathDict):
    locallogger = logging.getLogger(f'{__name__}.pullRulesFromSubscription')
    
    local_df = pd.DataFrame()

    if programName in ['azure_firewall', 'azure_firewall_jackson']:
        apiURL = f"https://management.azure.com/subscriptions/{subscription_id}/providers/Microsoft.Network/azureFirewalls?api-version=2018-11-01"
    elif programName in ['azure_nsg', 'azure_nsg_uk']:
        apiURL = f"https://management.azure.com/subscriptions/{subscription_id}/providers/Microsoft.Network/networkSecurityGroups?api-version=2018-11-01"
    else:
        locallogger.error(f"programName: {programName} is unrecognized.")
        sys.exit(1)
    while True:
        locallogger.debug(f'Subscription Name:{subscription_name}; Making request to URL: {apiURL}')
        response, dataPullStatus = getResponseFromURL(url=apiURL, method='GET', headers=authHeader, proxies=proxies, verify=False, timeout=120, loggingPrefix=locallogger.name)
        
        if dataPullStatus == 'Successful':
            jresponse = response.json()
            jsonPretty = json.dumps(jresponse, indent=4, sort_keys=True)
            fileName = f"response_{subscription_name}_{subscription_id}.json"
            filePath = os.path.join(os.path.normpath(pathDict['Output_Folder']), fileName)
            locallogger.debug(f"Writing file: {filePath}")
            with open(filePath, 'w') as outfile:
                outfile.write(jsonPretty)
            do_df = json_normalize(jresponse['value'])
            local_df = pd.concat([local_df, do_df], axis=0, ignore_index=True, sort=False)
            locallogger.debug(f"local_df.shape = {local_df.shape}")
            if 'nextLink' in jresponse:
                apiURL = jresponse['nextLink']
            else:
                break
        else:
            locallogger.debug(f"authHeader: {authHeader}")
            break
    if (len(local_df) > 0):
        local_df['Subscription_Name'] = subscription_name
    return local_df, dataPullStatus, response.text











def pull_raw_rule_details(vaultDf, pathDict, proxies, programName):
    
    locallogger = logging.getLogger(f'{__name__}.pull_raw_rule_details')
        
    
    timer = Timer()
    
    locallogger.info(f"Pulling Raw Rule Details from Azure")


    # Main thread to pull data from Azure API
    df = pd.DataFrame()
    failedAzureSubscriptionDf = pd.DataFrame()
    vaultDf['Data_Pull_Status'] = ''
    vaultDf['Error_Details'] = ''
    for index, row in vaultDf.iterrows():
        if programName == 'azure_nsg':
            if row['Subscription_Name'] == 'enterprise-tenant':
                continue
        dataPullStatus = 'Successful'
        errordetails = ''
        azureAuthHeader = getAzureAccessToken(client_id=row['Client_ID'], client_secret=row['Client_Secret'], tenant_id=row['Tenant_ID'], subscription_name=row['Subscription_Name'], proxies=proxies)
        if azureAuthHeader != '': # Got valid Authentication Header
            do_df, dataPullStatus, errordetails = pullRulesFromSubscription(subscription_name=row['Subscription_Name'], subscription_id=row['Subscription_ID'], authHeader=azureAuthHeader, proxies=proxies, programName=programName, pathDict=pathDict)
            vaultDf.loc[index, 'Data_Pull_Status'] = dataPullStatus
            vaultDf.loc[index, 'Error_Details'] = errordetails
            df = pd.concat([df, do_df], axis=0, ignore_index=True, sort=False)
            locallogger.debug(f"df.shape = {df.shape}")
        else:
            vaultDf.loc[index, 'Data_Pull_Status'] = 'Azure Authentication Failed'



    locallogger.info(f"Completed rule extraction from all azure subscritions.")


    writePickle(df, folderPath=pathDict['Output_Folder'], fileNamePrefix='Firewall_Rules_Raw_File')
    writeExcel(dfList=[df], folderPath=pathDict['Output_Folder'], fileNamePrefix='Firewall_Rules_Raw_File')    


    filterframe = (vaultDf['Data_Pull_Status'] != 'Successful' )
    failedAzureSubscriptionDf = vaultDf[filterframe]
    selectedColumns = list(x for x in vaultDf.columns if x not in ['Client_ID', 'Client_Secret'])
    failedAzureSubscriptionDf = failedAzureSubscriptionDf[selectedColumns]
    locallogger.debug(f"Shape of failedAzureSubscriptionDf: {failedAzureSubscriptionDf.shape}")
    if (len(failedAzureSubscriptionDf) > 0 ):
        locallogger.warning(f"Failed pulling rule details for {len(failedAzureSubscriptionDf)} subscriptions.")
        writeExcel(dfList=[failedAzureSubscriptionDf], folderPath=pathDict['Output_Folder'], fileNamePrefix='Failed_Azure_Tenant_Details') 


    numberOfSubscriptionsCheckedSuccessfully = len(vaultDf[~filterframe])


    locallogger.info(f"----------------------------------------------------------------------------------")
    locallogger.info(f"Pulling Raw Rule Details from Azure API.")
    locallogger.info(f"Collected Rule information from {numberOfSubscriptionsCheckedSuccessfully} Azure Subscriptions successfully.")
    locallogger.info(f"Could not get information from {len(failedAzureSubscriptionDf)} Azure Subscriptions.")
    processingTime = timer.getTimerTime()
    locallogger.info(f"Total processing time : {processingTime}")
    locallogger.info(f"----------------------------------------------------------------------------------")














def getPasswordFromCyberark(cyberarkAppID, cyberarkSafeName, cyberarkObjectName):
    locallogger = logging.getLogger(f'{__name__}.getPasswordFromCyberark')
        

    data = {
    	"appID": cyberarkAppID,
    	"safe": cyberarkSafeName,
    	"objectName": cyberarkObjectName
    }
    	
    headers = {'Content-Type': 'application/json',}
    
    
        
    url = f'https://cybraam.intranet.asia/api/RetrieveCredential'
    response, dataPullStatus = getResponseFromURL(url=url, method='POST', headers=headers, data=json.dumps(data), verify=False, timeout=120, loggingPrefix=locallogger.name)
    responsecode = response.status_code
    jresponse = response.json()
    if (responsecode != 200):
        locallogger.error(f"Error while withdrawing Active Directory acount password from Cyberark.  Responsecode: '{responsecode}'; JSON Response: '{jresponse}'.")
        sys.exit(1)
    elif 'Message' in jresponse.keys():
        locallogger.error(f"Error while withdrawing Active Directory acount password from Cyberark.  Responsecode: '{responsecode}'; JSON Response: '{jresponse}'.")
        sys.exit(1)
    elif 'errorMessage' in jresponse.keys():
        if jresponse['errorMessage'] != '' :
            locallogger.error(f"Error while withdrawing Active Directory acount password from Cyberark.  Responsecode: '{responsecode}'; JSON Response: '{jresponse}'.")
            sys.exit(1)
            
    return jresponse['content']




def readSecretFromStash(name):
    filePath = f"/usr/secret-stash/{name}"
    value = readFile(filePath=filePath, mode='r')
    return value


def getSecret(name):
    value = getParameter(name)
    if value == False:
        value = readSecretFromStash(name)
    return value


def seekManualAcknowledgement():
    inputValue = input(f"Running in Production Mode!  Did you intend this?  :  ")
    if inputValue in ["Yes", "yes"]:
        return True
    sys.exit(1)
    return False









#%%
    
# (cyberarkAuditID, cyberarkAuditPassword) = commonfunctions.core.getServiceAccount1Creds(userID=cyberarkAuditID, cyberarkAppID=cyberarkAuditAccountApplicationID, cyberarkSafeName=cyberarkAuditAccountSafeName, cyberarkObjectName=cyberarkAuditAccountObjectName)


def getServiceAccount1Creds(userID=None, cyberarkAppID=None, cyberarkSafeName=None, cyberarkObjectName=None):
    locallogger = logging.getLogger(f'{__name__}.getServiceAccount1Creds')

    if userID == None:
        userID = getSecret('AD_USER_NAME')
    
    if cyberarkAppID == None:
        cyberarkAppID = getSecret('CYBERARK_APPLICATION_ID')
    
    
    if cyberarkSafeName == None:
        cyberarkSafeName = getSecret('CYBERARK_SAFE_NAME')
    
    if cyberarkObjectName == None:
        cyberarkObjectName = getSecret('CYBERARK_OBJECT_NAME')

    password = getPasswordFromCyberark(cyberarkAppID, cyberarkSafeName, cyberarkObjectName)
    return (userID, password)


#%%

def getSecretDict(vaultURL, userID=None, password=None):

    locallogger = logging.getLogger(f'{__name__}.getSecretDict')

    if (userID == None) and (password == None):
        userID = getSecret('AD_USER_NAME')
        cyberarkAppID = getSecret('CYBERARK_APPLICATION_ID')
        cyberarkSafeName = getSecret('CYBERARK_SAFE_NAME')
        cyberarkObjectName = getSecret('CYBERARK_OBJECT_NAME')
    
        password = getPasswordFromCyberark(cyberarkAppID, cyberarkSafeName, cyberarkObjectName)
    


    proxies = determineProxies(userID=userID, password=password, url="{vaultURL}")
    secretDict = getSecretsFromHashicorpVault(userID, password, proxies)
    

    return (userID, password, secretDict)




def onboardKeyAndValueToHashicorpVault(key, value, vaultUserLoginURL, kvURL):

    locallogger = logging.getLogger(f'{__name__}.onboardKeyAndValueToHashicorpVault')
    
    
    outcome = "Onboarding_Failed"

    (userID, password)  = getServiceAccount1Creds()
    proxies = getProxies(userID, password)


    apiURL = f"{vaultUserLoginURL}/{userID}"
    headers = {'Accept': 'application/json'}
    data = {"password": password}
    
    response, _ = getResponseFromURL(url=apiURL, method='POST', headers=headers,  proxies=proxies, json=data, verify=False, timeout=600, loggingPrefix=locallogger.name)
    if response.status_code != 200:
        locallogger.error(f"Login to Hashicorp Vault Failed!")
        sys.exit(1)
    jresponse = response.json()
    vaultToken = jresponse['auth']['client_token']
    
    
    
    
    apiURL = f"{kvURL}"
    headers = {'x-vault-token': f'{vaultToken}'}
    
    response, dataPullStatus = getResponseFromURL(url=apiURL, method='GET', headers=headers,  proxies=proxies, verify=False, timeout=120, loggingPrefix=locallogger.name)
    if response.status_code != 200:
        locallogger.error(f"Retrieval of secrets from Hashicorp Vault failed!")
        sys.exit(1)    
    jresponse = response.json()
    secretsDict = jresponse['data']['data']
    
    if key in secretsDict.keys() and key != 'TEST':
        outcome = "Key_Already_Exists"
        return outcome
        
    

    paraDict = {key : value}
    secretsDict.update(paraDict)
    
    payloadDict = { 'data' : secretsDict}

    response, dataPullStatus = getResponseFromURL(url=apiURL, method='POST', headers=headers, json=payloadDict,  proxies=proxies, verify=False, timeout=120, loggingPrefix=locallogger.name)
    
    # Verify

    response, dataPullStatus = getResponseFromURL(url=apiURL, method='GET', headers=headers,  proxies=None, verify=False, timeout=120, loggingPrefix=locallogger.name)
    if response.status_code != 200:
        locallogger.error(f"Retrieval of secrets from Hashicorp Vault failed!")
        sys.exit(1)    
    jresponse = response.json()
    secretsDict2 = jresponse['data']['data']
    
    if key in secretsDict2.keys() and value == secretsDict2[key]:
        outcome = "Onboarding_Succeeded"
    
    if outcome != "Onboarding_Succeeded":
        locallogger.error(f"Error while onboarding new secret to Hashicorp Vault!")
    
    return outcome






def getProxies(userID, password, proxyNum=1, posixNeedsProxy=False):
    
    #password = urllib.parse.quote(password)
    password = urllib.parse.quote(password, safe='')
    
    if proxyNum == 1:
        proxyIPAndPort = '10.163.39.77:8080'
    elif proxyNum == 2:
        proxyIPAndPort = '10.163.38.22:8080'
    elif proxyNum == 3:
        proxyIPAndPort = '10.163.39.242:8080'
    
    
    
    proxy = {
            'http': f'http://{userID}:{password}@{proxyIPAndPort}', 
            'https': f'https://{userID}:{password}@{proxyIPAndPort}', 
        }
    
    proxies = proxy

    if (os.name == 'posix'):
        if (posixNeedsProxy == False):
            proxies = None


    return proxies




def determineProxies(userID, password, url, loggingPrefix=None, clearProxyInEnvironment=True, setProxyInEnvironment=False):
    
    locallogger = getLocalLogger(localloggername='determineProxies', rootFileName=__name__, loggingPrefix=loggingPrefix)
    
    httpProxyValue = None
    httpsProxyValue = None
    proxiesReturnValue = None
    keyValue = None
    
    if "HTTP_PROXY" in os.environ.keys():
        httpProxyValue = os.environ["HTTP_PROXY"]
        os.environ["HTTP_PROXY"] = ""
    
    if "HTTPS_PROXY" in os.environ.keys():
        httpsProxyValue = os.environ["HTTPS_PROXY"]
        os.environ["HTTPS_PROXY"] = ""
    
    
    response = None
    method = 'GET'
    
    locallogger.info(f"Determining proxies for url: '{url}'...")
    
    proxyDict = {
            'None' : None,
            'Proxy_1' : getProxies(userID=userID, password=password, posixNeedsProxy=True, proxyNum=1),
            'Proxy_2' : getProxies(userID=userID, password=password, posixNeedsProxy=True, proxyNum=2),
            'Proxy_3' : getProxies(userID=userID, password=password, posixNeedsProxy=True, proxyNum=3),
        }
    
    for key in list(proxyDict.keys()):
    
        loggerNameList = [
                'commonfunctions.core.getResponseFromURL',
                'urllib3.connectionpool',
            ]
        setLoggerLevel(loggerNameList=loggerNameList, loggingLevel=logging.CRITICAL)
        

        
        proxies = proxyDict[key]
        
        response, _ = getResponseFromURL(url, method, headers=None, auth=None,  proxies=proxies, data=None, json=None, params=None, cookies=None, verify=False, timeout=120, session=None, loggingPrefix=None, responseEncoding=None, maxRetry=3, sleepCodeList=None, sleepSeconds=300, returnJson=False)
    
        setLoggerLevel(loggerNameList=loggerNameList, loggingLevel=logging.DEBUG)
        if response != None:
            proxiesReturnValue = proxies
            keyValue = key
            break
    
    if clearProxyInEnvironment == False:
        if httpProxyValue != None:
            os.environ["HTTP_PROXY"] = httpProxyValue
        
        if httpsProxyValue != None:
            os.environ["HTTPS_PROXY"] = httpsProxyValue
    
    if setProxyInEnvironment == True:
        if proxiesReturnValue != None:
            os.environ["HTTP_PROXY"] = proxies['http']
            os.environ["HTTPS_PROXY"] = proxies['https']
    
    
    if proxiesReturnValue != None:
        locallogger.info(f"Proxy set as: '{keyValue}'...")
        return proxiesReturnValue
    else:
        locallogger.info(f"Proxy set as: 'None'...")
        return proxiesReturnValue
        














def pathJoin(*args):
    filePath = r''
    argCount = len(args)
    if argCount > 0:
        for element in args:
            filePath = os.path.join(filePath, element)
    return filePath


def pathExists(*args):
    pathExistance = False
    filePath = pathJoin(*args)
    if os.path.exists(filePath):
        pathExistance = True
    return pathExistance
    
        

#%%-----------------------------------------------------------------------------------




def activeDirectoryBindUser(samAccountName, password, ldapServerIP, adDomainName):
    server = ldap3.Server(ldapServerIP, get_info=ldap3.ALL, use_ssl=True)
    try:
        conn = ldap3.Connection(server, user=f'{adDomainName}\\{samAccountName}', password=password, auto_bind=True, authentication=ldap3.NTLM)
    except Exception:
        return False
    return conn



def activeDirectoryBindAndPullAttribute(samAccountName, password, dcValue, attribute='mail'):
    
    conn = activeDirectoryBindUser(samAccountName=samAccountName, password=password, ldapServerIP='10.185.7.103', adDomainName=dcValue)

    
    if conn:
        result = conn.search(
        search_base=f'DC={dcValue},DC=com',
        search_filter=f'(sAMAccountName={samAccountName})',
        attributes=[attribute],
        )
        if result:
            email = conn.response[0]['attributes'][attribute]
            email = str(email).lower()
            return email
    else:
        return False




#%%-----------------------------------------------------------------------------------






global domainToIPAddressCache
domainToIPAddressCache = {}


def domainResolver(domainName):
    locallogger = logging.getLogger(f"{__name__}.domainResolver")
    
    ipAddress = False
    global domainToIPAddressCache
    if domainName in domainToIPAddressCache.keys():
        return domainToIPAddressCache[domainName]
    else:
        i = 1
        while True:
            try:
                ipAddress = socket.gethostbyname(domainName)
                domainToIPAddressCache[domainName] = ipAddress
                break
            except Exception as e:
                if i >= 20:
                    locallogger.error(f"Could not resolve domain '{domainName}' to IP Address even after 10 attempts")
                    break
                i = i + 1
        return ipAddress



def domainResolvedURL(url):
    locallogger = logging.getLogger(f"{__name__}.domainResolvedURL")
    

    schemePresent = False
    if re.search(r'^[A-Za-z0-9+.\-]+://', url):
        schemePresent = True
    if not schemePresent: 
        url = f"https://{url}"
    parsed = urlparse(url)
    domainName = parsed.netloc
    ipAddress = domainResolver(domainName=domainName)
    replaced = parsed._replace(netloc=ipAddress)
    return replaced.geturl()






#%%

def runStage(func, programName, stageUniqueName, stageHeading, programStage, continueFromStage, continueTillStage, timer, fullProgramTimer):
    
    
    
    if (continueFromStage <= programStage) and (continueTillStage >= programStage):
        
        locallogger = logging.getLogger(f"{programName}.{stageUniqueName}")
        
        locallogger.info(f"------------------------------------------")
        locallogger.info(f"Starting Stage: {programStage}")
        locallogger.info(f"{stageHeading}")    
        locallogger.info(f"------------------------------------------")
        timer.restartTimer()
        
        # Call the actual Stage Function
        func(locallogger=locallogger, stageUniqueName=stageUniqueName)
        
        locallogger.info(f"------------------------------------------")
        locallogger.info(f"Completed Stage: '{stageHeading}'.")
        locallogger.info(f"Time taken: {timer.getTimerTime()}; Total time taken: {fullProgramTimer.getTimerTime()}")
        locallogger.info(f"------------------------------------------")
        

    programStage = programStage + 1
    return programStage


#%%

class ProgramFlowManager:
    def __init__(self, programName, continueFromStage, continueTillStage):
        self.timer = Timer()
        self.fullProgramTimer = Timer()
        self.programStage = 1
        self.programName = programName
        self.continueFromStage = continueFromStage
        self.continueTillStage = continueTillStage
        self.stageUniqueName = ''
        self.stageHeading = ''
        self.locallogger = logging.getLogger(f"{self.programName}.ProgramFlowManager")
        self.stagelogger = None
    def setStage(self, stageUniqueName, stageHeading):
        self.stageUniqueName = stageUniqueName
        self.stageHeading = stageHeading
        self.stagelogger = logging.getLogger(f"{self.programName}.{self.stageUniqueName}")
        return self.stagelogger
    def startStage(self):
        if self.stagelogger == None:
            self.locallogger.error(f"Stage has not been set")
            sys.exit(1)
        if (self.continueFromStage <= self.programStage) and (self.continueTillStage >= self.programStage):
            self.timer.restartTimer()         
            self.stagelogger.info(f"------------------------------------------")
            self.stagelogger.info(f"Starting Stage: {self.programStage}")
            self.stagelogger.info(f"{self.stageHeading}")    
            self.stagelogger.info(f"------------------------------------------")
            self.timer.restartTimer()    
            return True
        else:
            self.stagelogger = None
            return False
    def endStage(self):
        if self.stagelogger != None:
            self.stagelogger.info(f"------------------------------------------")
            self.stagelogger.info(f"Completed Stage - {self.programStage}: '{self.stageHeading}'.")
            self.stagelogger.info(f"Time taken: {self.timer.getTimerTime()}; Total time taken: {self.fullProgramTimer.getTimerTime()}")
            self.stagelogger.info(f"------------------------------------------")
        self.programStage = self.programStage + 1
        if self.continueTillStage <= self.programStage:
            self.stagelogger.info(f"continueTillStage: '{self.continueTillStage}' reached.  Exiting...")
            sys.exit(0)
        self.stagelogger = None
        
#%%

def getStageState(pathDict, stageUniqueName, locallogger):
    
    stateDict = {}
    filePath = os.path.join(pathDict['Output_Folder'], f"State_Info_{stageUniqueName}.pickle")
    if not os.path.exists(filePath):
        locallogger.info(f"No state information exists.")
        return stateDict
    else:
        stateDict = readPickleToObject(filePath=filePath)
        locallogger.info(f"Loaded state information from file: '{filePath}'")
        return stateDict
        

def setStageState(stateDict, pathDict, stageUniqueName, locallogger):
    filePath = os.path.join(pathDict['Output_Folder'], f"State_Info_{stageUniqueName}.pickle")
    writeObjectToPickle(objectValue=stateDict, filePath=filePath)
    locallogger.info(f"Saved state information to file: '{filePath}'")




#%% ====================================================  ======================================


def getInitialProgramDates(pathDict, today_date, today_date_raw, locallogger):
    state = State(pathDict=pathDict, stateName=f"programDate", locallogger=locallogger)
    if not 'program_initial_date_raw' in state.d.keys():
        state.d['program_initial_date_raw'] = today_date_raw
    if not 'program_initial_date' in state.d.keys():
        state.d['program_initial_date'] = today_date
    state.save()
    program_initial_date_raw = state.d['program_initial_date_raw']
    program_initial_date = state.d['program_initial_date']
    return (program_initial_date, program_initial_date_raw)








def sha256(value):
    value = str(value)
    # https://docs.python.org/3/library/stdtypes.html#str.encode
    hashValue = hashlib.sha256(value.encode()).hexdigest()
    return hashValue

def sha512(value):
    value = str(value)
    hashValue = hashlib.sha512(value.encode("utf-8")).hexdigest()
    return hashValue    



def convertStringToUniqueInt(df, column, pathDict):
    locallogger = logging.getLogger(f"{__name__}.convertStringToUniqueInt")
    tdf1 = df[column].to_frame()
    tdf2 = dropDuplicates(df=tdf1)
    tdf3 = tdf2.sort_values(by=column)
    tdf3[f"{column}_i"] = np.arange(1, (tdf3.shape[0] + 1))
    writePickle(tdf3, folderPath=pathDict['Output_Folder'], fileNamePrefix=f"{column}_unique_int_reference")
    tdf4 = mergeDataframes(dataframe1=df, dataframe2=tdf3, onColumns_Left=[column], onColumns_Right=[column], howMerge='left', ensureNoDuplicatesOnRightForLeftMerge=True, replaceNullValuesInOutput=True)
    locallogger.info(f"Added column {column}_i to the dataframe")
    return tdf4





def convertStringToDate(dateString):
    if type(dateString) == str:
        datetimeValue = datetime.datetime.strptime(dateString, "%d-%b-%Y")
        dateValue = datetimeValue.date()
        return dateValue
    else:
        return dateString



def convertDateToString(dateValue):
    if type(dateValue) == datetime.date:
        datetimeValue = datetime.datetime.combine(dateValue, datetime.datetime.min.time())
        dateString = datetimeValue.strftime("%d-%b-%Y")
        return dateString
    else:
        return dateValue


def eliminateEqualSignAtStart(stringValue):
    stringValue = str(stringValue)
    while stringValue.startswith("="):
        stringValue = stringValue[1:]
    return stringValue




def isVowel(character):
    locallogger = logging.getLogger(f"{__name__}.isVowel")
    c = str(character)
    cLower = c.lower()
    if cLower in "aeiou":
        return True
    else:
        return False











def xmlPrettyPrint(xmlString=None, xmlFilePath=None):
    locallogger = logging.getLogger(f"{__name__}.xmlPrettyPrint")
    if xmlString != None:
        dom = xml.dom.minidom.parseString(xmlString)
    elif xmlFilePath != None:
        dom = xml.dom.minidom.parse(xmlFilePath)
    else:
        locallogger.error(f"Both xmlString and xmlFilePath inputs are None!")
        sys.exit(1)
    prettyXMLString = dom.toprettyxml()
    return prettyXMLString




def xmlStructureShow(xmlString):
    locallogger = logging.getLogger(f"{__name__}.xmlStructureShow")

    def childStructure(children, depth):
        for child in children:
            indentation = "    " * depth
            print(f"{indentation}Tag: {child.tag}")
            print(f"{indentation}Text: {child.text}")
            numOfElements = len(child)
            print(f"{indentation}Number of Elements: {numOfElements}")
            print(f"{indentation}---------------")
            if numOfElements > 0:
                depth = depth + 1
                children = list(child)
                childStructure(children=children, depth=depth)
    root = ET.fromstring(xmlString)
    children = list(root)
    childStructure(children=children, depth=0)


def getValueFromXML(root, searchString, extractOnlySingleValue=True):
    elementList = root.findall(searchString)
    if extractOnlySingleValue:
        if len(elementList) == 1:
            return elementList[0].text
        else:
            return ''
    else:
        return elementList


def xmlIterElement(root):
    for element in root.iter():
        print(f"tag: {element.tag}, attrib: {element.attrib}, text: {element.text}")



def getValueFromJSONPath(jsonDict, pathList):
    returnValue = None
    locallogger = logging.getLogger(f"{__name__}.getValueFromJSONPath")
    length = len(pathList)
    if length == 0:
        locallogger.error(f"pathList is empty")
        sys.exit(1)
    if (type(jsonDict) != dict) and (type(jsonDict) != list):
        locallogger.error(f"jsonDict is not a dictionary or list")
        sys.exit(1)
    doDict = jsonDict
    search = True
    found = False
    index = 0
    while search:
        if index <= (length - 1):
            key = pathList[index]
            if type(doDict) == list:
                returnList = []
                remainingPathList = pathList[index:]
                for itemDict in doDict:
                    returnValue = getValueFromJSONPath(jsonDict=itemDict, pathList=remainingPathList)
                    returnList.append(returnValue)
                return returnList
            if type(doDict) == dict:
                if key in doDict.keys():
                    doDict = doDict[key]
                    if index == (length - 1):
                        returnValue = doDict
                        found = True
                        search = False
                    else:
                        index = index + 1
                else:
                    search = False
            else:
                search = False
    return returnValue

#%% -------------------------------------------------------------------------------------------------------------


class Email:
    def __init__(self, emailServer, emailPort, defaultSender):
        self.emailServer = emailServer
        self.emailPort = emailPort
        self.defaultSender = defaultSender
    def getEmailServerObject(self):
        # Create connection with server and send email
        emailServerObject = smtplib.SMTP(host=self.emailServer, port=self.emailPort)
        return emailServerObject
    def sendEmail(self, subject, body, senderEmail=None, recipientEmail=None, ccRecipientEmail=None, bccRecipientEmail=None, attachmentPathList=None, loggingPrefix=None, testing=True):
        
        if loggingPrefix == None:
            locallogger = logging.getLogger(f'{__name__}.sendEmail')
        else:
            locallogger = logging.getLogger(f'{loggingPrefix}.{__name__}.sendEmail')
            
        if senderEmail == None:
            senderEmail = self.defaultSender
        
        if recipientEmail == None:
            recipientEmail = self.defaultSender
            
        message = MIMEMultipart()
        message["From"] = senderEmail
        message["Subject"] = subject
        

        message["To"] = recipientEmail
        mailReceivers = f"{recipientEmail}"
        
        if ccRecipientEmail != None:
            message["Cc"] = ccRecipientEmail
            mailReceivers = f"{mailReceivers}, {ccRecipientEmail}"
            
        
        if bccRecipientEmail != None:
            mailReceivers = f"{mailReceivers}, {bccRecipientEmail}"    


        if testing == True:
            mailReceivers = self.defaultSender
            subject = f"Test: {subject}"
            message["Subject"] = subject



        mimeObject = MIMEText(body, "html")
        message.attach(mimeObject)
        
        
        
        if attachmentPathList != None:
            # Attach Attachment Files
            for filePath in attachmentPathList:
                folderPath, fileName = os.path.split(filePath)
                attachmentPart = MIMEBase('application', "octet-stream")
                attachmentPart.set_payload(open(filePath, "rb").read())
                encoders.encode_base64(attachmentPart)
                attachmentPart.add_header('Content-Disposition', f'attachment; filename="{fileName}"')
                message.attach(attachmentPart)  


        # Create connection with server and send email
        emailServerObject = self.getEmailServerObject()
        
        try:
            actualMailReceiversList = extractStringIntoListofStrings(fullString=mailReceivers)
            emailServerObject.sendmail(senderEmail, actualMailReceiversList, message.as_string())
            locallogger.info(f"Email sent with subject '{subject}' and actualMailReceiversList as '{actualMailReceiversList}'.")  
        except Exception:
            locallogger.exception(f"Error while trying to send email with subject '{subject}'.")   


#%% ------------------------------------------------------------------------------------------------------------- 

def selectiveMerge(df1, filterframe1, df2, onColumnsLeft, onColumnsRight, howMerge='left', ensureNoDuplicatesOnRightForLeftMerge=True, replaceNullValuesInOutput=True):
    
    df3 = df1[filterframe1]
    
    df2ColumnsListExceptMergeKeys = columnListExceptCertainOnes(df=df2, exceptList=onColumnsRight)
    
    selectedColumnsIndf3 = columnListExceptCertainOnes(df=df3, exceptList=df2ColumnsListExceptMergeKeys)
    
    df4 = mergeDataframes(dataframe1=df3[selectedColumnsIndf3], dataframe2=df2, onColumns_Left=onColumnsLeft, onColumns_Right=onColumnsRight, howMerge=howMerge, ensureNoDuplicatesOnRightForLeftMerge=ensureNoDuplicatesOnRightForLeftMerge, replaceNullValuesInOutput=replaceNullValuesInOutput)
    
    df5 = df1[~filterframe1]
    
    df6 = pd.concat([df5, df4], axis=0, ignore_index=True, sort=False)

    return df6

#%%





#%%


def checkNextLinkPresent(jresponse, quitIfPresent=True, loggingPrefix=None):
    if loggingPrefix == None:
        locallogger = logging.getLogger(f'{__name__}.checkNextLinkPresent')
    else:
        locallogger = logging.getLogger(f'{loggingPrefix}.{__name__}.checkNextLinkPresent')
    
    if type(jresponse) != dict:
        locallogger.error(f"jresponse is not a dictionary.  Quitting...")
        sys.exit(1)
    
    keyList = list(jresponse.keys())
    
    nextLinkPresent = False
    
    for key in keyList:
        key = key.lower()
        if key in ['nextlink', 'next_link']:
            nextLinkPresent = True
    
    if  nextLinkPresent and quitIfPresent:
        locallogger.error(f"Next Link is present in jresponse.  Handle it! Quitting for now...")
        sys.exit(1)
    return nextLinkPresent
        
            




def printJSON(jresponse):
    jsonPretty = json.dumps(jresponse, indent=4, sort_keys=True)
    return jsonPretty



def urlParameterOrderRandomizer(urlString, delimiter):
    
    parameterList = urlString.split(delimiter)
    
    random.shuffle(parameterList)
    
    newURLString = delimiter.join(parameterList)
    
    return newURLString





def removeNonesFromList(objectList):
    locallogger = getLocalLogger(localloggername='removeNonesFromList', rootFileName=__name__,)
    
    if type(objectList) != list:
        locallogger.error(f"objectList is not of type list!")
        sys.exit(1)
    
    newList = []
    
    for item in objectList:
        if item != None:
            #print(item)
            newList.append(item)
    
    return newList



def getEqualityInNumber(variable, value):
    if variable == value:
        return 1
    return 0






def requestsCABundle(pathDict, setValue=True):
    companyPEMFile = os.path.join(pathDict['Output_Folder_Root'], 'supporting', 'root_ca.pem')
    if setValue == True:
        os.environ["REQUESTS_CA_BUNDLE"] = companyPEMFile
    else:
        if 'REQUESTS_CA_BUNDLE' in os.environ.keys():
            del os.environ["REQUESTS_CA_BUNDLE"]


#%%
            
def getFilePath(filePath, folderPath, fileName, loggingPrefix=None):
    locallogger = getLocalLogger(localloggername='getFilePath', rootFileName=__name__, loggingPrefix=loggingPrefix)
    if filePath == None:
        if (folderPath != None) and (fileName != None):
            filePath = os.path.join(folderPath, fileName)
        else:
            locallogger.error(f"folderPath, filePath and fileName are None.")
            sys.exit(1)
        return filePath



    
#%% 

#%% ================================================ Costly Operation ============================================================================================== 

class CostlyOperation:
    def __init__(self, expirySeconds, state, operationFunction):
        self.expirySeconds = expirySeconds
        self.operationFunction = operationFunction
        self.state = state
        self.valueDictTracker = self.state.getSet(key='valueDictTracker', value={})
    def saveState(self):
        self.state.update(key='valueDictTracker', value=self.valueDictTracker)
    def clearState(self):
        self.state.delete()
    def operate(self, keyID, *args):
        nowTime = datetime.datetime.now()
        if keyID in self.valueDictTracker.keys():
            valueDict = self.valueDictTracker[keyID]
            deltaTime = datetime.timedelta(seconds=self.expirySeconds)
            operationTime = valueDict['Operation_Time']
            thresholdTime = operationTime + deltaTime
            if nowTime < thresholdTime: # The threshold time has not been breached
                valueObject = valueDict['Value_Object']
                return valueObject
        valueObject = self.operationFunction(keyID, *args)
        valueDict = {
            'Operation_Time' : nowTime,
            'Value_Object' : valueObject,
        }
        self.valueDictTracker[keyID] = valueDict
        return valueObject



#%% ================================================  ============================================================================================== 





#%%

#%%

def convertStringToNumber(value):
	intValue = value
	if type(intValue) == str:
		intValue = intValue.strip()
	try:
		intValue = int(value)
	except:
		pass

	return intValue


# %% ================================================== Add Percentage Column to Dataframe  =====================================================================================



def addPercentageToDataframe(df, numberColumnName, sortOrder=True):
    df['Percentage'] = df[numberColumnName] * 100 / df[numberColumnName].sum(axis=0)
    df['Percentage']= df['Percentage'].round(2)
    if(sortOrder):
        df = df.sort_values(numberColumnName, ascending=False)
    return df

    
# %% ================================================== Main =====================================================================================


if __name__ == '__main__':
    
    pass



# -------------- TESTING SPACE -------------------------------------------------------------------
#%%















