    

import datetime
import sys
import logging
import os
#from git import Repo
from pathlib import Path
#import hcl
import pandas as pd
import re
import requests
import commonfunctions.core
import commonfunctions.context
import commonfunctions.context
import subprocess
import commonfunctions.process
import shutil

class TimeoutExpired(Exception):
    pass


gitPortalURL = "https://default.com"

gitDestinationString = "default.com"


def getGitFileContent(fileURL, token):
    locallogger = logging.getLogger(f"{__name__}.getGitFileContent")
    
    headers = {'Content-Type': 'application/json', "Authorization" : f"Bearer {token}"}
    parameters = {'limit' : 1000,}
    # response = requests.get(fileURL, headers=headers, params=parameters, verify=False, timeout=120,)
    response, dataPullStatus = commonfunctions.core.getResponseFromURL(url=fileURL, method='GET', headers=headers, params=parameters, loggingPrefix=locallogger.name)
    responsecode = response.status_code
    print(responsecode)
    fileContent = response.text
    return fileContent  



def getFileListFromGitRepoURL(project_name, repo_name, token):
    apiURL = f"{gitPortalURL}/rest/api/1.0/projects/{project_name}/repos/{repo_name}/files"
    headers = {'Content-Type': 'application/json', "Authorization" : f"Bearer {token}"}
    parameters = {'limit' : 1000,}
    response = requests.get(apiURL, headers=headers, params=parameters, verify=False, timeout=120,)
    responsecode = response.status_code
    print(responsecode)
    if responsecode == 200:
        jresponse = response.json()
        return jresponse['values']
    else:
        return None


def getBitbucketProjectRepoDetails(project_name, token):
    apiURL = f"{gitPortalURL}/rest/api/1.0/projects/{project_name}/repos"
    headers = {'Content-Type': 'application/json', "Authorization" : f"Bearer {token}"}
    parameters = {'limit' : 1000,}
    response = requests.get(apiURL, headers=headers, params=parameters, verify=False, timeout=120,)
    responsecode = response.status_code
    print(responsecode)
    jresponse = response.json()
    return jresponse






def getBitbucketProjectRepoDetailsWithToken(project_name, token):
    apiURL = f"{gitPortalURL}/rest/api/1.0/projects/{project_name}/repos"
    headers = {'Content-Type': 'application/json', "Authorization" : f"Bearer {token}"}
    parameters = {'limit' : 1000,}
    response = requests.get(apiURL, headers=headers, params=parameters, verify=False, timeout=120,)
    responsecode = response.status_code
    print(responsecode)
    jresponse = response.json()
    return jresponse


def getRepositoryDetails(token=None, auth=None, destination='pca'):
    locallogger = logging.getLogger(f"{__name__}.getRepositoryDetails")
    if destination == 'pca':
        apiURL = f"{gitPortalURL}/rest/api/1.0/repos"
    else:
        locallogger.error(f"destination: {destination} is not recognized.")
        sys.exit(1)
    
    if token != None:
        headers = {'Content-Type': 'application/json', "Authorization": f"Bearer {token}"}
    else:
        headers = {'Content-Type': 'application/json'}
        
    method = 'GET'

    finalPageReached = False

    startingID = 0

    df1 = pd.DataFrame()

    while not finalPageReached:
        parameters = {'limit': 1000, 'start': startingID, }
        
        response, dataPullStatus = commonfunctions.core.getResponseFromURL(url=apiURL, method=method, headers=headers, auth=auth,  proxies=None, data=None, json=None, params=parameters, verify=False, timeout=120, session=None, responseEncoding=None, maxRetry=3)

        jresponse = response.json()
        do_df = commonfunctions.core.json_normalize(jresponse['values'])
        df1 = pd.concat([df1, do_df], axis=0, ignore_index=True, sort=False)

        if jresponse['isLastPage'] == True:
            finalPageReached = True
        else:
            startingID = jresponse['nextPageStart']

    selectedColumns = [
        'id',
        'name',
        'slug',
        'description',
        'links.self',
        'links.clone',
        'project.key',
        'project.id',
        'project.name',
        'project.description',
        'project.links.self',
        #'project.owner.name',
        #'project.owner.emailAddress',
        #'project.owner.displayName',
        #'project.owner.active',
    ]
    df1 = df1[selectedColumns]
    df1 = commonfunctions.core.replaceNullValuesInDataframe(df1)
    return df1




def getSpecificRepositoryDetails(projectKey, repoSlug, token=None, auth=None):

    apiURL = f"{gitPortalURL}/rest/api/1.0/projects/{projectKey}/repos/{repoSlug}"
    
    if token != None:
        headers = {'Content-Type': 'application/json', "Authorization": f"Bearer {token}"}
    else:
        headers = {'Content-Type': 'application/json'}
        
    method = 'GET'

        
    response, dataPullStatus = commonfunctions.core.getResponseFromURL(url=apiURL, method=method, headers=headers, auth=auth,  proxies=None, data=None, json=None, params=None, verify=False, timeout=120, session=None, responseEncoding=None, maxRetry=3)

    jresponse = response.json()
    df1 = commonfunctions.core.json_normalize(jresponse)


#    selectedColumns = [
#        'id',
#        'name',
#        'slug',
#        'description',
#        'links.self',
#        'links.clone',
#        'project.key',
#        'project.id',
#        'project.name',
#        'project.description',
#        'project.links.self',
#        #'project.owner.name',
#        #'project.owner.emailAddress',
#        #'project.owner.displayName',
#        #'project.owner.active',
#    ]
#    df1 = df1[selectedColumns]
    df1 = commonfunctions.core.replaceNullValuesInDataframe(df1)
    return df1










rulePattern = re.compile(r'(.*):(.*):(.*):(.*):(.*):(.*):(.*)')

def ruleExtract(nsgRulesList):
    locallogger = logging.getLogger(f"{__name__}.ruleExtract")
    locallogger.setLevel(logging.DEBUG)
    df = pd.DataFrame()
    ruleNoMatchSeries = pd.Series()
    if type(nsgRulesList) != list:
        locallogger.error(f"nsgRulesList is not a list: {nsgRulesList}")
        sys.exit(1)
    if len(nsgRulesList) > 0:
        for rule in nsgRulesList:
            # print(rule)
            matches = re.match(rulePattern, rule)
            if matches:
                dictValue = {}
                dictValue['Rule_Name'] = matches.group(1)
                dictValue['Rule_Access'] = matches.group(2)
                dictValue['Rule_Protocol'] = matches.group(3)
                dictValue['Rule_Source_Ports'] = matches.group(4)
                dictValue['Rule_Destination_Ports'] = matches.group(5)
                dictValue['S.IP_Addresses_String'] = matches.group(6)
                dictValue['D.IP_Addresses_String'] = matches.group(7)
                df = df.append(dictValue, ignore_index=True)
            else:
                locallogger.error(f"Pattern match failed for rule: {rule}")
                ruleNoMatchSeries.append(pd.Series(rule))
    return df, ruleNoMatchSeries



NSGVersionAndModuleNamePattern = re.compile(r"'([A-Za-z0-9_-]+)': {'source': 'git::ssh://git@code\.connect\.net:7999/rtsretm/network-security-group\.git\?ref=([0-9]+\.[0-9]+\.[0-9]+|latest)")




timeLastModifiedRegex = re.compile(r"format:([0-9]{4}-[0-9]{2}-[0-9]{2})")


def getGitFileLastModified(fileFullPath):
    locallogger = logging.getLogger(f"{__name__}.getGitFileLastModified")  

    # git log -1 --pretty="format:%ci"

    commandList = [
                "git",
                "log",
                "-1",
                '--pretty="format:%ci"',
                f"{fileFullPath}",
            ]
    
    parentDir = Path(fileFullPath).parent

    
    out, err = commonfunctions.process.runProcess(commandList=commandList, cwd=parentDir, waitTimeout= 2*60, callingFunctionLogger=locallogger.name)
    
    lastModifiedDateString = ''
    findings = timeLastModifiedRegex.findall(out)
    if len(findings):
        finding = findings[0]
        lastModifiedDate = datetime.datetime.strptime(finding, "%Y-%m-%d")
        lastModifiedDateString = lastModifiedDate.strftime("%d-%b-%Y")
    
    return lastModifiedDateString


#%%



def updateGitRepository(projectName, repoName, sourceCodeFolderRootPath, selectedBranchName='master', loggingPrefix=None):

    
    if loggingPrefix:
        locallogger = logging.getLogger(f"{loggingPrefix}.updateGitRepository")
    else:
        locallogger = logging.getLogger(f"{__name__}.updateGitRepository")  
    
    #locallogger.info("Updateing git repository")
    
    
    repoDir = os.path.join(sourceCodeFolderRootPath, str(projectName), str(repoName))

    


    commandList = [
                "git",
                "clean",
                "-fdx",   # Remove any changed directories and files
            ]
    
    out, err = commonfunctions.process.runProcess(commandList=commandList, cwd=repoDir, waitTimeout= 2*60, callingFunctionLogger=locallogger.name)
    
    #locallogger.debug(f"Cleaning any changed directories or files.  {out} {err}")
    
    
    if 'fatal' in err:
        locallogger.error(f"{err}")
        sys.exit(1)
        

    commandList = [
                "git",
                "checkout",
                f"{selectedBranchName}",   
            ]
    
    out, err = commonfunctions.process.runProcess(commandList=commandList, cwd=repoDir, waitTimeout= 2*60, callingFunctionLogger=locallogger.name)
    
    #locallogger.debug(f"Git checkout latest branch.  {out} {err}")
    
    
    if 'fatal' in err:
        locallogger.error(f"{err}")
        sys.exit(1)          
        
        
    commandList = [
                "git",
                "pull",
                "--all",   
            ]
    
    out, err = commonfunctions.process.runProcess(commandList=commandList, cwd=repoDir, waitTimeout= 2*60, callingFunctionLogger=locallogger.name)
    
    #locallogger.debug(f"Git pull all latest changes.  {out} {err}")
    
    
    if 'fatal' in err:
        locallogger.error(f"{err}")
        sys.exit(1)               



    # Get CommitVersion
    commandList = [
                "git",
                "ls-remote",  
            ]
    
    out, err = commonfunctions.process.runProcess(commandList=commandList, cwd=repoDir, waitTimeout= 2*60, callingFunctionLogger=locallogger.name)
    
    #locallogger.debug(f"Git get remote commit versions.  {out} {err}")
    
    
    if 'fatal' in err:
        locallogger.error(f"{err}")
        sys.exit(1)
        
    regexMatches = re.findall("(?P<commitVersion>\w{40})\s+refs/heads/(?P<branchName>.*)", out)

    commitOfSelectedBranch = None
    
    if len(regexMatches) > 0:
        # findings exist
        
        for (commitVersion, branchName) in regexMatches:
            if branchName == selectedBranchName:
                commitOfSelectedBranch = commitVersion
    
    if commitOfSelectedBranch == None:
        locallogger.error(f"Cannot find commitVersion of selected branch")
        return (commitOfSelectedBranch, f"Cannot find commitVersion of '{selectedBranchName}'")
    
    
    return (commitOfSelectedBranch, '')






#%%




def updateGitRepositoryForACA(projectName, repoName, sourceCodeFolderRootPath, loggingPrefix=None):

    
    if loggingPrefix:
        locallogger = logging.getLogger(f"{loggingPrefix}.updateGitRepositoryForACA")
    else:
        locallogger = logging.getLogger(f"{__name__}.updateGitRepositoryForACA")  
    
    #locallogger.info("Updateing git repository")
    
    
    repoDir = os.path.join(sourceCodeFolderRootPath, str(projectName), str(repoName))
    
    commandList = [
                "git",
                "branch",
                "--all",
            ]
    
    out, err = commonfunctions.process.runProcess(commandList=commandList, cwd=repoDir, waitTimeout= 2*60, callingFunctionLogger=locallogger.name)
    
    
    #locallogger.debug(f"Git branches in the folder: {out} {err}")
    
    if 'fatal' in err:
        locallogger.error(f"{err}")
        sys.exit(1)
        
    
    #latestbranch = re.findall(r"(?:remotes/origin/HEAD\s*->\s*.origin/)(?P<branch>.*)", out)
    latestbranch = re.findall(r"remotes/origin/HEAD\s*->\s*.origin/(.*)", out)


    if len(latestbranch) == 0:
        
        # Unable to find latest branch
        
        locallogger.error(f"Unable to find latest branch.  Message: {out}; {err}")
        locallogger.info(f"Skipping repository")
        return (None, None, f"Unable to find latest branch.  Message: {out}; {err}")

    
    else:   # Latest branch was found
        latestbranch = latestbranch[0]
        locallogger.debug(f"Latest branch: {latestbranch}")

        commandList = [
                    "git",
                    "clean",
                    "-fdx",   # Remove any changed directories and files
                ]
        
        out, err = commonfunctions.process.runProcess(commandList=commandList, cwd=repoDir, waitTimeout= 2*60, callingFunctionLogger=locallogger.name)
        
        #locallogger.debug(f"Cleaning any changed directories or files.  {out} {err}")
        
        
        if 'fatal' in err:
            locallogger.error(f"{err}")
            sys.exit(1)
            

        commandList = [
                    "git",
                    "checkout",
                    f"{latestbranch}",   
                ]
        
        out, err = commonfunctions.process.runProcess(commandList=commandList, cwd=repoDir, waitTimeout= 2*60, callingFunctionLogger=locallogger.name)
        
        #locallogger.debug(f"Git checkout latest branch.  {out} {err}")
        
        
        if 'fatal' in err:
            locallogger.error(f"{err}")
            sys.exit(1)          
            
           
        commandList = [
                    "git",
                    "pull",
                    "--all",   
                ]
        
        out, err = commonfunctions.process.runProcess(commandList=commandList, cwd=repoDir, waitTimeout= 2*60, callingFunctionLogger=locallogger.name)
        
        #locallogger.debug(f"Git pull all latest changes.  {out} {err}")
        
        
        if 'fatal' in err:
            locallogger.error(f"{err}")
            sys.exit(1)               



        # Get CommitVersion
        commandList = [
                    "git",
                    "ls-remote",  
                ]
        
        out, err = commonfunctions.process.runProcess(commandList=commandList, cwd=repoDir, waitTimeout= 2*60, callingFunctionLogger=locallogger.name)
        
        #locallogger.debug(f"Git get remote commit versions.  {out} {err}")
        
        
        if 'fatal' in err:
            locallogger.error(f"{err}")
            sys.exit(1)
            
        regexMatches = re.findall("(?P<commitVersion>\w{40})\s+refs/heads/(?P<branchName>.*)", out)
    
    commitOfLatestBranch = None
    
    if len(regexMatches) > 0:
        # findings exist
        
        for (commitVersion, branchName) in regexMatches:
            if branchName == latestbranch:
                commitOfLatestBranch = commitVersion
    
    if commitOfLatestBranch == None:
        locallogger.error(f"Cannot find commitVersion of latestbranch")
        return (latestbranch, commitOfLatestBranch, f"Cannot find commitVersion of latestbranch")
    
    
    return (latestbranch, commitOfLatestBranch, '')

#%%


def cloneUpdateGitRepository(projectKey, repositorySlug, sourceCodeFolderRootPath, loggingPrefix=None, destination='PCA', mode='ACA', selectedBranchName='master'):

    if loggingPrefix:
        locallogger = logging.getLogger(f"{loggingPrefix}.cloneUpdateGitRepository")
    else:
        locallogger = logging.getLogger(f"{__name__}.cloneUpdateGitRepository")    
    
    locallogger.info(f"Clone and update the repo from bitbuket: {projectKey}/{repositorySlug}")
    
    if destination == 'PCA':
        url = f"ssh://{gitDestinationString}/{projectKey}/{repositorySlug}.git"

    
    
    repoDir = os.path.join(sourceCodeFolderRootPath, str(projectKey), str(repositorySlug))
    
    # If repository directory does not exist, create it
    if not os.path.exists(repoDir):
        os.makedirs(repoDir)
        
    
    # Check if the directory is already a git repository
    commandList = [
                "git",
                "status",
            ]
    
    out, err = commonfunctions.process.runProcess(commandList=commandList, cwd=repoDir, waitTimeout= 2*60, callingFunctionLogger=locallogger.name)
    
    if out == '': # Not a git folder
        
        #locallogger.debug(f"Not an existing git folder: {repoDir}")
        
        # Perform clone
        
        commandList = [
                    "git",
                    "clone",
                    f"{url}",
                    f"{repoDir}",
                ]
        
        out, err = commonfunctions.process.runProcess(commandList=commandList, cwd=repoDir, waitTimeout= 2*60, callingFunctionLogger=locallogger.name)
        
        #locallogger.debug(f"Message: {out}; {err}")
        
        if 'fatal' in err:
            locallogger.error(f"err: {err}")
            return ('', '', 'repository_error', err)
        
        if "You appear to have cloned an empty repository" in err:
            return ('', '', 'empty_repository', '')

    
    if mode == 'ACA':
        (latestbranch, commitVersion, comments) = updateGitRepositoryForACA(projectKey, repositorySlug, sourceCodeFolderRootPath, loggingPrefix=locallogger.name)

    else:
        (commitVersion, comments) = updateGitRepository(projectKey, repositorySlug, sourceCodeFolderRootPath, selectedBranchName=selectedBranchName, loggingPrefix=loggingPrefix)
        latestbranch = selectedBranchName
    
    if commitVersion == None:
        return ('', '', 'repository_error', comments)
        
    return (latestbranch, commitVersion, 'success', '')
    




def getBranchesFromRepo(projectKey: str, repositorySlug: str, pathDict: dict):
    locallogger = logging.getLogger(f"{__name__}.getBranchesFromRepo")
    locallogger.info(f"Get all the branches from the cloned repository: {projectKey}/{repositorySlug}")
    workdir = os.path.join(pathDict['Output_Folder'], projectKey, repositorySlug)
    p = subprocess.Popen('git branch -r', shell=False,
                         stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=workdir)
    try:
        out, err = p.communicate()
        p.wait(timeout = 2 * 60)
    except TimeoutExpired:
        p.kill()
        out, err = p.communicate()
    if err:
        locallogger.error(err.decode('utf-8'))
        sys.exit(1)
    outputString = out.decode('utf-8')
    outputList = outputString.split('\n')
    branchList = []
    for item in outputList:
        itemStripped = item.strip()
        if (itemStripped != "") and ("->" not in itemStripped):
            branchList.append(itemStripped)
    return branchList





#def getCommitList(projectKey, repositorySlug, pathDict, startFromBranch='latest', NumberOfVersionsToScan=1):
#    
#    locallogger = logging.getLogger(f"{__name__}.getCommitList")
#    locallogger.info(f"Get the commit version list from the git repository: {projectKey}/{repositorySlug}")
#    
#    
#    repoDir = os.path.join(pathDict['Output_Folder'], projectKey, repositorySlug)
#    
#    if startFromBranch == 'latest':
#        commandList = [
#                "git", 
#                "log", 
#                "--all",
#                "-n",
#                "3",
#                ]
#        
#        out, err = commonfunctions.process.runProcess(commandList=commandList, cwd=repoDir, waitTimeout= 2*60, callingFunctionLogger=locallogger.name)
        
        
    # The below does not seem to only show commits for the 'selected' branch and needs further work and correction.  For later.
#    elif startFromBranch in getBranchesFromRepo(projectKey, repositorySlug, pathDict):
#        p = subprocess.Popen(f'git checkout {startFromBranch}',  shell=False,
#                             stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=workdir)
#        try:
#            out, err = p.communicate()
#            p.wait(timeout = 2 * 60)
#        except TimeoutExpired:
#            p.kill()
#            out, err = p.communicate()
#        command = "git log"
#    else:
#        raise Exception("Branch not found, please check your branch to scan. If you want to scan the latest of all branches, please use latest")
        

        
#    if isinstance(NumberOfVersionsToScan, int):
#        command = command + " -n " + str(NumberOfVersionsToScan)
#        
#
#        
#    p = subprocess.Popen(command, shell=False,  stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=repoDir)
#    try:
#        out, err = p.communicate()
#        p.wait(timeout = 2 * 60)
#    except TimeoutExpired:
#        p.kill()
#        out, err = p.communicate()
#    if err:
#        locallogger.error(err.decode('utf-8'))
#        return []
#    outputString = out.decode('utf-8')
#    commitsList = re.findall(r'commit (?P<commit>[a-f0-9]{40})', outputString)
#    return commitsList






def getBranchCommitMap(projectKey, repositorySlug, pathDict, startFromBranch="latest", NumberOfVersionsToScan=1):
    
    locallogger = logging.getLogger(f"{__name__}.getBranchCommitMap")
    locallogger.info(f"Generating Branch Commit Map for: {projectKey}/{repositorySlug}")
    
    
    commitList = getCommitList(projectKey, repositorySlug, pathDict,
                               startFromBranch, NumberOfVersionsToScan)
    
    workdir = os.path.join(pathDict['Output_Folder'], projectKey, repositorySlug)
    branchCommitMap = {}
    for commit in commitList:
        p = subprocess.Popen(f'git branch -a --contains {commit}', shell=False,
                             stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=workdir)
        try:
            out, err = p.communicate()
            p.wait(timeout = 2 * 60)
        except TimeoutExpired:
            p.kill()
            out, err = p.communicate()
        if err:
            locallogger.error(err.decode('utf-8'))
            continue
        outputString = re.sub(r'\*', "", out.decode('utf-8'))
        outputList = outputString.splitlines()
        branchList = []
        for item in outputList:
            branchList.append(item.strip())
        branchCommitMap[commit] = branchList
    return branchCommitMap












