#%% Import modules and setup the environment

#import datetime
from importlib import reload
import sys
import logging
from netaddr import IPSet
from netaddr.core import AddrFormatError
from IPy import IP
from netaddr import IPSet
import pandas as pd
import re
import os

try:
    dfDict
except NameError:
    dfDict = {}
    

try:
    pathDict
except NameError:
    pathDict = {}


########################## CHANGE FOLDER ##############################################
pathDict['Program_Folder'] = rf'{os.environ["AA_GIT_REPOSITORY_PATH"]}\files\common'
#######################################################################################


sys.path.append(pathDict['Program_Folder'])
import commonfunctions.core
import commonfunctions.context

reload(commonfunctions.core)







def getNetworkOwnership(IP_Address_CIDR, networkDf, buDf, envDf, timer):
    locallogger = logging.getLogger(f"{__name__}.getNetworkOwnership")
    locallogger.setLevel(logging.DEBUG)
    networkDf = networkDf.sort_values(by=['Network_Length'], ascending=True)
    networkDf = networkDf.reset_index(drop=True)
    intersectionIPSet = IPSet([])
    ipTypeCorrection = False
    buValueList = []
    expandedBuNameList = []
    buCategoryList = []
    tierValueList = []
    functionValueList = []
    functionCategoryList = []
    siteValueList = []
    matchingSubnetList = []
    if not (IP_Address_CIDR == '0.0.0.0/0'):
        try:
            ip1 = IPSet([IP_Address_CIDR])
            mask1 = commonfunctions.core.getMaskValue(IP_Address_CIDR)
            for index, row in networkDf.iterrows():
                ip2 = IPSet([row['Corrected_Subnet_Address']])
                intersection = ip1 & ip2
                intersectionIPSet = intersectionIPSet | intersection
                if len(intersection) != 0:
                    comparedSubnetValue = row['Corrected_Subnet_Address']
                    mask2 = commonfunctions.core.getMaskValue(comparedSubnetValue)
                    buValue = row['BU']
                    tierValue = row['Tier']
                    siteValue = row['Site']
                    functionValue = row['Function']
                    expandedBUNameValue = ''
                    buCategoryValue = ''
                    functionCategoryValue = ''
                    if buValue in buDf['Key'].values:
                        filterframe = (buDf['Key'] == buValue)
                        expandedBUNameValue = buDf[filterframe]['Expanded_Business_Unit_Name'].iloc[0]
                        buCategoryValue = buDf[filterframe]['BU_Category'].iloc[0]                
                    if functionValue in envDf['Key'].values:
                        filterframe = (envDf['Key'] == functionValue)
                        functionCategoryValue = envDf[filterframe]['Environment_Category'].iloc[0]
                    if (len(intersectionIPSet) == len(ip1)):
                        ip = IP(IP_Address_CIDR)
                        ipType = ip.iptype()
                        if ipType == 'PUBLIC':
                            ipTypeCorrection = True
                    if (len(intersection) == len(ip1)) and (mask1 >= mask2):
                        matchingSubnetList = [comparedSubnetValue]   
                        buValueList = [buValue]
                        tierValueList = [tierValue]
                        siteValueList = [siteValue]
                        functionValueList = [functionValue]
                        expandedBuNameList = [expandedBUNameValue]
                        buCategoryList = [buCategoryValue]
                        functionCategoryList = [functionCategoryValue]
                        break
                    else:
                        matchingSubnetList.append(comparedSubnetValue)   
                        buValueList.append(buValue)
                        tierValueList.append(tierValue)
                        siteValueList.append(siteValue)
                        functionValueList.append(functionValue)
                        expandedBuNameList.append(expandedBUNameValue)
                        buCategoryList.append(buCategoryValue)
                        functionCategoryList.append(functionCategoryValue)
        except AddrFormatError:
            #locallogger.warning(f"Improper IP Address received.  Unabled to convert to IPSet.  IP_Address_CIDR: {IP_Address_CIDR}")
            pass
    timer.progressReport(locallogger)
    #return dictionary
    return buValueList, expandedBuNameList, buCategoryList, tierValueList, functionValueList, \
        functionCategoryList, siteValueList, matchingSubnetList, len(intersectionIPSet), ipTypeCorrection





def getNetworkOwnershipModified(df, networkDf, buDf, envDf, timer):
    locallogger = logging.getLogger(f"{__name__}.getNetworkOwnershipModified")
    locallogger.setLevel(logging.DEBUG)
    networkDf = networkDf.sort_values(by=['Network_Length'], ascending=True)
    networkDf = networkDf.reset_index(drop=True)
    for indexDf, rowDf in df.iterrows():
        intersectionIPSet = IPSet([])
        ipTypeCorrection = False
        buValueList = []
        expandedBuNameList = []
        buCategoryList = []
        tierValueList = []
        functionValueList = []
        functionCategoryList = []
        siteValueList = []
        matchingSubnetList = []
        IP_Address_CIDR = rowDf['IP_Address_CIDR']
        if not (IP_Address_CIDR == '0.0.0.0/0'):
            try:
                ip1 = IPSet([IP_Address_CIDR])
                mask1 = commonfunctions.core.getMaskValue(IP_Address_CIDR)
                for index, row in networkDf.iterrows():
                    ip2 = IPSet([row['Corrected_Subnet_Address']])
                    intersection = ip1 & ip2
                    intersectionIPSet = intersectionIPSet | intersection
                    if len(intersection) != 0:
                        comparedSubnetValue = row['Corrected_Subnet_Address']
                        mask2 = commonfunctions.core.getMaskValue(comparedSubnetValue)
                        buValue = row['BU']
                        tierValue = row['Tier']
                        siteValue = row['Site']
                        functionValue = row['Function']
                        expandedBUNameValue = ''
                        buCategoryValue = ''
                        functionCategoryValue = ''
                        if buValue in buDf['Key'].values:
                            filterframe = (buDf['Key'] == buValue)
                            expandedBUNameValue = buDf[filterframe]['Expanded_Business_Unit_Name'].iloc[0]
                            buCategoryValue = buDf[filterframe]['BU_Category'].iloc[0]                
                        if functionValue in envDf['Key'].values:
                            filterframe = (envDf['Key'] == functionValue)
                            functionCategoryValue = envDf[filterframe]['Environment_Category'].iloc[0]
                        if (len(intersectionIPSet) == len(ip1)):
                            ip = IP(IP_Address_CIDR)
                            ipType = ip.iptype()
                            if ipType == 'PUBLIC':
                                ipTypeCorrection = True
                        if (len(intersection) == len(ip1)) and (mask1 >= mask2):
                            matchingSubnetList = [comparedSubnetValue]   
                            buValueList = [buValue]
                            tierValueList = [tierValue]
                            siteValueList = [siteValue]
                            functionValueList = [functionValue]
                            expandedBuNameList = [expandedBUNameValue]
                            buCategoryList = [buCategoryValue]
                            functionCategoryList = [functionCategoryValue]
                            break
                        else:
                            matchingSubnetList.append(comparedSubnetValue)   
                            buValueList.append(buValue)
                            tierValueList.append(tierValue)
                            siteValueList.append(siteValue)
                            functionValueList.append(functionValue)
                            expandedBuNameList.append(expandedBUNameValue)
                            buCategoryList.append(buCategoryValue)
                            functionCategoryList.append(functionCategoryValue)
            except AddrFormatError:
                #locallogger.warning(f"Improper IP Address received.  Unabled to convert to IPSet.  IP_Address_CIDR: {IP_Address_CIDR}")
                pass
        df.loc[indexDf]['Matching_Network_Subnet'] = tuple(matchingSubnetList)
        df.loc[indexDf]['BU'] = tuple(buValueList)
        df.loc[indexDf]['Tier'] = tuple(tierValueList)
        df.loc[indexDf]['Site'] = tuple(siteValueList)
        df.loc[indexDf]['Function'] = tuple(functionValueList)
        df.loc[indexDf]['Expanded_BU_Name'] = tuple(expandedBuNameList)
        df.loc[indexDf]['BU_Category'] = tuple(buCategoryList)
        df.loc[indexDf]['Function_Category'] = tuple(functionCategoryList)
        df.loc[indexDf]['Matching_IP_Address_Length'] = len(intersectionIPSet)
        df.loc[indexDf]['IP_Type_Correction'] = ipTypeCorrection
        timer.progressReport(locallogger)
    #return dictionary
    return df




def getIPSet(CIDRAddress, progressTracker=None):
    locallogger = logging.getLogger(f"{__name__}.getIPSet")
    locallogger.setLevel(logging.DEBUG)
    if type(CIDRAddress) == str:
        if CIDRAddress == '*':
            CIDRAddress = '0.0.0.0/0'
        CIDRAddress = [CIDRAddress]
    elif type(CIDRAddress) == list:
        if CIDRAddress == ['*']:
            CIDRAddress = ['0.0.0.0/0']
    try:
        ipSet = IPSet(CIDRAddress) # Expecting that CIDRAddress will be a List
    except AddrFormatError:
        #locallogger.warning(f"Improper IP Address received.  Unabled to convert to IPSet.  CIDRAddress: {CIDRAddress}")
        return IPSet([])
    if progressTracker:
        progressTracker.progressReport(locallogger)
    return ipSet




def checkAgainstAllowedPublicIPAddressIPSet(df, srcMask, dstMask, allowedPublicIPAddressesIPSet, progressTracker):
    locallogger = logging.getLogger(f"{__name__}.checkAgainstAllowedPublicIPAddressIPSet")
    locallogger.setLevel(logging.DEBUG)
    do_df = df[srcMask]
    for index, row in do_df.iterrows():
        #locallogger.debug(f"index: {index}")
        intersection = row['Rule_Source_Addresses_CIDR.IPSet'] & allowedPublicIPAddressesIPSet
        #locallogger.debug(f"intersection: {intersection}, len(intersection): {len(intersection)}, len(sourceCIDR): {len(row['Rule_Source_Addresses_CIDR.IPSet'])}")
        if len(intersection) != len(row['Rule_Source_Addresses_CIDR.IPSet']):
            df.loc[index, 'Risk_External_Connection'] = True
        progressTracker.progressReport(locallogger)
    do_df = df[dstMask]
    for index, row in do_df.iterrows():
        intersection = row['Rule_Destination_Addresses_CIDR.IPSet'] & allowedPublicIPAddressesIPSet
        if len(intersection) != len(row['Rule_Destination_Addresses_CIDR.IPSet']):
            df.loc[index, 'Risk_External_Connection'] = True
        progressTracker.progressReport(locallogger)








def findNetworkMatches(ipSet, networkRefDf, internalNetworkFullIPset=None, progressTracker=None, ipList=None):
    locallogger = logging.getLogger(f"{__name__}.findNetworkMatches")
    locallogger.setLevel(logging.DEBUG)
    selectedColumns = [
        'IP_Addresses_IPSet',
        'Intersection_Network_Length',
        'BU',
        'Tier',
        'Network_Classification',
        'Function',
        'Site',
        'Expanded_Business_Unit_Name',
        'BU_Category',
        'Expanded_Environment_Name',
        'Environment_Category',
        ]
    ipType = 'PRIVATE'
    data = [tuple(IPSet([])), (0,), tuple(''), tuple(''), tuple(''), tuple(''), tuple(''), tuple(''), tuple(''), tuple(''), tuple('')]
    if internalNetworkFullIPset == None:
        internalNetworkFullIPset = IPSet([])
    # --------- handle '0.0.0.0/0', otherwise it will take a long time -------
    if (ipSet == IPSet(['0.0.0.0/0'])):
        if progressTracker:
            progressTracker.progressReport(locallogger)
        ipType = 'PUBLIC'
        return pd.Series(data, index=selectedColumns).append(pd.Series(ipType, index=['IP_Type']))
    # --------- handle empty IPSet -------
    elif (ipSet == IPSet([])):
        if progressTracker:
            progressTracker.progressReport(locallogger)
        ipType = 'PRIVATE'
        if 'internet' in str(ipList).lower():
            ipType = 'PUBLIC'
        return pd.Series(data, index=selectedColumns).append(pd.Series(ipType, index=['IP_Type']))
    else:    
        # -------- Checking IP Type --------------------------
        ipSet_Minus_internalNetworkFullIPset = ipSet - internalNetworkFullIPset
        if isIPSetPublic(ipSet, ipList):
            ipType = 'PUBLIC'
        if ipType == 'PUBLIC':
            if isIPSetPublic(ipSet_Minus_internalNetworkFullIPset, ipList):
                ipType = 'PUBLIC'
            else:
                ipType = 'LEGACYPRIVATE'
        # --------- finding perfect match -------------
        mask = ( networkRefDf['Network_Length'] >= len(ipSet) )
        tdf = networkRefDf[mask]
        tdf['Intersection_Network_Length'] = tdf['IP_Addresses_IPSet'].apply(lambda x: len(x & ipSet))
        mask2 = ( tdf['Intersection_Network_Length'] == len(ipSet) )
        tdf2 = tdf[mask2]
        if len(tdf2):
            tdf2 = tdf2.sort_values('Network_Length', ascending=True)
            tdf2 = tdf2.reset_index(drop=True)
            match = tdf2.loc[0]
            if progressTracker:
                progressTracker.progressReport(locallogger)
            match[selectedColumns] = match[selectedColumns].apply(lambda x: tuple([x]))
            return match[selectedColumns].append(pd.Series(ipType, index=['IP_Type']))
        else:
            # --------------      
            tdf = networkRefDf
            tdf['Intersection_Network_Length'] = tdf['IP_Addresses_IPSet'].apply(lambda x: len(x & ipSet))
            mask2 = ( tdf['Intersection_Network_Length'] > 0 )
            tdf2 = tdf[mask2]
            if len(tdf2):
                tdf2 = tdf2.reset_index(drop=True)
                tdf2 = tdf2.sort_values('Intersection_Network_Length', ascending=False)
                tdf2['Group_By'] = True
                tdf3 = tdf2.groupby('Group_By', sort=False, as_index=False).agg({
                        'IP_Addresses_IPSet' : lambda x: tuple(x),
                        'Intersection_Network_Length' : lambda x: tuple(x),
                        'BU' : lambda x: tuple(x),     
                        'Tier' : lambda x: tuple(x),
                        'Network_Classification' : lambda x: tuple(x),
                        'Function' : lambda x: tuple(x),
                        'Site' : lambda x: tuple(x),
                        'Expanded_Business_Unit_Name' : lambda x: tuple(x),
                        'BU_Category' : lambda x: tuple(x),
                        'Expanded_Environment_Name' : lambda x: tuple(x),
                        'Environment_Category': lambda x: tuple(x),
                        })
                if progressTracker:
                    progressTracker.progressReport(locallogger)
                return tdf3.iloc[0][selectedColumns].append(pd.Series(ipType, index=['IP_Type']))
            else:
                if progressTracker:
                    progressTracker.progressReport(locallogger)
                return pd.Series(data, index=selectedColumns).append(pd.Series(ipType, index=['IP_Type']))    





portRangePattern = re.compile(r'([0-9]{1,6})-([0-9]{1,6})')

portGreaterThanPattern = re.compile(r'>([0-9]{1,6})')


def checkNamedPortExistanceInPortDf(uniquePortTupleDf, portTupleColumnName, portDf):
    locallogger = logging.getLogger(f"{__name__}.checkPortExistanceInPortDf")
    locallogger.setLevel(logging.DEBUG)
    newPortPatternsSet = set([])
    for index, row in uniquePortTupleDf.iterrows():
        portTuple = row[portTupleColumnName]
        for portString in portTuple:
            if portString != '*':
                matches = re.match(portRangePattern, portString)
                if matches:
                    # portString is in range format.
                    pass
                if not matches:
                    try:
                        int(portString)
                        # portString is a 'number'. skip this round of the for loop.
                        continue
                    except ValueError:
                        # portString is not a 'number'. Check if it is present in the portDf.
                        if not (portString in portDf['Key'].values):
                            newPortPatternsSet.add(portString)
    if len(newPortPatternsSet):
        locallogger.error(f"newNamedPortPatternsSet: {newPortPatternsSet}")  
        sys.exit(1)
    else:
        locallogger.info(f"All named ports exist in portDf") 








def getPortEnrichedData(portTuple, ruleProtocol, portDf, progressTracker=None):
    locallogger = logging.getLogger(f"{__name__}.getPortEnrichedData")
    locallogger.setLevel(logging.DEBUG)
    clearTextPortsSetMatches = set([])
    remoteAccessPortsSetMatches = set([])
    webServicePortsSetMatches = set([])
    portSize = 0
    portDf['Key'] = portDf['Key'].apply(lambda x: str(x))
    #locallogger.debug(f"portTuple: {portTuple}, ruleProtocol: {ruleProtocol}")
    for portGroupString in portTuple:
        if portGroupString == '*':
            portGroupString = '1-65535'

        try:
            portGroupStringNum = int(portGroupString)
        except ValueError:
            portGroupStringNum = -1
            
        matches = re.match(portRangePattern, portGroupString)
        # locallogger.debug(f"portGroupString: {portGroupString}, matches: {matches}")
        if matches:
            # portGroupString is in a port range format
            startPort = int(matches.group(1))
            endPort = int(matches.group(2))
            portRange = range(startPort, endPort+1)
            portSize = portSize + len(portRange)
            for index, row in portDf.iterrows():
                portString = str(row['Key'])
                protocolType = row['Protocol_Type']
                protocolCategory = row['Protocol_Category']
                try:
                    portNumber = int(portString)
                except ValueError:
                    continue
                if portNumber in portRange:
                    if (protocolType in ruleProtocol) or (ruleProtocol == '*'):
                        if protocolCategory == 'Cleartext-Protocol':
                            clearTextPortsSetMatches.add(f"{portString}_{protocolType}")
                        elif protocolCategory == 'Remote-Access-Protocol':
                            remoteAccessPortsSetMatches.add(f"{portString}_{protocolType}")
                        elif protocolCategory == 'Web-Service-Protocol':
                            webServicePortsSetMatches.add(f"{portString}_{protocolType}")
        elif portGroupString in portDf['Key'].values:
            # portGroupString is in a number and is present in the portDf
            filterframe = (portDf['Key'] == portGroupString)
            protocolType = portDf[filterframe]['Protocol_Type'].iloc[0]
            protocolCategory = portDf[filterframe]['Protocol_Category'].iloc[0]
            portSize = portSize + 1
            if (protocolType in ruleProtocol) or (ruleProtocol == '*'):
                if protocolCategory == 'Cleartext-Protocol':
                    clearTextPortsSetMatches.add(f"{portGroupString}_{protocolType}")
                elif protocolCategory == 'Remote-Access-Protocol':
                    remoteAccessPortsSetMatches.add(f"{portGroupString}_{protocolType}") 
                elif protocolCategory == 'Web-Service-Protocol':
                    webServicePortsSetMatches.add(f"{portGroupString}_{protocolType}")
        elif portGroupStringNum != -1:
            # portGroupString is in a number format and is not present in portDf
            portSize = portSize + 1
            
            
        else:
            # portGroupString is in an unrecognizable format
            locallogger.error(f"portGroupString: '{portGroupString}' is in an unrecognizable format.  Exiting...")
            sys.exit(1)            
            
    numClearTextPorts = len(clearTextPortsSetMatches)
    numRemoteAccessPorts = len(remoteAccessPortsSetMatches)
    numWebServicePorts = len(webServicePortsSetMatches)
    dictionary = {
            'D.Number_of_Ports' : int(portSize),
            'D.Number_of_Cleartext_Ports' : int(numClearTextPorts),
            'D.Clear_Text_Port_Matches' : list(clearTextPortsSetMatches),
            'D.Number_of_Remote_Access_Ports' : int(numRemoteAccessPorts),
            'D.Remote_Access_Port_Matches' : list(remoteAccessPortsSetMatches),
            'D.Number_of_Web_Service_Port_Matches' : int(numWebServicePorts),
            'D.Web_Service_Port_Matches' : list(webServicePortsSetMatches),
    }
    if progressTracker:
        progressTracker.progressReport(locallogger)
    return pd.Series(dictionary)   



def getPortEnrichedDataCheckpoint(portTupleSet, portDf, progressTracker=None):
    locallogger = logging.getLogger(f"{__name__}.getPortEnrichedDataCheckpoint")
    locallogger.setLevel(logging.DEBUG)
    clearTextPortsSetMatches = set([])
    remoteAccessPortsSetMatches = set([])
    webServicePortsSetMatches = set([])
    portSize = 0
    allPortsIncluded = False
    portDf['Key'] = portDf['Key'].apply(lambda x: str(x))
    #locallogger.debug(f"portTuple: {portTuple}, ruleProtocol: {ruleProtocol}")
    for portTuple in portTupleSet:
        if portTuple == ('*', '*'):
            portGroupString = '1-65535'
            allPortsIncluded = True
        else:
            portGroupString = portTuple[0]
        ruleProtocol = portTuple[1]
        if ruleProtocol == 'service-tcp':
            ruleProtocol = 'TCP'
        elif ruleProtocol == 'service-udp':
            ruleProtocol = 'UDP'
        
        
        try:
            portGroupStringNum = int(portGroupString)
        except ValueError:
            portGroupStringNum = 0
        
        matchesRange = re.match(portRangePattern, portGroupString)
        # locallogger.debug(f"portGroupString: {portGroupString}, matchesRange: {matchesRange}")
        
        matchesGreaterThan = re.match(portGreaterThanPattern, portGroupString)
        
        
        if matchesRange:
            # portGroupString is in a port range format
            startPort = int(matchesRange.group(1))
            endPort = int(matchesRange.group(2))
            portRange = range(startPort, endPort+1)
            portSize = portSize + len(portRange)
            for index, row in portDf.iterrows():
                portString = str(row['Key'])
                protocolType = row['Protocol_Type']
                protocolCategory = row['Protocol_Category']
                try:
                    portNumber = int(portString)
                except ValueError:
                    locallogger.error(f"portString: '{portString}' in portDf not a number.  Exiting.....")
                    sys.exit(1)
                if portNumber in portRange:
                    if (protocolType in ruleProtocol) or (ruleProtocol == '*'):
                        if protocolCategory == 'Cleartext-Protocol':
                            clearTextPortsSetMatches.add(f"{portString}_{protocolType}")
                        elif protocolCategory == 'Remote-Access-Protocol':
                            remoteAccessPortsSetMatches.add(f"{portString}_{protocolType}")
                        elif protocolCategory == 'Web-Service-Protocol':
                            webServicePortsSetMatches.add(f"{portString}_{protocolType}")
                            
        elif portGroupString in portDf['Key'].values:
            # portGroupString is in a number and is present in the portDf
            filterframe = (portDf['Key'] == portGroupString)
            protocolType = portDf[filterframe]['Protocol_Type'].iloc[0]
            protocolCategory = portDf[filterframe]['Protocol_Category'].iloc[0]
            portSize = portSize + 1
            if (protocolType in ruleProtocol) or (ruleProtocol == '*'):
                if protocolCategory == 'Cleartext-Protocol':
                    clearTextPortsSetMatches.add(f"{portGroupString}_{protocolType}")
                elif protocolCategory == 'Remote-Access-Protocol':
                    remoteAccessPortsSetMatches.add(f"{portGroupString}_{protocolType}") 
                elif protocolCategory == 'Web-Service-Protocol':
                    webServicePortsSetMatches.add(f"{portGroupString}_{protocolType}")
        elif portGroupStringNum != 0:
            # portGroupString is in a number format and is not present in portDf
            portSize = portSize + 1
            
        elif matchesGreaterThan:
            # portGroupString is in a port 'greater than' format: eg: > 1023
            lowerLimitPort = int(matchesGreaterThan.group(1))
            portRange = range((lowerLimitPort + 1), (65535 + 1))
            portSize = portSize + len(portRange)
            for index, row in portDf.iterrows():
                portString = str(row['Key'])
                protocolType = row['Protocol_Type']
                protocolCategory = row['Protocol_Category']
                try:
                    portNumber = int(portString)
                except ValueError:
                    locallogger.error(f"portString: '{portString}' in portDf not a number.  Exiting.....")
                    sys.exit(1)
                if portNumber in portRange:
                    if (protocolType in ruleProtocol) or (ruleProtocol == '*'):
                        if protocolCategory == 'Cleartext-Protocol':
                            clearTextPortsSetMatches.add(f"{portString}_{protocolType}")
                        elif protocolCategory == 'Remote-Access-Protocol':
                            remoteAccessPortsSetMatches.add(f"{portString}_{protocolType}")
                        elif protocolCategory == 'Web-Service-Protocol':
                            webServicePortsSetMatches.add(f"{portString}_{protocolType}")            
            
            
        else:
            # portGroupString is in an unrecognizable format
            locallogger.error(f"portGroupString: '{portGroupString}' is in an unrecognizable format.  Exiting...")
            sys.exit(1)
    numClearTextPorts = len(clearTextPortsSetMatches)
    numRemoteAccessPorts = len(remoteAccessPortsSetMatches)
    numWebServicePorts = len(webServicePortsSetMatches)
    dictionary = {
            'D.Number_of_Ports' : int(portSize),
            'D.Number_of_Cleartext_Ports' : int(numClearTextPorts),
            'D.Clear_Text_Port_Matches' : list(clearTextPortsSetMatches),
            'D.Number_of_Remote_Access_Ports' : int(numRemoteAccessPorts),
            'D.Remote_Access_Port_Matches' : list(remoteAccessPortsSetMatches),
            'D.Number_of_Web_Service_Port_Matches' : int(numWebServicePorts),
            'D.Web_Service_Port_Matches' : list(webServicePortsSetMatches),
            'D.All_Service_Ports_Included' : allPortsIncluded,
    }
    if progressTracker:
        progressTracker.progressReport(locallogger)
    return pd.Series(dictionary)   



def getIPEnrichedDetails(Original_Subnet_Address):
    if (Original_Subnet_Address == '*') or (Original_Subnet_Address == '0.0.0.0/0'):
        ip_type = 'PUBLIC'
        net_length = int(4294967296)
        Corrected_Subnet_Address = Original_Subnet_Address
        Original_Subnet_Address_Correct = True
    else:
        try:
            ip = IP(Original_Subnet_Address, make_net=1)
            ip_type = ip.iptype()    
            net_length = ip.len()
            Corrected_Subnet_Address = ip.strNormal()
            Original_Subnet_Address_Correct = False
            if (Corrected_Subnet_Address == Original_Subnet_Address):
                Original_Subnet_Address_Correct = True
            elif (f"{Corrected_Subnet_Address}/32" == Original_Subnet_Address):
                Original_Subnet_Address_Correct = True
        except ValueError:
            netString = Original_Subnet_Address
            netString = netString.lower()
            netString = netString.strip()
            if netString.startswith('virtualnetwork') or netString.startswith('storage') or netString.startswith('sql') or netString.startswith('azure'):
                ip_type = 'PRIVATE'
            elif netString.startswith('internet'):
                ip_type = 'PUBLIC'
            else:
                ip_type = 'UNKNOWN'
            net_length = 0
            Corrected_Subnet_Address = Original_Subnet_Address
            Original_Subnet_Address_Correct = True
    return str(ip_type), int(net_length), str(Corrected_Subnet_Address), str(Original_Subnet_Address_Correct)
    dictionary = {
         'IP_Type' : str(ip_type),
         'Network_Length' : int(net_length),
         'Corrected_Subnet_Address' : str(Corrected_Subnet_Address),
         'Original_Subnet_Address_Correct' : str(Original_Subnet_Address_Correct),
    }
    return dictionary
    

IP_Address_Pattern = re.compile('([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})\/([0-9]{1,2})')


def getMaskValue(netAddress):
    mask_match = re.findall(IP_Address_Pattern, netAddress)
    if mask_match:
        mask = int(mask_match[0][1])
    else:
        mask = 32    
    return mask


def getCorrectedIPAddress(originalIPAddress):
    try:
        ip = IP(originalIPAddress, make_net=1)
        correctedIPAddress = ip.strNormal()
    except ValueError:
        correctedIPAddress = originalIPAddress
    return correctedIPAddress


def isIPSetPublic(ipSet, ipList=None):
    isPublic = False
    for network in ipSet.iter_cidrs():
        if not network.is_private():
            isPublic = True
            break
    if (not isPublic) and (ipList != None) :
        if 'internet' in str(ipList).lower():
            isPublic = True
    return isPublic




def getIPType(ipSet, ipList=None):
    ipType = 'PRIVATE'
    if (ipSet == IPSet(['0.0.0.0/0'])):
        ipType = 'PUBLIC'
    elif isIPSetPublic(ipSet, ipList):
        ipType = 'PUBLIC'
    return ipType





def getIntersectionIPSet(addressIPSet, allowedPublicIPAddressesIPSet, progressTracker=None):
    locallogger = logging.getLogger(f"{__name__}.getIntersectionIPSet")
    locallogger.setLevel(logging.DEBUG)
    if progressTracker:
        progressTracker.progressReport(locallogger)
    return (addressIPSet & allowedPublicIPAddressesIPSet)



def getFullNetworkIPSet(networkRefDf, progressTracker=None):
    locallogger = logging.getLogger(f"{__name__}.getFullNetworkIPSet")
    locallogger.setLevel(logging.DEBUG)
    internalNetworkFullIPset = IPSet([])
    for index, row in networkRefDf.iterrows():
        internalNetworkFullIPset = internalNetworkFullIPset | row['IP_Addresses_IPSet']
        if progressTracker:
            progressTracker.progressReport(locallogger)
    return internalNetworkFullIPset




