"""
    mlmddr.py

    Version:: 1.0.30 10.31.22 Update mlmddr.py to use consistent paths for infra-files  
    Version:: 1.0.29 10.25.22 Update mlmddr.py for xe config and python install fixes  
    Version:: 1.0.26 10.13.22 Update mlmddr.py for ddr-clips 1.1.7
    
    This module implements the integration with RADKit used to manage the DDR usecase lifecycle.
    The radkit_client package includes functions used to securely interact with devices using the 
    RADKit connector.
    
    Users can implement python scripts using mlmddr functions to perform the following actions
    on devices:
    
        deploy_xe_infra - 
            Copy all python modules required to run DDR to the devices and install the
            the packages on the device.  Python packages are transferred to devices
            using scp.  The packages are deployed as .whl files including all dependencies
            so that the pip install run on the device does not require access to external
            repositories.  The deploy_xe_infra method for xe deploys and initials the device
            guestshell environment where the ddr usecases execute.  The deploy_infra method 
            is updated with new python .whl files when ddr-clips ro ddr-python are updated
                       
        deploy_xe_configs - 
            ddr when running on XE platforms requires configurations to enable IOS features
            used to enable asynchronous notifications generated by feature code to trigger
            ddr usecase execution.  The XE platform uses the CISCO-SYSLOG-MIB to define the
            structure of messages included in NETCONF notifications.  When these configurations
            are applied to the device SYSLOG messages can be used to trigger NETCONF notifications
            and trigger DDR usecase execution.  If this function is called by a user script
            during usecase deployment, the existing IOS configuration is saved before changes are
            made.  If the restore_xe_config function is called, the IOS configuration present
            before calling deploy_xe_configs is restored

        deploy_xe_usecase - 
            This function deploys the ddr usecase to the device.  DDR usecases are deployed into 
            directories in the /bootflash/guest-share folder.  The guest-share folder is accessible
            to python scripts running in the guestshell.  These guest-share directory is also
            accessible from off the device.  Each DDR usecase is stored in a separate directory in
            /bootflash/guest-share/ddr/USECASE_NAME, DDR-python usecases are implemented as
            a single python script "usecase_name.py".  DDR-clips usecases are implemented by a collection of
            files: ddr-facts, ddr-rules, ddr-devices, ddr-flags, ddr-sim, ddr-control, ddr-parsers.
            The usecase directories remain on the device after DDR is removed from the device.  Each usecase
            directory includes the log files generated by the execution of the usecase
            
        execute_xe_use_case -
            This function executes the ddr usecase.  For DDR-python usecases the usecase_name.py script is
            executed.  For DDR-clips usecases the ddrrun.py script copied into the usecase directory is
            executed.  The ddrrun.py script uses the files: ddr-facts, ddr-rules, ddr-devices, ddr-flags, 
            ddr-sim, ddr-control to control usecase execution.  If the ddr-parsers file is included
            when the usecase is deployed, the content of ddr-parsers is copied into genie_parsers.py.
            When genie_parsers.py is present in the usecase directory, the text parsers in this file
            are used instead of the general purpose parsers included in the ddr-clips package
            
        collect_xe_data -
            This function returns the most recent DDR log file generated by the DDR usecase in the
            Python dictionary format.  The log content is returned into a variable when the function is
            called
            
        collect_xe_log -
            This function returns the name of the most recent DDR log file generated by the DDR usecase

"""

from enum import Enum
from os import path
from os.path import exists
import time
import re
import json
from typing import Optional, Union, List, Dict
#
# radkit_client functions used to deploy and control usecase execution
#
from radkit_common.nglog import debug, warning, info, error, basicConfig, DEBUG
from radkit_client.client import *
from radkit_client.device import DeviceDict, Device
import radkit_client.helpers as lmerrors
#
# DDR module used to support DDR usecase content transfer to devices
#
from lmautomation import Automation, scp_upload_from_buffer, Model
#
# definitions used for generating error messages consistent with RADKit infrastructure
#
basicConfig()

class Color:
    PURPLE = "\033[95m"
    CYAN = "\033[96m"
    DARKCYAN = "\033[36m"
    BLUE = "\033[94m"
    GREEN = "\033[92m"
    YELLOW = "\033[93m"
    RED = "\033[91m"
    BOLD = "\033[1m"
    UNDERLINE = "\033[4m"
    END = "\033[0m"


def deploy_xe_infra(device_list: DeviceDict, usecase: Automation) -> None:
    """
            Copy all python modules required to run DDR to the devices and install the
            the packages on the device.  Python packages are transferred to devices
            using scp.  The packages are deployed as .whl files including all dependencies
            so that the pip install run on the device does not require access to external
            repositories.  The deploy_xe_infra method for xe deploys and initials the device
            guestshell environment where the ddr usecases execute.  
            
            The deploy_xe_infra method must be updated with new python .whl files when 
            ddr-clips or ddr-python are updated with names of new .whl files
   
            :param device_list: radkit_client DeviceDict object containing one or more device identifies
            :param usecase: Automation object defined in lmautomation.py with usecase definition content
        
        Usage::

              deploy_xe_infra(devices, usecase_object)

        :raises none:
    """

    info("DDR XE Infra Deploy: DDR Infra Deployment Started")

    flow = lmerrors.DeviceFlow(device_list)
       
    ###################
    # CONFIG SCP SERVER
    ###################

    info("DDR XE Infra Deploy: enable scp")
    scp_con = flow.exec_wait(["config terminal", "ip scp server enable", "exit"], timeout=300)
    info("DDR XE Infra Deploy: enable scp complete")

    ###################
    # CONFIG GUESTSHELL
    ###################

    info("DDR XE Infra Deploy: configure guestshell")
    gs_con = flow.exec_wait(["config terminal", "iox", "app-hosting appid guestshell", "app-vnic management guest-interface 0", "end"], timeout=600)
    info("DDR XE Infra Deploy: configure guestshell complete")

    #################################
    # ENABLE GUESTSHELL IN THE DEVICE
    #################################

    try:
        info(
            "DDR XE Infra Deploy: DDR Guestshell Installation Started, may take few minutes"
        )
        g_enable = flow.exec_wait("guestshell enable", timeout=1000)
    except Exception as e:
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR XE Infra Deploy: An Exception occured in the section enable guestshell in the device"
            + " "
            + str(e)
            + Color.END
        )


    ################################################
    # VERIFY THE GUESTSHELL IS ENABLED IN THE DEVICE
    ################################################

    try:
        for result in g_enable.result.values():
            g_enable_list = result.data.split("\n")

        if "Guestshell enabled successfully" in g_enable_list:
            info("DDR XE Infra Deploy: Guestshell Enabled Successfully")
        else:
            raise ValueError(
                "DDR XE Infra Deploy: Guestshell not enabled,, terminating the session\n"
            )
    except Exception as e:
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR XE Infra Deploy: An Exception occured in the section verify guestshell in the device"
            + " "
            + str(e)
            + Color.END
        )
                       
    #################################
    # CONFIG THE Passwordless NETCONF
    #################################

    try:
        info("DDR XE Infra Deploy: enable passwordless NETCONF")
        ddr_engine_ins = flow.exec_wait(
            ["guestshell", "iosp_client -c 'netconf-yang' -f netconf_enable guestshell 830", "iosp_client -f netconf_enable_passwordless guestshell guestshell", "exit"],
            exec_error_regex=[],
        )
        info("DDR XE Infra Deploy: enable passwordless NETCONF complete")

    except Exception as e:
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR Infra Deploy: An Exception occured in Passwordless NETCONF enable in the guestshell"
            + " "
            + str(e)
            + Color.END
        )
        
    #######################
    # CREATE the /bootflash/guest-share/ddr directory for DDR execution
    #######################

    try:
        info("DDR XE Infra Deploy: create guest-share ddr directory")
        ddr_engine_ins = flow.exec_wait(
            ["guestshell", "cd /bootflash", "mkdir guest-share", "cd guest-share", "mkdir ddr", "exit"],
            exec_error_regex=[],
        )
        info("DDR XE Infra Deploy: create guest-share ddr directory complete")

    except Exception as e:
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR Infra Deploy: An Exception occured creating /bootflash/guest-share/ddr directory"
            + " "
            + str(e)
            + Color.END
        )
    
    ###################################
    # COPY THE DDR ENGINE TO THE DEVICE
    ###################################
    try:
        basedir_infr_files = "./files/xe/infra-files"
        run_file = path.join(basedir_infr_files, "ddrrun.py")
        pipfiles = path.join(basedir_infr_files, "pipfiles.sh")
        xe_commands = path.join(basedir_infr_files, "xe_commands.py")
        ddrkill = path.join(basedir_infr_files, "ddrkill.sh")

        w1 = path.join(basedir_infr_files, "pip-21.3.1-py3-none-any.whl")
        w2 = path.join(basedir_infr_files, "ddr_python-1.0.9-py3-none-any.whl")
        w3 = path.join(basedir_infr_files, "ptyprocess-0.7.0-py2.py3-none-any.whl")
        w4 = path.join(basedir_infr_files, "pexpect-4.8.0-py2.py3-none-any.whl")
        w5 = path.join(basedir_infr_files, "ncclient-0.6.9-py2.py3-none-any.whl")
        w6 = path.join(basedir_infr_files, "xmltodict-0.13.0-py2.py3-none-any.whl")
        w7 = path.join(basedir_infr_files, "regex-2022.6.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl")
        w8 = path.join(basedir_infr_files, "ddr_clips-1.1.7-py3-none-any.whl")
        w9 = path.join(basedir_infr_files, "clipspy-1.0.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl")
        w10 = path.join(basedir_infr_files, "clips-1.1.0-py3-none-any.whl")
        GROUP1 = True
        GROUP2 = True
        for device in flow.active_devices.values():
            info("DDR XE Infra Deploy: Copy DDR Python libraries to devices")
            if GROUP1:
            
                d_file = device.scp_upload_from_file(
                run_file, "bootflash:/guest-share/ddr/ddrrun.py").wait()

                pip_file = device.scp_upload_from_file(
                pipfiles, "bootflash:/guest-share/ddr/pipfiles.sh").wait()

                xe_commands = device.scp_upload_from_file(
                xe_commands, "bootflash:/guest-share/ddr/xe_commands.py").wait()

                ddrkill_command = device.scp_upload_from_file(
                ddrkill, "bootflash:/guest-share/ddr/ddrkill.sh").wait()

                w2_file = device.scp_upload_from_file(
                w2, "bootflash:/guest-share/ddr/ddr_python-1.0.9-py3-none-any.whl").wait()

                w3_file = device.scp_upload_from_file(
                w3, "bootflash:/guest-share/ddr/ptyprocess-0.7.0-py2.py3-none-any.whl").wait()

                w4_file = device.scp_upload_from_file(
                w4, "bootflash:/guest-share/ddr/pexpect-4.8.0-py2.py3-none-any.whl").wait()

                w5_file = device.scp_upload_from_file(
                w5, "bootflash:/guest-share/ddr/ncclient-0.6.9-py2.py3-none-any.whl").wait()

                w6_file = device.scp_upload_from_file(
                w6, "bootflash:/guest-share/ddr/xmltodict-0.13.0-py2.py3-none-any.whl").wait()
                
                w8_file = device.scp_upload_from_file(
                w8, "bootflash:/guest-share/ddr/ddr_clips-1.1.7-py3-none-any.whl").wait()

                w10_file = device.scp_upload_from_file(
                w10, "bootflash:/guest-share/ddr/clips-1.1.0-py3-none-any.whl").wait()
            
            if GROUP2:
                w7_file = device.scp_upload_from_file(
                w7, "bootflash:/guest-share/ddr/regex-2022.6.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl").wait()

                w9_file = device.scp_upload_from_file(w9, "bootflash:/guest-share/ddr/clipspy-1.0.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl").wait()

                w1_file = device.scp_upload_from_file(w1, "bootflash:/guest-share/ddr/pip-21.3.1-py3-none-any.whl").wait()

            info("DDR XE Infra Deploy: Copy DDR Python libraries complete")

    except Exception as e:
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR XE Infra Deploy: An Exception occured in the section copy the ddr engine files - uploading to bootflash"
            + " "
            + str(e)
            + Color.END
        )

    ##########################################
    # INSTALL PYTHON PACKAGES AND DEPENDENCIES
    ##########################################

    try:
        info("DDR XE Infra Deploy: Install Platform Independent DDR Python Packages and Dependencies")
        export_ins1 = flow.exec_wait(
                [
                    "guestshell",
                    "python3 -m pip install /bootflash/guest-share/ddr/ddr_python-1.0.9-py3-none-any.whl --user",
                    "python3 -m pip install /bootflash/guest-share/ddr/ptyprocess-0.7.0-py2.py3-none-any.whl --user",
                    "python3 -m pip install /bootflash/guest-share/ddr/pexpect-4.8.0-py2.py3-none-any.whl --user",
                    "python3 -m pip install /bootflash/guest-share/ddr/ncclient-0.6.9-py2.py3-none-any.whl --user",
                    "python3 -m pip install /bootflash/guest-share/ddr/xmltodict-0.13.0-py2.py3-none-any.whl --user",
                    "python3 -m pip install /bootflash/guest-share/ddr/ddr_clips-1.1.7-py3-none-any.whl --user",
                    "python3 -m pip install /bootflash/guest-share/ddr/clips-1.1.0-py3-none-any.whl --user",
                    "exit"],
                exec_error_regex=[],
        )

    #
    # Install python modules that require an updated pip install using a shell script
    # 
        ddr_engine_ins = flow.exec_wait(
                "guestshell run bash /bootflash/guest-share/ddr/pipfiles.sh",
                timeout=10000,
        )

        info("DDR XE Infra Deploy: Install DDR Python Packages and Dependencies complete")

    except Exception as e:
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR XE Infra Deploy: An Exception occured in the section Install DDR Python Packages and Dependencies - 2"
            + " "
            + str(e)
            + Color.END
        )

    info("DDR XE Infra Deploy: DDR Infra Deployment Complete")

def deploy_xe_configs(device_list: DeviceDict, usecase: Automation) -> None:
    """
            ddr when running on XE platforms requires configurations to enable IOS features
            used to enable asynchronous notifications generated by feature code to trigger
            ddr usecase execution.  The XE platform uses the CISCO-SYSLOG-MIB to define the
            structure of messages included in NETCONF notifications.  When these configurations
            are applied to the device SYSLOG messages can be used to trigger NETCONF notifications
            and trigger DDR usecase execution.  If this function is called by a user script
            during usecase deployment, the existing IOS configuration is saved before changes are
            made.  If the restore_xe_config function is called, the IOS configuration present
            before calling deploy_xe_configs is restored

            :param device_list: radkit_client DeviceDict object containing one or more device identifies
            :param usecase: Automation object defined in lmautomation.py with usecase definition content
        
        Usage::

              deploy_xe_configs(devices, usecase_object)

        :raises none:

    """
    
    flow = lmerrors.DeviceFlow(device_list)
    
    ###########################
    # DDR INITIAL BASIC CONFIGS
    ###########################

    # Configuring IOS
    info("DDR XE Infra Deploy: Add configurations for NETCONF notifications")

    try:
        ios_config1 = flow.exec_wait(
            [
                "delete /force flash:saved-before-ddr-configuration",
                "copy running-config flash:saved-before-ddr-configuration",
                "\n",
                "\n"
            ],
            timeout=300
        )

    #
    # Install IOS-XE configs required to enable using NETCONF notifications
    # 
        ios_config2 = flow.exec_wait(
                "guestshell run python3 /bootflash/guest-share/ddr/xe_commands.py",
                timeout=10000
        )

    except Exception as e:
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR XE Configs Deploy: "
            + " "
            + str(e)
            + Color.END
        )

    if "FAILURE" in str(ios_config1):
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR XE Config Save: Failed: ~/.radkit/session_logs/service/XXXXXX.log "
            + "\n"
            + str(ios_config1)
            + Color.END
        )

    if "FAILURE" in str(ios_config2):
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR XE NETCONF Notification Configs Deploy: Failed: ~/.radkit/session_logs/service/XXXXXX.log"
            + "\n"
            + str(ios_config2)
            + Color.END
        )

    info("DDR XE Infra Deploy: Add configurations for NETCONF notifications complete")

def deploy_xe_use_case(device_list: DeviceDict, usecase: Automation) -> None:
    """
            This function deploys the ddr usecase to the device.  DDR usecases are deployed into 
            directories in the /bootflash/guest-share folder.  The guest-share folder is accessible
            to python scripts running in the guestshell.  These guest-share directory is also
            accessible from off the device.  Each DDR usecase is stored in a separate directory in
            /bootflash/guest-share/ddr/USECASE_NAME, DDR-python usecases are implemented as
            a single python script "usecase_name.py".  DDR-clips usecases are implemented by a collection of
            files: ddr-facts, ddr-rules, ddr-devices, ddr-flags, ddr-sim, ddr-control, ddr-parsers.
            The usecase directories remain on the device after DDR is removed from the device.  Each usecase
            directory includes the log files generated by the execution of the usecase

            :param device_list: radkit_client DeviceDict object containing one or more device identifies
            :param usecase: Automation object defined in lmautomation.py with usecase definition content
        
        Usage::

              deploy_xe_/ddrdevices, usecase_object)

        :raises none:
    """

    flow = lmerrors.DeviceFlow(device_list)

    ######################################
    # UPLOAD THE DDR FILES INTO THE DEVICE
    ######################################

    info("DDR XE Workflow Deploy: Start DDR Workflow deployment to device")

    try:
        if usecase.model is Model.CLIPS:

            guestshell = flow.exec_wait(
                [
                    "guestshell",
                    "cd /bootflash/guest-share/ddr/",
                    f"mkdir {usecase.name}",
                    "exit",
                ],
                timeout=300,
            )

        # Required ddr CLIPs control files
        
            debug("Deploying files")
            f1 = "bootflash:/guest-share/ddr/{}/ddr-devices".format(usecase.name)
            f2 = "bootflash:/guest-share/ddr/{}/ddr-facts".format(usecase.name)
            f3 = "bootflash:/guest-share/ddr/{}/ddr-flags".format(usecase.name)
            f4 = "bootflash:/guest-share/ddr/{}/ddr-rules".format(usecase.name)
            f5 = "bootflash:/guest-share/ddr/{}/ddr-sim".format(usecase.name)
            f6 = "bootflash:/guest-share/ddr/{}/ddr-control".format(usecase.name)
            f7 = "bootflash:/guest-share/ddr/{}/ddr-parsers".format(usecase.name)

            debug("Deploying files")

            basedir_usecase_files = "./files/clips-files/"
            usecase_path = path.join(basedir_usecase_files, f"{usecase.name}/")

            for device in flow.active_devices.values():

                device.scp_upload_from_file(
                    usecase_path + "ddr-devices", f"bootflash:/guest-share/ddr/{usecase.name}/ddr-devices")
                device.scp_upload_from_file(
                    usecase_path + "ddr-facts", f"bootflash:/guest-share/ddr/{usecase.name}/ddr-facts")
                device.scp_upload_from_file(
                    usecase_path + "ddr-flags", f"bootflash:/guest-share/ddr/{usecase.name}/ddr-flags")
                device.scp_upload_from_file(
                    usecase_path + "ddr-rules", f"bootflash:/guest-share/ddr/{usecase.name}/ddr-rules")
                device.scp_upload_from_file(
                    usecase_path + "ddr-sim", f"bootflash:/guest-share/ddr/{usecase.name}/ddr-sim")
                device.scp_upload_from_file(
                    usecase_path + "ddr-control", f"bootflash:/guest-share/ddr/{usecase.name}/ddr-control")
            try:
                device.scp_upload_from_file(
                    usecase_path + "ddr-parsers", f"bootflash:/guest-share/ddr/{usecase.name}/ddr-parsers").wait()

            except: pass

            guestshell = flow.exec_wait(
                [
                    "guestshell",
                    f"cd /bootflash/guest-share/ddr/{usecase.name}",
                    "cp ddr-parsers genie_parsers.py",
                    "cp ../ddrrun.py .",
                    "exit"
                ],
                timeout=300
            )
            
        else: # usecase.model is Model.PYTHON:

            guestshell = flow.exec_wait(
                [
                    "guestshell",
                    "cd /bootflash/guest-share/ddr/",
                    f"mkdir {usecase.name}",
                    "exit",
                ],
                timeout=300,
            )

            for device in flow.active_devices.values():

                basedir_usecase_files = "./files/py-files/"
                script = f"{usecase.name}.py"
                usecase_file = path.join(basedir_usecase_files, f"{usecase.name}/{script}")
                upload_usecase = device.scp_upload_from_file(
                    usecase_file, f"bootflash:/guest-share/ddr/{usecase.name}/{script}"
                ).wait()
    
    #
    # copy ddr-parsers file if present into the DDR-Python usecase directory
    # Name the file genie_parsers.py
    # If this file is present it will override the genie_parsers.py installed with the ddr-python package
    #
    
                basedir_usecase_files = "./files/py-files/"
                parser_file = path.join(basedir_usecase_files, f"{usecase.name}/ddr-parsers")
                if exists(parser_file):
                    upload_parsers = device.scp_upload_from_file(
                    parser_file, f"bootflash:/guest-share/ddr/{usecase.name}/genie_parsers.py"
                ).wait()
                time.sleep(5)       
    except Exception as e:
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR XE Use Case Files Deploy: An Exception occured in the section upload the ddr files into the device"
            + " "
            + str(e)
            + Color.END
        )

    info("DDR XE Workflow Deploy: DDR Workflow deployment to device complete")
  
def execute_xe_use_case(device_list: DeviceDict, usecase: Automation) -> None:
    """
            This function executes the ddr usecase.  For DDR-python usecases the usecase_name.py script is
            executed.  For DDR-clips usecases the ddrrun.py script copied into the usecase directory is
            executed.  The ddrrun.py script uses the files: ddr-facts, ddr-rules, ddr-devices, ddr-flags, 
            ddr-sim, ddr-control to control usecase execution.  If the ddr-parsers file is included
            when the usecase is deployed, the content of ddr-parsers is copied into genie_parsers.py.
            When genie_parsers.py is present in the usecase directory, the text parsers in this file
            are used instead of the general purpose parsers included in the ddr-clips package

            :param device_list: radkit_client DeviceDict object containing one or more device identifies
            :param usecase: Automation object defined in lmautomation.py with usecase definition content
        
        Usage::

              execute_xe_usecase(devices, usecase_object)

        :raises none:
    """

    flow = lmerrors.DeviceFlow(device_list)

    try:
        if usecase.model is Model.CLIPS:
            info(f"DDR Use Case Execute: Executing DDR Clips Use Case: {usecase.name}")
            ddr_execute = flow.exec_wait(
                f"guestshell run python3 /bootflash/guest-share/ddr/{usecase.name}/ddrrun.py",
                timeout=10000,
            )
            
        else: # usecase.model is Model.PYTHON:
            script = f"{usecase.name}.py"
            info(f"DDR Use Case Execute: Executing DDR Python Use Case: {usecase.name}/{script}")
            ddr_execute = flow.exec_wait(
                f"guestshell run python3 /bootflash/guest-share/ddr/{usecase.name}/{script}",
                timeout=10000,
            )

        info("DDR Use Case Execute: DDR Use Case Execution Completed")
    except Exception as e:
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR Use Case Execute: An Exception occured in the section ddr use case execute"
            + " "
            + str(e)
            + Color.END
        )

def collect_xe_data(device_list: DeviceDict, usecase: Automation) -> Union[dict, None]:
    """
            This function returns the most recent DDR log file generated by the DDR usecase in the
            Python dictionary format.  The log content is returned into a variable when the function is
            called
        
        Usage::

              collect_xe_data(devices, usecase_object)

        :raises none:
    """

    info("DDR Collect XE Data: Save Use Case Results")
    flow = lmerrors.DeviceFlow(device_list)
    ######################################################
    # OBTAIN THE DDR SERVICE IMPACT NOTIFICATION LOG FILES
    ######################################################
#
# DDR Clips usecase logs are stored in the use case directory
# Get the most recent use case log which will be the first in the directory
# Return the path to the most recent usecase log
#
# ddmi-cat9300-2#dir flash:guest-share/ddr/ddr_xe_interfaces
# Directory of flash:guest-share/ddr/ddr_xe_interfaces/
# 
# 622697  -rw-            15355   Oct 5 2022 15:53:43 +00:00  ddr_xe_interfaces_TS_TS_10-05-2022_15_53_40.896204.1
# 622693  -rw-            16201   Oct 5 2022 15:53:43 +00:00  ddr_xe_interfaces_TS_TS_10-05-2022_15_53_40.896204
#
# The use case filename is the last token in the line
#
    try:
        log_path = "bootflash:/guest-share"

        if usecase.model is Model.CLIPS:
            log_files = f"dir bootflash:/guest-share/ddr/{usecase.name}"
            all_files = flow.exec_wait(log_files, timeout=100)
            for result in all_files.result.values():
                all_files_list = result.data.split("\n")

            pattern = ".*_TS_.*"
            for ddr_log in all_files_list:
                result = re.match(pattern, ddr_log)
                if result:
                    dir_tokens = ddr_log.split(" ")
                    filename = dir_tokens[-1:]
                    log_path = f"more bootflash:/guest-share/ddr/{usecase.name}/" + str(filename[0])
                    log_content = flow.exec_wait(log_path, timeout=100)
                    break # Exit after selecting most recent use case log

#
# DDR Python usecase logs are stored in the use case directory
# Get the most recent use case log which will be the first in the directory
# Return the path to the most recent usecase log
#
# ddmi-cat9300-2#dir flash:guest-share/ddr/ddr_high_cpu
# Directory of flash:guest-share/ddr/ddr_high_cpu/
# 
# 622665  -rw-           305252   Oct 5 2022 15:28:53 +00:00  ddr_high_cpu_LOG_TS_10-05-2022_15:28:38.928490
# 622661  -rw-           305251   Oct 5 2022 15:26:40 +00:00  ddr_high_cpu_LOG_TS_10-05-2022_15:26:25.914357
# 622660  -rw-           304352   Oct 5 2022 15:07:33 +00:00  ddr_high_cpu_LOG_TS_10-05-2022_15:07:19.162174
# 622659  -rw-             4583   Oct 5 2022 15:06:11 +00:00  ddr_high_cpu.py
#
# The use case filename is the last token in the line
#
        elif usecase.model is Model.PYTHON:
            log_files = f"dir bootflash:/guest-share/ddr/{usecase.name}"
            all_files = flow.exec_wait(log_files, timeout=100)
            for result in all_files.result.values():
                all_files_list = result.data.split("\n")

            pattern = ".*_TS_.*"
            for ddr_log in all_files_list:
                result = re.match(pattern, ddr_log)
                if result:
                    dir_tokens = ddr_log.split(" ")
                    filename = dir_tokens[-1:]
                    log_path = f"more bootflash:/guest-share/ddr/{usecase.name}/" + str(filename[0])
                    log_content = flow.exec_wait(log_path, timeout=100)
                    break # Exit after selecting most recent use case log
            
    except Exception as e:
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR Collect XE Data: An Exception occured in the section collect the use case logs: Verify DDR infra deployed correctly"
            + " "
            + str(e)
            + Color.END
        )

    info(f"DDR Collect XE Data: DDR Use Case Execution Logs saved in /bootflash/guest-share/ddr/{usecase.name}/")

    ###############################################
    # PRINT THE DDR SERVICE IMPACT NOTIFICATION LOG
    ###############################################

    try:
        if log_path:
            action_func = flow.exec_wait(log_path, timeout=10000)
            if usecase.model is Model.CLIPS:
                return {
                    device.name: json.loads(
                        json.dumps("".join(action_func.result[device.name].data.split("\n")[1:]))
                    )
                    for device in flow.active_devices.values()
                }
            elif usecase.model is Model.PYTHON:
                return {
                    device.name: json.loads(
                        json.dumps("".join(action_func.result[device.name].data.split("\n")[1:]))
                    )
                    for device in flow.active_devices.values()
                }
            else:
                info(f"DDR Collect XE Data: Model type passed is invalid")
                raise ValueError("Invalid model")
                return {}
        else:
            info (f"DDR Collect XE Data: No DDR Use Case Log Stored")
            raise ValueError("No DDR Use Case Stored")

        info("DDR Collect XE Data: DDR Collect Data Execution Ended")
    except Exception as e:
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR Collect XE Data: An Exception occured in the section print the DDR use case logs"
            + " "
            + str(e)
            + Color.END
        )

def collect_xe_log(device_list: DeviceDict, usecase: Automation) -> Union[dict, None]:
    """
        This function returns the name of the most recent DDR log file generated by the DDR usecase
        
        Usage::

              deploy_xe_log(devices, usecase_object)

        :raises none:
    """

    info("DDR Collect XE Data: Save Use Case Results")
    flow = lmerrors.DeviceFlow(device_list)
    ######################################################
    # OBTAIN THE DDR SERVICE IMPACT NOTIFICATION LOG FILES
    ######################################################
#
# DDR Clips usecase logs are stored in the use case directory
# Get the most recent use case log which will be the first in the directory
# Return the path to the most recent usecase log
#
# ddmi-cat9300-2#dir flash:guest-share/ddr/ddr_xe_interfaces
# Directory of flash:guest-share/ddr/ddr_xe_interfaces/
# 
# 622697  -rw-            15355   Oct 5 2022 15:53:43 +00:00  ddr_xe_interfaces_TS_TS_10-05-2022_15_53_40.896204.1
# 622693  -rw-            16201   Oct 5 2022 15:53:43 +00:00  ddr_xe_interfaces_TS_TS_10-05-2022_15_53_40.896204
#
# The use case filename is the last token in the line
#

    try:
        log_path = "bootflash:/guest-share"

        if usecase.model is Model.CLIPS:
            log_files = f"dir bootflash:/guest-share/ddr/{usecase.name}"
            all_files = flow.exec_wait(log_files, timeout=100)
            for result in all_files.result.values():
                all_files_list = result.data.split("\n")

            pattern = ".*_TS_.*"
            for ddr_log in all_files_list:
                result = re.match(pattern, ddr_log)
                if result:
                    dir_tokens = ddr_log.split(" ")
                    filename = dir_tokens[-1:]
                    filestring = filename[0]
                    log_path = f"bootflash:/guest-share/ddr/{usecase.name}/" + str(filename[0])
                    info(f"DDR Collect XE Log: DDR Use Case Execution Logs saved in /bootflash/guest-share/ddr/{usecase.name}/{filestring}")
                    return log_path
#
# DDR Python usecase logs are stored in the use case directory
# Get the most recent use case log which will be the first in the directory
# Return the path to the most recent usecase log
#
# ddmi-cat9300-2#dir flash:guest-share/ddr/ddr_high_cpu
# Directory of flash:guest-share/ddr/ddr_high_cpu/
# 
# 622665  -rw-           305252   Oct 5 2022 15:28:53 +00:00  ddr_high_cpu_LOG_TS_10-05-2022_15:28:38.928490
# 622661  -rw-           305251   Oct 5 2022 15:26:40 +00:00  ddr_high_cpu_LOG_TS_10-05-2022_15:26:25.914357
# 622660  -rw-           304352   Oct 5 2022 15:07:33 +00:00  ddr_high_cpu_LOG_TS_10-05-2022_15:07:19.162174
# 622659  -rw-             4583   Oct 5 2022 15:06:11 +00:00  ddr_high_cpu.py
#
# The use case filename is the last token in the line
#
        elif usecase.model is Model.PYTHON:
            log_files = f"dir bootflash:/guest-share/ddr/{usecase.name}"
            all_files = flow.exec_wait(log_files, timeout=10000)
            for result in all_files.result.values():
                all_files_list = result.data.split("\n")

            pattern = ".*_TS_.*"
            for ddr_log in all_files_list:
                result = re.match(pattern, ddr_log)
                if result:
                    dir_tokens = ddr_log.split(" ")
                    filename = dir_tokens[-1:]
                    filestring = filename[0]
                    log_path = f"bootflash:/guest-share/ddr/{usecase.name}/" + str(filename[0])
                    info(f"DDR Collect XE Log: DDR Use Case Execution Logs saved in /bootflash/guest-share/ddr/{usecase.name}/{filestring}")
                    return log_path
            
    except Exception as e:
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR Collect XE Log: An Exception occured in the section to collect the use case log name"
            + " "
            + str(e)
            + Color.END
        )

def collect_data(device_list: DeviceDict, usecase: Automation) -> Union[dict, None]:

    """
    Collect the service impact notification logs generated by the DDR engine in the inventory passed as parameter.

    :device_list: an DeviceDictionary containing the list of devices on which the DDR infra needs to be deployed

    :returns: nothing
    :raises: Lazy Meastro exception in case the connectivity is fully lost during deployment.
    """

    info("DDR Collect Data: DDR Collect Data Execution Started\n")
    flow = lmerrors.DeviceFlow(device_list)
    
    #######################################################
    # EXECUTE THE OS INFRA BASED ON THE OS VERSION DETECTED
    #######################################################

    try:
        xedevices = []
        flow.exec_wait("\x1a")
        ver_check = flow.exec_wait("show version", timeout=300)
        for device in flow.active_devices.values():
            if "Cisco IOS XE Software" in ver_check.result[device.name].data:
                xedevices.append(device.name)
                device.attributes.ephemeral['os']='iosxe'
            else:
                raise ValueError(
                f"DDR Infra Deploy: Valid Cisco OS Version not detected in device - {device.name}, terminating the session\n"
            )

        xe_dict = flow.active_devices.subset(xedevices)

        if xe_dict:
            info(collect_xe_data(xe_dict, usecase))

    except Exception as e:
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR Collect Data: An Exception occured in the section verify cisco os version on the device"
            + " "
            + str(e)
            + Color.END
        )


def cleanup_infra(device_list: DeviceDict):

    """
    Cleans the DDR infra on all the devices in the inventory passed as parameter.

    :device_list: an DeviceDictionary containing the list of devices on which the DDR infra needs to be deployed

    :returns: nothing
    :raises: Lazy Meastro exception in case the connectivity is fully lost during deployment.
    """
    info("DDR Clean Deploy: DDR Clean Deployment Started")
    flow = lmerrors.DeviceFlow(device_list)

    #####################################
    # REMOVE THE DDR FILES FROM THE FLASH
    #####################################

    try:
        delete_files = flow.exec_wait(
            [
                "delete /force flash:guest-share/ddr/ddrrun.py",
                "delete /force flash:guest-share/ddr/pipfiles.sh",
                "delete /force flash:guest-share/ddr/xe_commands.py",
                "delete /force flash:guest-share/ddr/ddrkill.sh",
                "delete /force flash:guest-share/ddr/xmltodict-0.13.0-py2.py3-none-any.whl",
                "delete /force flash:guest-share/ddr/ddr_python-1.0.9-py3-none-any.whl",
                "delete /force flash:guest-share/ddr/ptyprocess-0.7.0-py2.py3-none-any.whl",
                "delete /force flash:guest-share/ddr/pexpect-4.8.0-py2.py3-none-any.whl",
                "delete /force flash:guest-share/ddr/ncclient-0.6.9-py2.py3-none-any.whl",
                "delete /force flash:guest-share/ddr/xmltodict-0.13.0-py2.py3-none-any.whl",
                "delete /force flash:guest-share/ddr/ddr_clips-1.1.7-py3-none-any.whl",
                "delete /force flash:guest-share/ddr/clips-1.1.0-py3-none-any.whl",
                "delete /force flash:guest-share/ddr/pip-21.3.1-py3-none-any.whl",
                "delete /force flash:guest-share/ddr/clipspy-1.0.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
                "delete /force flash:guest-share/ddr/regex-2022.6.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
            ],
            exec_error_regex=[]
        )

    except Exception as e:
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR Clean Deploy: An Exception occured in the section remove the ddr files from the flash"
            + " "
            + str(e)
            + Color.END
        )

    info(f"DDR Remove: DDR Runtime is removed from device")


def restore_xe_config(device_list: DeviceDict):

    ###################################################
    # COPY THE RUNNING CONFIGS BEFORE THE DDR INSTALLED
    ###################################################

    flow = lmerrors.DeviceFlow(device_list)

    try:
        copy_run_configs = flow.exec_wait(
            ["config replace flash:saved-before-ddr-configuration force",
             "delete /force flash:saved-before-ddr-configuration"
            ],
            exec_error_regex=[],
        )
        info("DDR Device running-config restored: DDR applied configurations removed")
    except Exception as e:
        raise Exception (
            Color.BOLD
            + Color.RED
            + f"DDR Clean Deploy: An Exception occured in restore_xe_config"
            + " "
            + str(e)
            + Color.END
        )
        
def guestshell_remove(device_list: DeviceDict):

    #################################
    # REMOVE GUESTSHELL IN THE DEVICE
    #################################

    flow = lmerrors.DeviceFlow(device_list)

    try:
        info("DDR Remove: Guestshell Removed from device")
        
        g_enable = flow.exec_wait("guestshell destroy", timeout=300)
    except Exception as e:
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR XE Infra Deploy: An Exception occured in guestshell_remove from the device"
            + " "
            + str(e)
            + Color.END
        )
        
def stop_ddr(device_list: DeviceDict):

    #################################
    # STOP DDR EXECUTION IN THE DEVICE
    #################################

    flow = lmerrors.DeviceFlow(device_list)

    try:
        info("DDR Stop: Stop all Python processes in the guestshell")
        
        g_enable = flow.exec_wait("guestshell run bash flash:guest-share/ddr/ddrkill.sh", timeout=300)
    except Exception as e:
        raise Exception(
            Color.BOLD
            + Color.RED
            + f"DDR XE Infra Deploy: An Exception occured in stop_ddr "
            + " "
            + str(e)
            + Color.END
        )
