"""Step Implementer for the container-image-static-compliance-scan step for OpenSCAP.

Step Configuration
------------------

Step configuration expected as input to this step.
Could come from either configuration file or
from runtime configuration.

| Configuration Key              | Required? | Default | Description
|--------------------------------|-----------|---------|-----------
| `oscap-input-definitions-uri`  | Yes       |         | URI to the OpenSCAP definitions file
|                                |           |         | to do the evaluation with.
|                                |           |         | Must use protocol file://|http://|https://.
|                                |           |         | Must have file extension .xml|.bz2.
| `oscap-profile`                | No        |         | OpenSCAP profile to evaluate.
| `oscap-tailoring-uri`          | No        |         | URI to OpenSCAP tailoring file
|                                |           |         | to do the evaluation with.
|                                |           |         | Must use protocol file://|http://|https://.
|                                |           |         | Must have file extension .xml|.bz2.
| `oscap-fetch-remote-resources` | No        | True    | For Source DataStream and XCCDF files
|                                |           |         | that have remote references fetch them if
|                                |           |         | True, else don't.
|                                |           |         |
|                                |           |         | WARNING: evaluations will not be complete
|                                |           |         | if input defintions require remote
|                                |           |         | resources and this is not True.
|                                |           |         | For disconnected environments the remote
|                                |           |         | references should be brought to an internal
|                                |           |         | mirror.

Expected Previous Step Results
------------------------------

Results expected from previous steps that this step requires.

| Step Name                | Result Key       | Description
|--------------------------|------------------|--------------
| `create-container-image` | `image-tar-file` | Image to scan

Results
-------

Results output by this step.

| Result Key      | Description
|-----------------|------------
| `html-report`   | HTML report generated by oscap eval
| `xml-report`    | XML report generated by oscap eval
| `stdout-report` | stdout report generated by oscap eval
"""

import os
import re
import sys
from distutils.util import strtobool
from io import StringIO

import sh
from tssc import DefaultSteps, StepImplementer
from tssc.exceptions import TSSCException
from tssc.utils.file import download_and_decompress_source_to_destination
from tssc.utils.io import create_sh_redirect_to_multiple_streams_fn_callback

DEFAULT_CONFIG = {
    'oscap-fetch-remote-resources': True
}

REQUIRED_CONFIG_KEYS = [
    'oscap-input-definitions-uri'
]

class OpenSCAPGeneric(StepImplementer):
    """A generic OpenSCAP step implementer that can be used for more then one step.

    Expected uses:
    * container-image-static-compliance-scan
    * container-image-static-vulnerability-scan
    """

    # Example Input:
    #    Title	RHSA-2020:4186: spice and spice-gtk security update (Important)
    #    Rule	xccdf_com.redhat.rhsa_rule_oval-com.redhat.rhsa-def-20204186
    #    Ident	RHSA-2020:4186
    #    Ident	CVE-2020-14355
    #    Result	pass
    #
    #    Title	RHSA-2020:3658: librepo security update (Important)
    #    Rule	xccdf_com.redhat.rhsa_rule_oval-com.redhat.rhsa-def-20203658
    #    Ident	RHSA-2020:3658
    #    Ident	CVE-2020-14352
    #    Result	fail
    #
    # Matches:
    #    (Title	RHSA-2020:4186: spice and spice-gtk security update (Important)
    #    Rule	xccdf_com.redhat.rhsa_rule_oval-com.redhat.rhsa-def-20204186
    #    Ident	RHSA-2020:4186
    #    Ident	CVE-2020-14355
    #    Result	(pass))
    #
    #    (Title	RHSA-2020:3658: librepo security update (Important)
    #    Rule	xccdf_com.redhat.rhsa_rule_oval-com.redhat.rhsa-def-20203658
    #    Ident	RHSA-2020:3658
    #    Ident	CVE-2020-14352
    #    Result	(fail))
    #
    # Named Groups:
    #    [0]ruleblock
    #        Title	RHSA-2020:4186: spice and spice-gtk security update (Important)
    #        Rule	xccdf_com.redhat.rhsa_rule_oval-com.redhat.rhsa-def-20204186
    #        Ident	RHSA-2020:4186
    #        Ident	CVE-2020-14355
    #        Result	pass
    #    [0]ruleresult
    #        pass
    #
    #    [1]ruleblock
    #        Title	RHSA-2020:3658: librepo security update (Important)
    #        Rule	xccdf_com.redhat.rhsa_rule_oval-com.redhat.rhsa-def-20203658
    #        Ident	RHSA-2020:3658
    #        Ident	CVE-2020-14352
    #        Result	fail
    #    [1]ruleresult
    #        fail
    OSCAP_XCCDF_STDOUT_PATTERN = re.compile(
        r'(?P<ruleblock>Title.+?Result\s+(?P<ruleresult>[^\n]+))\n',
        re.DOTALL
    )
    OSCAP_XCCDF_STDOUT_FAIL_PATTERN = re.compile(r'fail')

    # NOTE: oval output far less useful then xccdf output but it is all but given some content
    #       is only given in oval format and therefor supporting this is important
    #
    # Example Input:
    #   Definition oval:com.redhat.rhsa:def:20202031: false
    #   Definition oval:com.redhat.rhsa:def:20201998: true
    #
    # Matches:
    #   (Definition oval:com.redhat.rhsa:def:20202031: (false))
    #   (Definition oval:com.redhat.rhsa:def:20201998: (true))
    #
    # Named Groups:
    #   [0]ruleblock
    #       Definition oval:com.redhat.rhsa:def:20202031: false
    #   [0]ruleresult
    #       false
    #
    #   [1]ruleblock
    #       Definition oval:com.redhat.rhsa:def:20201998: true
    #   [1]ruleresult
    #       true
    OSCAP_OVAL_STDOUT_PATTERN = re.compile(
        r'(?P<ruleblock>^.*:\s*(?P<ruleresult>true|false)\s*$)$',
        re.MULTILINE
    )
    OSCAP_OVAL_STDOUT_FAIL_PATTERN = re.compile(r'true')

    OSCAP_INFO_DOC_TYPE_PATTERN = re.compile(r'Document type: (?P<doctype>.+)')

    @staticmethod
    def step_implementer_config_defaults():
        """
        Getter for the StepImplementer's configuration defaults.

        Notes
        -----
        These are the lowest precedence configuration values.

        Returns
        -------
        dict
            Default values to use for step configuration values.
        """
        return DEFAULT_CONFIG

    @staticmethod
    def required_runtime_step_config_keys():
        """
        Getter for step configuration keys that are required before running the step.

        See Also
        --------
        _validate_runtime_step_config

        Returns
        -------
        array_list
            Array of configuration keys that are required before running the step.
        """
        return REQUIRED_CONFIG_KEYS

    def _validate_runtime_step_config(self, runtime_step_config):
        """Validates required runtime step configuration is valid.

        Validates that:
        * required configuration is given
        * oscap-input-definitions-uri
          - starts with file://|http://|https://
          - ends with .xml|.bz2

        Raises
        ------
        AssertionError
        * if runtime step configuration is invalid.
        """
        super()._validate_runtime_step_config(runtime_step_config) # pylint: disable=protected-access

        # validate that the given 'oscap-input-definitions-uri' starts with file://|http://|https://
        oscap_input_definitions_uri = runtime_step_config['oscap-input-definitions-uri']
        assert (re.match(r'^file://|http://|https://', oscap_input_definitions_uri)), \
            f"Open SCAP input definitions source ({oscap_input_definitions_uri})" \
            f" must start with known protocol (file://|http://|https://)."

        # validate that the given 'oscap-input-definitions-uri' is an xml or bz2 file
        oscap_input_definitions_uri_extension = os.path.splitext(oscap_input_definitions_uri)[1]
        assert (re.match(r'\.xml|\.bz2', oscap_input_definitions_uri_extension)), \
            f"Open SCAP input definitions source ({oscap_input_definitions_uri})" \
            f" must be of known type (xml|bz2), got: {oscap_input_definitions_uri_extension}"

    def _run_step(self): # pylint: disable=too-many-locals
        """Runs the OpenSCAP eval for a given input file against a given container.
        """
        image_tar_file = ''
        if(self.get_step_results(DefaultSteps.CREATE_CONTAINER_IMAGE) and \
          self.get_step_results(DefaultSteps.CREATE_CONTAINER_IMAGE).get('image-tar-file')):
            image_tar_file = self.\
            get_step_results(DefaultSteps.CREATE_CONTAINER_IMAGE)['image-tar-file']
        else:
            raise RuntimeError('Missing image tar file from ' + DefaultSteps.CREATE_CONTAINER_IMAGE)

        oscap_profile = self.get_config_value('oscap-profile')
        oscap_fetch_remote_resources = self.get_config_value('oscap-fetch-remote-resources')

        # create a container name from the tar file name, step name, and sub step name
        container_name = os.path.splitext(os.path.basename(image_tar_file))[0]
        container_name += f"-{self.step_name}-{self.sub_step_name}"

        # import image tar file to vfs file system
        print(f"\nImport image: {image_tar_file}")
        OpenSCAPGeneric.__buildah_import_image_from_tar(
            image_tar_file=image_tar_file,
            container_name=container_name
        )
        print(f"Imported image: {image_tar_file}")

        # baking `buildah unshare` command to wrap other buildah commands with
        # so that container does not need to be running in a privilaged mode to be able
        # to function
        buildah_unshare_comand = sh.buildah.bake('unshare') # pylint: disable=no-member

        # mount the container filesystem and get mount path
        #
        # NOTE: run in the context of `buildah unshare` so that container does not
        #       need to be run in a privilaged mode
        print(f"\nMount container: {container_name}")
        container_mount_path = OpenSCAPGeneric.__buildah_mount_container(
            buildah_unshare_comand=buildah_unshare_comand,
            container_id=container_name
        )
        print(f"Mounted container ({container_name}) with mount path: '{container_mount_path}'")

        # download the open scap input file
        oscap_input_definitions_uri = self.get_config_value('oscap-input-definitions-uri')
        print(f"\nDownload input definitions: {oscap_input_definitions_uri}")
        oscap_input_file = download_and_decompress_source_to_destination(
            source_url=oscap_input_definitions_uri,
            destination_dir=self.get_working_dir()
        )
        print(f"Download input definitions to: {oscap_input_file}")

        # if specified download oscap tailoring file
        oscap_tailoring_file = None
        oscap_tailoring_file_uri = self.get_config_value('oscap-tailoring-uri')
        if oscap_tailoring_file_uri:
            print(f"\nDownload oscap tailoring file: {oscap_tailoring_file_uri}")
            oscap_tailoring_file = download_and_decompress_source_to_destination(
                source_url=oscap_tailoring_file_uri,
                destination_dir=self.get_working_dir()
            )
            print(f"Download oscap tailoring file to: {oscap_tailoring_file}")

        # determine oscap eval type based on document type
        print(f"\nDetermine OpenSCAP document type of input file: {oscap_input_file}")
        oscap_document_type = OpenSCAPGeneric.__get_oscap_document_type(
            oscap_input_file=oscap_input_file
        )
        print(
            "Determined OpenSCAP document type of input file"
            f" ({oscap_input_file}): {oscap_document_type}"
        )
        print(
            f"\nDetermine OpenSCAP eval type for input file ({oscap_input_file}) "
            f"of document type: {oscap_document_type}"
        )
        oscap_eval_type = OpenSCAPGeneric.__get_oscap_eval_type_based_on_document_type(
            oscap_document_type=oscap_document_type
        )
        print(
            f"Determined OpenSCAP eval type of input file ({oscap_input_file}): {oscap_eval_type}"
        )

        # Execute scan in the context of buildah unshare
        #
        # NOTE: run in the context of `buildah unshare` so that container does not
        #       need to be run in a privilaged mode
        oscap_out_file_path = self.write_working_file(f'oscap-{oscap_eval_type}-out')
        oscap_xml_results_file_path = self.write_working_file(
            f'oscap-{oscap_eval_type}-results.xml'
        )
        oscap_html_report_path = self.write_working_file(f'oscap-{oscap_eval_type}-report.html')
        print("\nRun oscap scan")
        oscap_eval_success, oscap_eval_fails = OpenSCAPGeneric.__run_oscap_scan(
            buildah_unshare_comand=buildah_unshare_comand,
            oscap_eval_type=oscap_eval_type,
            oscap_input_file=oscap_input_file,
            oscap_out_file_path=oscap_out_file_path,
            oscap_xml_results_file_path=oscap_xml_results_file_path,
            oscap_html_report_path=oscap_html_report_path,
            container_mount_path=container_mount_path,
            oscap_profile=oscap_profile,
            oscap_tailoring_file=oscap_tailoring_file,
            oscap_fetch_remote_resources=oscap_fetch_remote_resources
        )
        print(
            f"OpenSCAP Scan Completed.  Report: {oscap_html_report_path}\n"
        )

        # NOTE: this should not raise an exception once we have new Results object
        #       since this is an "expected" "valid" failure...
        #       but this is the solution for now
        if not oscap_eval_success:
            raise TSSCException(f"OSCAP eval found issues:\n{oscap_eval_fails}")

        results = {
            'result': {
                'success': True
            },
            'report-artifacts': [
                {
                    'name': 'html-report',
                    'path': f'file://{oscap_html_report_path}'
                },
                {
                    'name': 'xml-report',
                    'path': f'file://{oscap_xml_results_file_path}'
                },
                {
                    'name': 'stdout-report',
                    'path': f'file://{oscap_out_file_path}'
                }
            ]
        }
        return results

    @staticmethod
    def __buildah_import_image_from_tar(image_tar_file, container_name):
        """Import a container image using buildah form a TAR file.

        Parameters
        ----------
        image_tar_file : str
            Path to TAR file to import as a container image.
        container_name : str
            name for the working container.

        Returns
        -------
        str
            Name of the imported container.
        """
        # import image tar file to vfs file system
        try:
            sh.buildah( # pylint: disable=no-member
                'from',
                '--storage-driver', 'vfs',
                '--name', container_name,
                f"docker-archive:{image_tar_file}",
                _out=sys.stdout,
                _err=sys.stderr,
                _tee='err'
            )
        except sh.ErrorReturnCode as error:
            raise RuntimeError(
                f'Unexpected runtime error importing the image ({image_tar_file}): {error}'
            ) from error

        return container_name

    @staticmethod
    def __buildah_mount_container(buildah_unshare_comand, container_id):
        """Use buildah to mount a container.

        Parameters
        ----------
        buildah_unshare_comand : sh.buildah.unshare.bake()
            A baked sh.buildah.unshare command to use to run this command in the context off
            so that this can be done "rootless".
        container_id : str
            ID of the container to mount.

        Returns
        -------
        str
            Absolute path to the mounted container.
        """
        mount_path = None
        try:
            buildah_mount_out_buff = StringIO()
            buildah_mount_out_callback = create_sh_redirect_to_multiple_streams_fn_callback([
                sys.stdout,
                buildah_mount_out_buff
            ])
            buildah_mount_command = buildah_unshare_comand.bake("buildah", "mount")
            buildah_mount_command(
                '--storage-driver', 'vfs',
                container_id,
                _out=buildah_mount_out_callback,
                _err=sys.stderr,
                _tee='err'
            )
            mount_path = buildah_mount_out_buff.getvalue().rstrip()
        except sh.ErrorReturnCode as error:
            raise RuntimeError(
                f'Unexpected runtime error mounting container ({container_id}): {error}'
            ) from error

        return mount_path

    @staticmethod
    def __get_oscap_document_type(oscap_input_file):
        """Gets the OpenSCAP document type for a given input file.

        Parameters
        ----------
        oscap_input_file : path
            Path to OSCAP file to determine the OpenSCAP document type of.

        Returns
        -------
        str
            OpenSCAP document type. For example:
            * Source Data Stream
            * XCCDF Checklist
            * OVAL Definitions

        Raises
        ------
        ErrorReturnCode
            If unexpected error occurred
        """

        oscap_document_type = None
        try:
            oscap_info_out_buff = StringIO()
            sh.oscap.info( # pylint: disable=no-member
                oscap_input_file,
                _out=oscap_info_out_buff
            )
            oscap_info_out = oscap_info_out_buff.getvalue().rstrip()
            oscap_document_type_match = OpenSCAPGeneric.OSCAP_INFO_DOC_TYPE_PATTERN.search(
                oscap_info_out
            )
            oscap_document_type = oscap_document_type_match.groupdict()['doctype']
        except sh.ErrorReturnCode as error:
            raise RuntimeError(
                f"Unexpected error getting document type of oscap input file"
                f" ({oscap_input_file}): {error}"
            ) from error

        return oscap_document_type

    @staticmethod
    def __get_oscap_eval_type_based_on_document_type(oscap_document_type):
        """Given an OSCAP document type returns the type of oscap eval that should be used.

        Parameters
        ----------
        oscap_document_type : str
            OSCAP Document type to get the oscap eval type for.

        Returns
        -------
        str
            OSCAP eval type to perform on document with given oscap document type.
        """
        oscap_eval_type = None

        if oscap_document_type == 'Source Data Stream':
            oscap_eval_type = 'xccdf'
        elif oscap_document_type == 'XCCDF Checklist':
            oscap_eval_type = 'xccdf'
        elif oscap_document_type == 'OVAL Definitions':
            oscap_eval_type = 'oval'

        return oscap_eval_type

    @staticmethod
    def __run_oscap_scan( # pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements
        buildah_unshare_comand,
        oscap_eval_type,
        oscap_input_file,
        oscap_out_file_path,
        oscap_xml_results_file_path,
        oscap_html_report_path,
        container_mount_path,
        oscap_profile=None,
        oscap_tailoring_file=None,
        oscap_fetch_remote_resources=True
    ):
        """Run an oscap scan in the context of a buildah unshare to run "rootless".

        Parameters
        ----------
        buildah_unshare_comand : sh.buildah.unshare.bake()
            A baked sh.buildah.unshare command to use to run this command in the context off
            so that this can be done "rootless".
        oscap_eval_type : str
            The type of oscap eval to perform. Must be a valid oscap eval type.
            EX: xccdf, oval
        oscap_input_file : str
            Path to rules file passed to the oscap command.
        oscap_out_file_path : str
            Path to write the stdout and stderr of running the oscap command to.
        oscap_xml_results_file_path : str
            Write the scan results into this file.
        oscap_html_report_path : str
            Write the human readable (HTML) report into this file.
        container_mount_path : str
            Path to the mounted container to scan.
        oscap_tailoring_file : str
            XCCF Tailoring file.
            See:
            - https://www.open-scap.org/security-policies/customization/
            - https://www.open-scap.org/resources/documentation/customizing-scap-security-guide-for-your-use-case/ # pylint: disable=line-too-long
            - https://static.open-scap.org/openscap-1.2/oscap_user_manual.html#_how_to_tailor_source_data_stream # pylint: disable=line-too-long
        oscap_profile : str
            OpenSCAP profile to evaluate. Must be a valid profile in the given oscap_input_file.
            EX: if you perform an `oscap info oscap_input_file` the profile must be listed.

        Returns
        -------
        oscap_eval_success : bool
            True if oscap eval passed all rules
            False if oscap eval failed any rules
        oscap_eval_fails : str
            If oscap_eval_success is True then indeterminate.
            If oscap_eval_success is False then string of all of the failed rules.

        Raises
        ------
        RuntimeError
            If unexpected error running oscap scan.
        """

        oscap_profile_flag = None
        if oscap_profile is not None:
            oscap_profile_flag = f"--profile={oscap_profile}"

        oscap_fetch_remote_resources_flag = None
        if isinstance(oscap_fetch_remote_resources, str):
            oscap_fetch_remote_resources = strtobool(oscap_fetch_remote_resources)
        if oscap_fetch_remote_resources:
            oscap_fetch_remote_resources_flag = "--fetch-remote-resources"

        oscap_tailoring_file_flag = None
        if oscap_tailoring_file is not None:
            oscap_tailoring_file_flag = f"--tailoring-file={oscap_tailoring_file}"

        oscap_eval_success = None
        oscap_eval_out_buff = StringIO()
        oscap_eval_out = ""
        oscap_eval_fails = None
        try:
            oscap_chroot_command = buildah_unshare_comand.bake("oscap-chroot")
            with open(oscap_out_file_path, 'w') as oscap_out_file:
                out_callback = create_sh_redirect_to_multiple_streams_fn_callback([
                    oscap_eval_out_buff,
                    oscap_out_file
                ])
                err_callback = create_sh_redirect_to_multiple_streams_fn_callback([
                    oscap_eval_out_buff,
                    oscap_out_file
                ])
                oscap_chroot_command(
                    container_mount_path,
                    oscap_eval_type,
                    'eval',
                    oscap_profile_flag,
                    oscap_fetch_remote_resources_flag,
                    oscap_tailoring_file_flag,
                    f'--results={oscap_xml_results_file_path}',
                    f'--report={oscap_html_report_path}',
                    oscap_input_file,
                    _out=out_callback,
                    _err=err_callback,
                    _tee='err'
                )
                oscap_eval_success = True
        except sh.ErrorReturnCode_1 as error: # pylint: disable=no-member
            oscap_eval_success = error
        except sh.ErrorReturnCode_2 as error: # pylint: disable=no-member
            # XCCDF: If there is at least one rule with either fail or unknown result,
            #           oscap-scan finishes with return code 2.
            # OVAL:  Never returned
            #
            # Source: https://www.systutorials.com/docs/linux/man/8-oscap/
            if oscap_eval_type == 'xccdf':
                oscap_eval_success = False
            else:
                oscap_eval_success = error
        except sh.ErrorReturnCode as error:
            oscap_eval_success = error

        # get the oscap output
        oscap_eval_out = oscap_eval_out_buff.getvalue()

        # parse the oscap output
        # NOTE: oscap is puts carrage returns (\r / ^M) in their output, remove them
        oscap_eval_out = re.sub('\r', '', oscap_eval_out)

        # print the oscap output no matter the results
        print(oscap_eval_out)

        # if unexpected error throw error
        if isinstance(oscap_eval_success, Exception):
            raise RuntimeError(
                f"Unexpected error running 'oscap {oscap_eval_type} eval': {oscap_eval_success} "
            ) from oscap_eval_success

        # NOTE: oscap oval eval returns exit code 0 whether or not any rules failed
        #       need to search output to determine if there were any rule failures
        if oscap_eval_type == 'oval' and oscap_eval_success:
            oscap_eval_fails = ""
            for match in OpenSCAPGeneric.OSCAP_OVAL_STDOUT_PATTERN.finditer(oscap_eval_out):
                # NOTE: need to do regex and not == because may contain xterm color chars
                if OpenSCAPGeneric.OSCAP_OVAL_STDOUT_FAIL_PATTERN.search(
                    match.groupdict()['ruleresult']
                ):
                    oscap_eval_fails += match.groupdict()['ruleblock']
                    oscap_eval_fails += "\n"
                    oscap_eval_success = False

        # if failed xccdf eval then parse out the fails
        if oscap_eval_type == 'xccdf' and not oscap_eval_success:
            oscap_eval_fails = ""
            for match in OpenSCAPGeneric.OSCAP_XCCDF_STDOUT_PATTERN.finditer(oscap_eval_out):
                # NOTE: need to do regex and not == because may contain xterm color chars
                if re.search(r'fail', match.groupdict()['ruleresult']):
                    oscap_eval_fails += "\n"
                    oscap_eval_fails += match.groupdict()['ruleblock']
                    oscap_eval_fails += "\n"

        return oscap_eval_success, oscap_eval_fails
