# -*- coding: utf-8 -*-
import os
import re
import sys

import pytest
import yaml
import time
import string
import random
import fileinput

from datetime import datetime
from mock.mock import patch, Mock, ANY

try:
    from urllib import quote as urlencode  # python 2
    import __builtin__ as builtins
except ImportError:
    from urllib.parse import quote as urlencode  # python 3
    import builtins

from insights.client.config import InsightsConfig
from insights.client.constants import InsightsConstants as constants
from insights.core.exceptions import CalledProcessError
from insights.specs.manifests import manifests, content_types
from insights.tests.helpers import getenv_bool
from insights.util.subproc import call

from insights.specs.datasources.malware_detection import (
    DEFAULT_MALWARE_CONFIG,
    MalwareDetectionClient,
    InsightsConnection,
    get_toplevel_dirs,
    get_parent_dirs,
    remove_child_items,
    remove_included_excluded_items,
    process_include_items,
    process_exclude_items,
    process_include_exclude_items,
    logger,
    MIN_YARA_VERSION,
)

# Temporary directory for testing stuff in
RANDOM_STRING = ''.join(random.choice(string.ascii_lowercase) for _ in range(5))
TEMP_TEST_DIR = "/tmp/malware-detection_test_dir_%s" % RANDOM_STRING

FAKE_YARA = '/bin/yara'  # Need to fake yara for a number of tests
RULES_FILE = os.path.join(TEMP_TEST_DIR, 'malware-detection_yara_rules.yar')
TEST_RULE_FILE = os.path.join(TEMP_TEST_DIR, 'test-rule.yar')
TEST_RULE_SCRIPT = os.path.join(TEMP_TEST_DIR, 'test-rule_process_match.sh')
MATCHING_ENTITY_FILE = os.path.join(TEMP_TEST_DIR, 'matching_entity')
ANOTHER_MATCHING_ENTITY_FILE = os.path.join(TEMP_TEST_DIR, 'another matching_entity')
CONFIG = yaml.safe_load(DEFAULT_MALWARE_CONFIG)  # Config 'returned' from _load_config
TEMP_CONFIG_FILE = os.path.join(TEMP_TEST_DIR, 'malware-detection-config.yml')
TEST_PID = str(os.getpid())  # This running processes ID

# Get the number of CPU threads to run yara
CPUS = 1 if int(call('nproc').strip()) <= 2 else 2

# Some of the toplevel directories that will be included/excluded by default when listing root (/)
TLDS = ['/boot', '/dev', '/etc', '/home', '/opt', '/proc', '/root', '/sys', '/tmp', '/usr', '/var']
INCLUDED_TLDS = [
    '/boot',
    '/dev',
    '/etc',
    '/home',
    '/opt',
    '/root',
    '/tmp',
    '/usr',
    '/var',
]  # after removing exclude items
DEFAULT_SCAN_EXCLUDE = ['/cgroup', '/media', '/mnt', '/net', '/proc', '/selinux', '/sys']

# Various patch targets
LOGGER_TARGET = "insights.specs.datasources.malware_detection.logger"
LOAD_CONFIG_TARGET = (
    "insights.specs.datasources.malware_detection.MalwareDetectionClient._load_config"
)
FIND_YARA_TARGET = "insights.specs.datasources.malware_detection.MalwareDetectionClient._find_yara"
GET_RULES_TARGET = "insights.specs.datasources.malware_detection.MalwareDetectionClient._get_rules"
DISABLED_RULES_TARGET = (
    "insights.specs.datasources.malware_detection.MalwareDetectionClient._get_disabled_rules"
)
BUILD_YARA_COMMAND_TARGET = (
    "insights.specs.datasources.malware_detection.MalwareDetectionClient._build_yara_command"
)
FINDMNT_TARGET = "insights.specs.datasources.malware_detection.MalwareDetectionClient._parse_exclude_network_filesystem_mountpoints_option"
CALL_TARGET = "insights.specs.datasources.malware_detection.call"
CONFIG_FILE_TARGET = "insights.specs.datasources.malware_detection.MALWARE_CONFIG_FILE"
NAMEDTMPFILE_TARGET = "insights.specs.datasources.malware_detection.NamedTemporaryFile"

# Run these slowish tests?
TEST_PROCESSES_SCAN_SINCE = getenv_bool("TEST_PROCESSES_SCAN_SINCE", False)
TEST_DOWNLOAD_FAILURE_RETRIES = getenv_bool("TEST_DOWNLOAD_FAILURE_RETRIES", False)
TEST_GET_DISABLED_RULES_FAILURE = getenv_bool("TEST_GET_DISABLED_RULES_FAILURE", False)
TEST_ALL = getenv_bool("TEST_ALL", False)

# Are we running on RHEL6? (well actually, with python 2.6)
IS_RHEL6 = sys.version_info < (2, 7)
SKIP_IF_RHEL6_REASON = "The malware-detection client isn't supported on RHEL6 / python 2.6"
# Is the best way to determine if we are running in a container?
IS_CONTAINER = (
    os.path.exists("/.dockerenv")
    or os.path.exists("/run/.containerenv")
    or '1' not in call('pidof init systemd').strip().split()
)
SKIP_IF_CONTAINER_REASON = (
    "This test uses running process that may not exist when run in a container"
)


@pytest.fixture
def create_test_files_fake_yara():
    # Write the test files to the temp directory
    if not os.path.exists(TEMP_TEST_DIR):
        os.mkdir(TEMP_TEST_DIR)
    with open(TEMP_CONFIG_FILE, 'w') as tcf:
        tcf.write(DEFAULT_MALWARE_CONFIG)
    test_files = [
        (MATCHING_ENTITY_FILE, MATCHING_ENTITY_FILE_CONTENTS),
        (ANOTHER_MATCHING_ENTITY_FILE, ANOTHER_MATCHING_ENTITY_FILE_CONTENTS),
        (TEST_RULE_FILE, TEST_RULE_FILE_CONTENTS),
        (TEST_RULE_SCRIPT, TEST_RULE_SCRIPT_CONTENTS),
    ]
    for test_file, contents in test_files:
        if not os.path.exists(test_file):
            with open(test_file, 'w') as f:
                f.write(contents)
    os.chmod(TEST_RULE_SCRIPT, 0o755)
    yield
    os.system('rm -rf %s' % TEMP_TEST_DIR)


@pytest.fixture
def extract_tmp_files():
    if not os.path.exists(TEMP_TEST_DIR):
        os.mkdir(TEMP_TEST_DIR)
    os.system("echo '%s' | base64 -d - | tar -C %s -zxf -" % (SCAN_FILES_BASE64, TEMP_TEST_DIR))
    yield
    os.system('rm -rf %s' % TEMP_TEST_DIR)


#######################################################################
# The following section tests functionality that does not require yara
#######################################################################
@pytest.mark.skipif(IS_RHEL6, reason=SKIP_IF_RHEL6_REASON)
class TestsNotUtilizingYara:

    def test_default_spec(self):
        # Read in the default malware spec and check its values
        manifest = yaml.safe_load(manifests['malware-detection'])
        content_type = content_types['malware-detection']
        assert content_type == "application/vnd.redhat.malware-detection.results+tgz"
        specs = manifest['plugins']['configs']
        for spec in [
            'mac_addresses',
            'etc_machine_id',
            'hostname',
            'dmidecode',
            'machine_id',
            'ip_addresses',
            'subscription_manager_id',
        ]:
            assert {
                'enabled': True,
                'name': 'insights.specs.default.DefaultSpecs.%s' % spec,
            } in specs
            assert {'enabled': True, 'name': 'insights.specs.Specs.%s' % spec} in specs

    def test_default_options(self):
        # Read in the default malware_detection_config options and check their values
        assert CONFIG['test_scan'] is True
        assert CONFIG['scan_filesystem'] is True
        assert CONFIG['scan_processes'] is False
        assert CONFIG['filesystem_scan_only'] is None
        assert CONFIG['processes_scan_only'] is None
        assert CONFIG['filesystem_scan_since'] is None
        assert CONFIG['processes_scan_since'] is None
        assert all(
            [
                x in CONFIG['filesystem_scan_exclude']
                for x in ['/proc', '/sys', '/cgroup', '/selinux', '/net', '/mnt', '/media']
            ]
        )
        assert CONFIG['processes_scan_exclude'] is None
        assert CONFIG['exclude_network_filesystem_mountpoints'] is True

    def test_toplevel_dirs(self):
        tlds = get_toplevel_dirs()
        assert all([x in tlds for x in TLDS])
        assert any([x in tlds for x in DEFAULT_SCAN_EXCLUDE])

    def test_get_parent_dirs(self):
        parent_dir_list = []
        get_parent_dirs('/usr/lib/systemd/user/basic.target', parent_dir_list)
        assert sorted(parent_dir_list) == [
            '/usr',
            '/usr/lib',
            '/usr/lib/systemd',
            '/usr/lib/systemd/user',
            '/usr/lib/systemd/user/basic.target',
        ]

        parent_dir_list = []
        get_parent_dirs('/usr/lib/systemd/user/basic.target', parent_dir_list, '/usr/lib')
        assert sorted(parent_dir_list) == [
            '/usr/lib/systemd',
            '/usr/lib/systemd/user',
            '/usr/lib/systemd/user/basic.target',
        ]

        parent_dir_list = []
        get_parent_dirs('/usr/lib/systemd/user/basic.target', parent_dir_list, '/var')
        assert sorted(parent_dir_list) == [
            '/usr',
            '/usr/lib',
            '/usr/lib/systemd',
            '/usr/lib/systemd/user',
            '/usr/lib/systemd/user/basic.target',
        ]

    def test_remove_child_items(self):
        # Simple example from the function docstring
        items = ['/path/to/some/item/child', '/path/to/another/item', '/path/to/some/item']
        assert remove_child_items(items) == ['/path/to/another/item', '/path/to/some/item']

        # More complex test with duplicate items and items whose names start with another's name
        # (rather than start with the same path) and names with spaces in them
        items = [
            '/var/lib64',
            '/home/bob',
            '/var/lib/docker',
            '/home/bob',
            '/var/lib',
            '/home/bobby-droptables',
            r'/home/bob/this\ is\ bobs/child',
            '/var/lib63/im ok',
            '/var/lib64/im not',
        ]
        assert remove_child_items(items) == [
            '/home/bob',
            '/home/bobby-droptables',
            '/var/lib',
            '/var/lib63/im ok',
            '/var/lib64',
        ]

        # /path is common to both so that will be all that is returned
        items = ['/path/to/another/item', '/path/to/some/item', '/path']
        assert remove_child_items(items) == ['/path']

        # The root directory will always win
        items = ['/path/to/another/item', '/path/to/some/item', '/']
        assert remove_child_items(items) == ['/']

        # Any non-full path items (doesn't start with /) are removed from the list
        items = ['/path/to/another/item', '', '/path/to/some/item']
        assert remove_child_items(items) == ['/path/to/another/item', '/path/to/some/item']

    def test_remove_included_excluded_items(self):
        include_items = ['/home/bob', '/tmp', '/var/www/html']
        exclude_items = ['/home/bo', '/home/bob/exclude', '/home/bobby', '/temp', '/var/www']
        include_items = remove_included_excluded_items(include_items, exclude_items)
        assert include_items == ['/home/bob', '/tmp']

        # Some fairly random lists similar to before
        include_items = [
            '/var/lib64/docker',
            r'/home/bob/this\ is\ bobs/child',
            '/var/lib/docker',
            '/home/bob',
            '/var/lib',
            '/usr',
        ]
        exclude_items = [
            '/var/lib/docker2',
            '/home/bob',
            '/home/bobby-droptables',
            r'/home/bob/this\ is\ bobs/child',
            '/var/lib63/im ok',
            '/var/lib64/im not',
            '/boot',
        ]
        include_items = remove_included_excluded_items(include_items, exclude_items)
        assert include_items == ['/usr', '/var/lib', '/var/lib64/docker']

    def test_default_include_items(self):
        # Call process_include_items with an empty list.
        # If nothing is passed to the function, the a list of the top level directories is produced
        default_list = process_include_items()
        assert default_list == get_toplevel_dirs()

    def test_default_exclude_items(self):
        # Call process_exclude_items with an empty list.
        # If nothing is passed to the function, then nothing to exclude
        default_list = process_exclude_items()
        assert default_list == []

    def test_process_include_items(self, caplog):
        # Call process_include_items with variously populated lists
        logger.setLevel('DEBUG')

        # Add some valid entries to include_items list, esp subdirectories
        include_items = ['/etc/pam.d', '/tmp', '/var/log/']
        processed_items = process_include_items(include_items)
        assert processed_items == ['/etc/pam.d', '/tmp', '/var/log']

        # Add some more subdirectories
        include_items.extend(['/etc/alternatives', '/tmp', '/var/lib/'])
        processed_items = process_include_items(include_items)
        assert processed_items == [
            '/etc/alternatives',
            '/etc/pam.d',
            '/tmp',
            '/var/lib',
            '/var/log',
        ]

        # Add some top level directories to override the subdirectories
        include_items.extend(['/etc', '/var'])
        processed_items = process_include_items(include_items)
        assert processed_items == ['/etc', '/tmp', '/var']

        # Add some invalid entries that will get ignored
        caplog.clear()
        include_items.extend(['..', '/var/run', '/missing'])
        processed_items = process_include_items(include_items)
        assert "Skipping partial directory path '..' ..." in caplog.text
        assert "Skipping link '/var/run' ..." in caplog.text
        assert "Skipping missing item '/missing' ..." in caplog.text
        assert processed_items == ['/etc', '/tmp', '/var']

        # Add the root directory (/) which will override all the other entries
        caplog.clear()
        include_items.append('/')
        processed_items = process_include_items(include_items)
        assert (
            "Found root directory in list of items to scan.  Ignoring the other items ..."
            in caplog.text
        )
        assert all([x in processed_items for x in TLDS])
        assert any([x in processed_items for x in DEFAULT_SCAN_EXCLUDE])

    def test_process_exclude_items(self, caplog):
        # Call process_exclude_items with variously populated lists
        logger.setLevel('DEBUG')

        # Remove the default entries from the exclude file
        processed_items = process_exclude_items()
        assert "No items specified to be excluded" in caplog.text
        assert processed_items == []

        # Add some valid entries to exclude items (links are ok in the exclude list ... why?)
        exclude_items = ['/etc/ssh', '/tmp', '/var/run/']
        processed_items = process_exclude_items(exclude_items)
        assert processed_items == ['/etc/ssh', '/tmp', '/var/run']

        # Add some more subdirectories
        exclude_items.extend(['/etc/alternatives', '/tmp', '/var/lock/'])
        processed_items = process_exclude_items(exclude_items)
        assert processed_items == ['/etc/alternatives', '/etc/ssh', '/tmp', '/var/lock', '/var/run']

        # Add some top level directories to override the subdirectories
        exclude_items.extend(['/etc', '/var'])
        processed_items = process_exclude_items(exclude_items)
        assert processed_items == ['/etc', '/tmp', '/var']

        # Add some invalid entries to exclude items
        caplog.clear()
        exclude_items.extend(['..', '/missing'])
        processed_items = process_exclude_items(exclude_items)
        assert "Skipping partial directory path '..' ..." in caplog.text
        assert "Skipping missing item '/missing' ..." in caplog.text
        assert processed_items == ['/etc', '/tmp', '/var']

        # Add the root directory
        caplog.clear()
        exclude_items.append('/')
        processed_items = process_exclude_items(exclude_items)
        assert (
            "Found root directory in the exclude list.  Expanding it to all toplevel directories ..."
            in caplog.text
        )
        assert processed_items == get_toplevel_dirs()

    def test_process_include_exclude_tmp_files(self, extract_tmp_files):
        # Test the including/excluding some of the files in the tmp archive
        # Specifically tests excluding link files (good or broken) and pipe files (as well as explicit exclude items)

        include_items = list(
            map(lambda x: os.path.join(TEMP_TEST_DIR, x), ['scan_me', 'scan_me_too'])
        )
        exclude_items = list(
            map(
                lambda x: os.path.join(TEMP_TEST_DIR, x),
                ['scan_me_not', 'scan_me/dont_scan_me', 'scan_me_too/dont_scan_me_too'],
            )
        )
        scan_dict = process_include_exclude_items(
            include_items=include_items, exclude_items=exclude_items
        )
        assert list(scan_dict.keys()) == ['/tmp']
        assert sorted(list(scan_dict['/tmp']['exclude']['items'])) == sorted(exclude_items)

        include_files = sorted(
            list(
                map(
                    lambda x: os.path.join(TEMP_TEST_DIR, x),
                    [
                        'scan_me/new_file',
                        'scan_me/old_file',
                        'scan_me/scan_me',
                        'scan_me/scan_me_file',
                        'scan_me_too/new_file',
                        'scan_me_too/old_file',
                        'scan_me_too/scan_me_too',
                        'scan_me_too/scan_me_too_file',
                    ],
                )
            )
        )
        dont_include_files = sorted(
            list(
                map(
                    lambda x: os.path.join(TEMP_TEST_DIR, x),
                    [
                        'scan_me/link_file',
                        'scan_me/pipe_file',
                        'scan_me/broken_link',
                        'scan_me/dont_scan_me',
                        'scan_me_too/link_file',
                        'scan_me_too/pipe_file',
                        'scan_me_too/broken_link',
                        'scan_me_too/dont_scan_me_too',
                    ],
                )
            )
        )
        assert sorted(scan_dict['/tmp']['include']) == include_files
        assert all([x not in scan_dict['/tmp']['include'] for x in dont_include_files])

        # Another test to assert a bug I found is fixed ... due to only having scan_items = set([])
        # Basically include_files should = ['scan_me/scan_me'] but the bug made include_files = []
        include_items = list(map(lambda x: os.path.join(TEMP_TEST_DIR, x), ['scan_me/scan_me']))
        exclude_items = list(map(lambda x: os.path.join(TEMP_TEST_DIR, x), ['scan_me_not']))
        scan_dict = process_include_exclude_items(
            include_items=include_items, exclude_items=exclude_items
        )

        include_files = sorted(
            list(map(lambda x: os.path.join(TEMP_TEST_DIR, x), ['scan_me/scan_me']))
        )
        assert sorted(scan_dict['/tmp']['include']) == include_files
        dont_include_files = sorted(
            list(
                map(
                    lambda x: os.path.join(TEMP_TEST_DIR, x),
                    ['scan_me/scan_me_file', 'scan_me/dont_scan_me', 'scan_me/scan_me/here_i_am'],
                )
            )
        )
        assert all([x not in scan_dict['/tmp']['include'] for x in dont_include_files])


###################################################################################################
# The following section tests functionality that requires yara but can do with its execution faked
###################################################################################################
@pytest.mark.skipif(IS_RHEL6, reason=SKIP_IF_RHEL6_REASON)
class TestsUtilizingFakeYara:

    @patch(BUILD_YARA_COMMAND_TARGET, return_value=(['lol'], []))
    @patch(GET_RULES_TARGET, return_value=[TEST_RULE_FILE])
    @patch(DISABLED_RULES_TARGET, return_value=[])
    @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
    @patch(LOGGER_TARGET)
    class TestDefaultValues:

        @patch('insights.client.utilities.write_to_disk', Mock())
        def test_running_default_options(
            self, log_mock, yara, disabled, rules, cmd, create_test_files_fake_yara
        ):
            # Try running malware-detection with the default options
            # With the default options, test_scan is true, so some of the option values will be changed for that and
            # will be different from those in the default config file.
            # For example, do_filesystem_scan AND do_process_scan are both True when doing a test scan
            # Use a real config file so scan_fsobjects will be populated properly
            with patch(CONFIG_FILE_TARGET, TEMP_CONFIG_FILE):
                mdc = MalwareDetectionClient(None)
            assert mdc.yara_binary == FAKE_YARA
            assert mdc.rules_files == [TEST_RULE_FILE]
            assert mdc.disabled_rules == []
            assert mdc.do_filesystem_scan is True
            assert mdc.do_process_scan is True
            assert mdc.scan_fsobjects == [TEMP_CONFIG_FILE]
            assert mdc.scan_pids == [TEST_PID]
            assert mdc.filesystem_scan_since_dict == {'timestamp': None}
            assert mdc.filesystem_scan_exclude_list == []
            assert mdc.processes_scan_exclude_list == []
            assert mdc.processes_scan_since_dict == {'timestamp': None}
            assert mdc.network_filesystem_mountpoints == []
            assert mdc.scan_timeout == 3600
            assert mdc.nice_value == 19
            assert mdc.rules_location == '/etc/insights-client/signatures'
            assert mdc.use_remote_rules is True

            with patch(CALL_TARGET) as call_mock:
                # Mock all the calls to 'call' to get the yara matches and the metadata about the matches for the test scan
                # 1st call is yara output from scanning TEST_RULE_FILE, calls 2-6 are to get its metadata
                # 7th call is yara output from scanning the current process and 8th is to get its metadata
                call_mock.side_effect = [
                    'TEST_RedHatInsightsMalwareDetection [author="Red Hat Insights",description="Verifies the Red Hat Insights Malware Detection Client app is present on the system",usage="Verification",date_created="11 June 2021",date_updated="18 October 2021"] %s\n0x4a:$re1: Malware Detection Client'
                    % TEST_RULE_FILE,
                    "ASCII text",
                    "text/plain; charset=us-ascii",
                    "d5b0aeb3e18df68f47287e14ef144489",
                    "2:74:Malware Detection Client",
                    "// Verifies the Red Hat Insights Malware Detection Client app is present on the system",
                    'TEST_RedHatInsightsMalwareDetection [author="Red Hat Insights",description="Verifies the Red Hat Insights Malware Detection Client app is present on the system",usage="Verification",date_created="11 June 2021",date_updated="18 October 2021"] %s\n0x4a:$re1: Malware Detection Client'
                    % TEST_PID,
                    "python insights_client/run.py --collector malware-detection",
                ]
                mutation = mdc.run()

            log_mock.info.assert_any_call("Found %d rule match%s.", 2, "es")
            # Test scan results in mutation format
            assert 'ruleName: "TEST_RedHatInsightsMalwareDetection"' in mutation[0]
            assert 'source: "%s"' % TEST_RULE_FILE in mutation[0]
            assert 'source: "%s"' % TEST_PID in mutation[0]
            assert re.search('metadata:.*line_number', mutation[0])
            assert re.search('metadata:.*process_name', mutation[0])
            # Test scan results in JSON format
            rule = 'TEST_RedHatInsightsMalwareDetection'
            assert len(mutation[1][rule]['matches']) == 2
            assert mutation[1][rule]['matches'][0]['source'] == TEST_RULE_FILE
            assert mutation[1][rule]['matches'][1]['source'] == TEST_PID
            assert mutation[1][rule]['matches'][0]['string_data'] == 'Malware Detection Client'
            assert mutation[1][rule]['matches'][1]['string_data'] == 'Malware Detection Client'
            assert mutation[1][rule]['matches'][0]['metadata']['source_type'] == 'file'
            assert (
                mutation[1][rule]['matches'][0]['metadata']['line']
                == '//%20Verifies%20the%20Red%20Hat%20Insights%20Malware%20Detection%20Client%20app%20is%20present%20on%20the%20system'
            )
            assert mutation[1][rule]['matches'][1]['metadata']['source_type'] == 'process'
            assert (
                mutation[1][rule]['matches'][1]['metadata']['process_name']
                == 'python insights_client/run.py --collector malware-detection'
            )

    @patch(CONFIG_FILE_TARGET, TEMP_CONFIG_FILE)
    @patch(BUILD_YARA_COMMAND_TARGET, return_value=(['lol'], []))
    @patch(GET_RULES_TARGET, return_value=[RULES_FILE])
    @patch(DISABLED_RULES_TARGET, return_value=[])
    @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
    @patch.dict(
        os.environ, {'TEST_SCAN': 'false', 'EXCLUDE_NETWORK_FILESYSTEM_MOUNTPOINTS': 'false'}
    )
    class TestMalwareDetectionOptions:

        @patch(LOAD_CONFIG_TARGET, return_value=CONFIG)
        def test_running_modified_options(self, conf, yara, disabled, rules, cmd):
            # Disable test_scan and the mdc attribute values should mostly match what's in the config file
            mdc = MalwareDetectionClient(None)
            assert mdc.rules_files == [RULES_FILE]
            assert mdc.disabled_rules == []
            assert mdc.yara_binary == FAKE_YARA
            assert mdc.do_filesystem_scan is True
            assert mdc.do_process_scan is False
            assert mdc.scan_fsobjects == []
            assert mdc.scan_pids == []
            assert mdc.filesystem_scan_since_dict == {'timestamp': None, 'datetime': None}
            assert all(
                [d in mdc.filesystem_scan_exclude_list for d in ['/proc', '/sys', '/mnt', '/media']]
            )
            # filesystem_scan_* attributes will exist whereas processes_scan_* will not
            assert all(
                [
                    hasattr(mdc, 'filesystem_scan_' + attr) is True
                    and hasattr(mdc, 'processes_scan_' + attr) is False
                    for attr in ('exclude_list', 'since_dict')
                ]
            )

        @patch(LOAD_CONFIG_TARGET, return_value=CONFIG)
        @patch(LOGGER_TARGET)
        def test_scan_only_options(self, log_mock, conf, yara, disabled, rules, cmd):
            # Test various combinations of filesystem_scan_only, process_scan_only, scan_filesystem & scan_processes
            # Firstly, test the default option values
            mdc = MalwareDetectionClient(None)
            assert mdc.do_filesystem_scan is True
            assert mdc.scan_fsobjects == []
            assert mdc.do_process_scan is False
            assert mdc.scan_pids == []

            # Add some directories
            os.environ['FILESYSTEM_SCAN_ONLY'] = '/tmp,/var'
            mdc = MalwareDetectionClient(None)
            assert mdc.scan_fsobjects == ['/tmp', '/var']

            # Disable filesystem scanning and only expect the process to be scanned
            os.environ['SCAN_FILESYSTEM'] = 'false'
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_called_with(
                "Both scan_filesystem and scan_processes are disabled.  Nothing to scan."
            )

            # Add scan_only for a process
            os.environ['PROCESSES_SCAN_ONLY'] = '1'
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_called_with(
                "Both scan_filesystem and scan_processes are disabled.  Nothing to scan."
            )

            # Enable process scanning and now the scan_only value should be used
            os.environ['SCAN_PROCESSES'] = 'true'
            mdc = MalwareDetectionClient(None)
            assert mdc.do_filesystem_scan is False
            assert mdc.do_process_scan is True
            assert mdc.scan_pids == ['1']

        @patch(LOGGER_TARGET)
        @patch.dict(os.environ)
        def test_removed_config_values(
            self, log_mock, yara, disabled, rules, cmd, create_test_files_fake_yara
        ):
            # If the user uses old config items, eg scan_only and scan_exclude, then notify them
            with open(TEMP_CONFIG_FILE, 'a') as f:
                f.write('scan_only: /tmp\nscan_exclude: /home\n')
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_any_call(
                "The 'scan_only' option has been replaced with the 'filesystem_scan_only' and 'processes_scan_only' options in "
                + TEMP_CONFIG_FILE
            )
            log_mock.error.assert_called_with(
                "Please remove the %s file and a new config file will be written with the new options",
                TEMP_CONFIG_FILE,
            )

            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "#scan_only: /tmp" if line.startswith("scan_only: ") else line
                print(line)
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_any_call(
                "The 'scan_exclude' option has been replaced with the 'filesystem_scan_exclude' and 'processes_scan_exclude' options in "
                + TEMP_CONFIG_FILE
            )

            os.environ['SCAN_ONLY'] = '/tmp'
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "#scan_exclude: /home" if line.startswith("scan_exclude: ") else line
                print(line)
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_any_call(
                "The 'scan_only' option has been replaced with the 'filesystem_scan_only' and 'processes_scan_only' options in "
                + TEMP_CONFIG_FILE
            )

            del os.environ['SCAN_ONLY']
            os.environ['FILESYSTEM_SCAN_ONLY'] = '/tmp'
            os.environ['SCAN_EXCLUDE'] = '/home'
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_any_call(
                "The 'scan_exclude' option has been replaced with the 'filesystem_scan_exclude' and 'processes_scan_exclude' options in "
                + TEMP_CONFIG_FILE
            )

            del os.environ['SCAN_EXCLUDE']
            os.environ['FILESYSTEM_SCAN_EXCLUDE'] = '/home'
            mdc = MalwareDetectionClient(None)
            assert mdc.scan_fsobjects == ['/tmp']
            assert mdc.filesystem_scan_exclude_list == [
                '/home',
                '/etc/insights-client/signatures'
            ]

        @patch(LOGGER_TARGET)
        def test_invalid_config_values(
            self, log_mock, yara, disabled, rules, cmd, create_test_files_fake_yara
        ):
            # Check the malware client app behaves in a predictable way if the user specifies invalid option values
            # in the config file.  Some of these will fail yaml parsing, others will fail type checking

            # Invalid value for nice - fails casting to an integer
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "nice_value: nineteen" if line.startswith("nice_value") else line
                print(line)
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_called_with(
                "Problem setting configuration option %s: %s", "nice_value", ANY
            )
            yara.assert_called_once()  # It failed after the _find_yara method

            # Missing colon for nice_value option - fails yaml parsing
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "test_scan: false" if line.startswith("test_scan:") else line
                line = "nice_value 19\n" if line.startswith("nice_value") else line
                print(line)
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_called_with(
                "Error encountered loading the malware-detection app config file %s:\n%s",
                TEMP_CONFIG_FILE,
                ANY,
            )
            yara.assert_called_once()  # It failed before the _find_yara method because it was invalid yaml

            # Bad list items for scan_only, mixing single item and list items - fails yaml parsing
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "nice_value: 19\n" if line.startswith("nice_value") else line
                line = (
                    "filesystem_scan_only: /bad\n- /bad"
                    if line.startswith("filesystem_scan_only:")
                    else line
                )
                print(line)
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_called_with(
                "Error encountered loading the malware-detection app config file %s:\n%s",
                TEMP_CONFIG_FILE,
                ANY,
            )
            yara.assert_called_once()  # It failed before the _find_yara method because it was invalid yaml

            # Bad list items for scan_only, not a list item -  fails yaml parsing
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "filesystem_scan_only:" if line.startswith("filesystem_scan_only:") else line
                line = "/bad" if line.startswith("- /bad") else line
                print(line)
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_called_with(
                "Error encountered loading the malware-detection app config file %s:\n%s",
                TEMP_CONFIG_FILE,
                ANY,
            )
            yara.assert_called_once()  # It failed before the _find_yara method because it was invalid yaml

            # Bad list items for scan_only, not enough spaces -  fails yaml parsing
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "-/bad" if line.startswith("/bad") else line
                print(line)
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_called_with(
                "Error encountered loading the malware-detection app config file %s:\n%s",
                TEMP_CONFIG_FILE,
                ANY,
            )
            yara.assert_called_once()  # It failed before the _find_yara method because it was invalid yaml

            # Bad list items for scan_only, using tabs instead of spaces -  fails yaml parsing
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "\t- /bad" if line.startswith("-/bad") else line
                print(line)
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_called_with(
                "Error encountered loading the malware-detection app config file %s:\n%s",
                TEMP_CONFIG_FILE,
                ANY,
            )
            yara.assert_called_once()  # It failed before the _find_yara method because it was invalid yaml

        # Patch the os.environ dict so all the changes are only temporary
        @patch(LOAD_CONFIG_TARGET, return_value=CONFIG)
        @patch.dict(os.environ)
        def test_using_env_vars(self, conf, yara, disabled, rules, cmd):
            # Set certain option values via environment variables
            env_var_list = [
                ('REMOTE_RULES_LOCATION', RULES_FILE),
                ('RULES_LOCATION', '/etc/insights-client/signatures'),
                ('TEST_SCAN', 'hello'),  # will be interpreted as false
                ('SCAN_FILESYSTEM', 'YES'),
                ('SCAN_PROCESSES', 'T'),
                ('FILESYSTEM_SCAN_ONLY', '/tmp'),
                ('FILESYSTEM_SCAN_EXCLUDE', '/tmp'),
                ('PROCESSES_SCAN_ONLY', '1,2'),
                ('PROCESSES_SCAN_EXCLUDE', '2,1'),
                ('FILESYSTEM_SCAN_SINCE', '2'),
                ('PROCESSES_SCAN_SINCE', '10'),
                ('SCAN_TIMEOUT', '1800'),
                ('CPU_THREAD_LIMIT', '1'),
            ]
            for key, value in env_var_list:
                os.environ[key] = value

            mdc = MalwareDetectionClient(None)
            assert mdc.yara_binary == FAKE_YARA
            assert mdc.rules_files == [RULES_FILE]
            assert mdc.test_scan is False
            assert mdc.do_filesystem_scan is True
            assert mdc.do_process_scan is True
            assert mdc.scan_fsobjects == ['/tmp']
            assert mdc.filesystem_scan_exclude_list == ['/tmp', '/etc/insights-client/signatures']
            if not IS_CONTAINER:
                assert mdc.scan_pids == ['1', '2']
                assert mdc.processes_scan_exclude_list == ['1', '2']
            assert mdc.filesystem_scan_since_dict['timestamp'] < time.time() - (2 * 86400)
            assert mdc.processes_scan_since_dict['timestamp'] < time.time() - (10 * 86400)
            assert mdc.scan_timeout == 1800
            # Not env vars, but just checking they have the expected values
            assert mdc.nice_value == 19
            assert mdc.cpu_thread_limit == 1
            assert mdc.rules_location == '/etc/insights-client/signatures'
            assert mdc.use_remote_rules is True

            # Start a filesystem scan and expect scan_only and scan_exclude to cancel each other out
            mdc.scan_filesystem()
            assert mdc.do_filesystem_scan is False

            # Start a process scan and expect scan_only and scan_exclude to cancel each other out
            mdc.scan_processes()
            assert mdc.do_process_scan is False

            # Test when SCAN_ONLY and SCAN_EXCLUDE values are comma separated
            # No items will be scanned because '/' is to be excluded
            for key, value in [
                ('FILESYSTEM_SCAN_ONLY', '/tmp,/,/var/tmp'),
                ('FILESYSTEM_SCAN_EXCLUDE', '/home,/,/fred,barney'),
            ]:
                os.environ[key] = value
            mdc = MalwareDetectionClient(None)
            assert mdc.do_filesystem_scan is True
            assert mdc.scan_fsobjects == ['/tmp', '/', '/var/tmp']
            assert mdc.filesystem_scan_exclude_list == ['/home', '/', '/etc/insights-client/signatures']
            mdc.scan_filesystem()
            assert mdc.do_filesystem_scan is False

            # Test when FILESYSTEM_SCAN_ONLY is empty, so no items will be scanned because '/' is to be excluded
            os.environ['FILESYSTEM_SCAN_ONLY'] = ''
            mdc = MalwareDetectionClient(None)
            assert mdc.do_filesystem_scan is True
            assert mdc.scan_fsobjects == []
            assert mdc.filesystem_scan_exclude_list == ['/home', '/', '/etc/insights-client/signatures']
            mdc.scan_filesystem()
            assert mdc.do_filesystem_scan is False

            # Test when FILESYSTEM_SCAN_EXCLUDE is empty
            os.environ['FILESYSTEM_SCAN_EXCLUDE'] = ''
            mdc = MalwareDetectionClient(None)
            assert mdc.scan_fsobjects == []
            assert mdc.filesystem_scan_exclude_list == ['/etc/insights-client/signatures']

            if not IS_CONTAINER:
                # Test when FILESYSTEM_SCAN_EXCLUDE is empty
                os.environ['PROCESSES_SCAN_ONLY'] = 'systemd,2,3..10'
                os.environ['PROCESSES_SCAN_EXCLUDE'] = '3..100,systemd,2'
                mdc = MalwareDetectionClient(None)
                assert all([pid in mdc.scan_pids for pid in ['1', '2', '3']])
                assert all([pid in mdc.processes_scan_exclude_list for pid in ['1', '2', '3']])

            # Further testing of list type env vars
            os.environ['NETWORK_FILESYSTEM_TYPES'] = ''
            assert mdc._get_config_option('network_filesystem_types') == []
            os.environ['NETWORK_FILESYSTEM_TYPES'] = 'nfs'
            assert mdc._get_config_option('network_filesystem_types') == ['nfs']
            os.environ['NETWORK_FILESYSTEM_TYPES'] = 'nfs,nfs4'
            assert mdc._get_config_option('network_filesystem_types') == ['nfs', 'nfs4']

        @patch(LOAD_CONFIG_TARGET, return_value=CONFIG)
        @patch(LOGGER_TARGET)
        @patch.dict(
            os.environ,
            {'TEST_SCAN': 'false', 'NICE_VALUE': 'nineteen', 'FILESYSTEM_SCAN_SINCE': 'blast'},
        )
        def test_invalid_env_vars(self, log_mock, conf, yara, disabled, rules, cmd):
            # NICE_VALUE and FILESYSTEM_SCAN_SINCE have invalid values
            # First time through the NICE_VALUE should generate an error
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_called_with(
                "Problem parsing environment variable %s: %s", "NICE_VALUE", ANY
            )

            # Set NICE_VALUE to proper value to avoid it giving an error this time.
            # Only FILESYSTEM_SCAN_SINCE should generate an error
            os.environ['NICE_VALUE'] = '19'
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_called_with(
                "Unknown value '%s' for %s option.  " "Valid values are integers >= 1 and 'last'",
                "blast",
                "filesystem_scan_since",
            )

            # Fix FILESYSTEM_SCAN_SINCE to proper value to avoid it giving an error this time.
            # Give an invalid value for PROCESSES_SCAN_SINCE, but need to enable SCAN_PROCESSES too
            os.environ['FILESYSTEM_SCAN_SINCE'] = 'last'
            os.environ['SCAN_PROCESSES'] = 'true'
            os.environ['PROCESSES_SCAN_SINCE'] = '0'
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_called_with(
                "Invalid processes_scan_since value 0.  Valid values are integers >= 1 and 'last'"
            )

            # Disable SCAN_PROCESSES and the invalid PROCESSES_SCAN_SINCE doesn't raise an error anymore
            # because it isn't even parsed
            os.environ['SCAN_PROCESSES'] = 'false'
            mdc = MalwareDetectionClient(None)
            assert hasattr(mdc, 'processes_scan_since_dict') is False

        def test_filesystem_scan_only_root(
            self, yara, disabled, rules, cmd, create_test_files_fake_yara
        ):
            # Nothing special about root when parsing the filesystem_scan_only option
            # There is no parsing of root to individual toplevel directories until running scan_filesystem
            filesystem_scan_only = '/'
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "test_scan: false" if line.startswith("test_scan:") else line
                line = (
                    "filesystem_scan_only: %s" % filesystem_scan_only
                    if line.startswith("filesystem_scan_only:")
                    else line
                )
                print(line)
            mdc = MalwareDetectionClient(None)
            assert mdc.scan_fsobjects == [filesystem_scan_only]
            # This is called by scan_filesystem to convert '/' into its top level subdirectories
            scan_dict = process_include_exclude_items(
                include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
            )
            assert all([x in list(scan_dict.keys()) for x in INCLUDED_TLDS])
            assert '/' not in list(scan_dict.keys())

            # Multiple directories aren't consolidated until later
            filesystem_scan_only = ['/', '/tmp', '/home']
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = (
                    "filesystem_scan_only: %s" % filesystem_scan_only
                    if line.startswith("filesystem_scan_only:")
                    else line
                )
                print(line)
            mdc = MalwareDetectionClient(None)
            assert mdc.scan_fsobjects == filesystem_scan_only
            scan_dict = process_include_exclude_items(
                include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
            )
            assert all([x in list(scan_dict.keys()) for x in INCLUDED_TLDS])
            assert '/' not in list(scan_dict.keys())

        @patch(LOGGER_TARGET)
        def test_filesystem_scan_exclude_root(
            self, log_mock, yara, disabled, rules, cmd, create_test_files_fake_yara
        ):
            # Nothing special about root when parsing the filesystem_scan_exclude option
            # There is no parsing of root to individual toplevel directories until running scan_filesystem
            # Add '/' to the list of filesystem_scan_exclude items.  Add it directly after the filesystem_scan_exclude: line
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "test_scan: false" if line.startswith("test_scan:") else line
                line = line + "\n- /" if line.startswith("filesystem_scan_exclude:") else line
                print(line)
            mdc = MalwareDetectionClient(None)
            assert '/' in mdc.filesystem_scan_exclude_list
            # When scan_filesystem is run, '/' will be expanded into toplevel directories that cancel out everything
            scan_dict = process_include_exclude_items(
                include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
            )
            assert scan_dict == {}
            log_mock.error.assert_called_with(
                "No filesystem items to scan because the specified exclude items cancel them out"
            )

        @patch(LOGGER_TARGET)
        def test_filesystem_scan_only_exclude_nullify(
            self, log_mock, yara, disabled, rules, cmd, create_test_files_fake_yara
        ):
            # Testing filesystem scan_only and scan_exclude items such that the exclude items nullify the scan_only items
            # In which case there will be nothing to scan
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "test_scan: false" if line.startswith("test_scan:") else line
                line = (
                    line + "\n- /var/log\n- /usr/lib/systemd\n- /tmp"
                    if line.startswith("filesystem_scan_only:")
                    else line
                )
                line = (
                    line + "\n- /tmp/\n- /usr/lib/\n- /var/log"
                    if line.startswith("filesystem_scan_exclude:")
                    else line
                )
                print(line)
            mdc = MalwareDetectionClient(None)
            assert mdc.scan_fsobjects == ['/var/log', '/usr/lib/systemd', '/tmp']
            assert all(
                [x in mdc.filesystem_scan_exclude_list for x in ['/tmp', '/usr/lib', '/var/log']]
            )
            # The exclude list covers all the items to be scanned, thus there is nothing to scan
            mdc.scan_filesystem()
            assert mdc.do_filesystem_scan is False
            log_mock.error.assert_called_with(
                "No filesystem items to scan because the specified exclude items cancel them out"
            )

            # Both filesystem scan_only and scan_exclude contain root
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = line + "\n- /" if line.startswith("filesystem_scan_only:") else line
                line = line + "\n- /" if line.startswith("filesystem_scan_exclude:") else line
                print(line)
            mdc = MalwareDetectionClient(None)
            assert mdc.scan_fsobjects == ['/', '/var/log', '/usr/lib/systemd', '/tmp']
            assert all(
                [
                    x in mdc.filesystem_scan_exclude_list
                    for x in ['/', '/tmp', '/usr/lib', '/var/log']
                ]
            )
            # Because both lists contain /, they will cancel each other out and there is nothing to scan
            mdc.scan_filesystem()
            assert mdc.do_filesystem_scan is False
            log_mock.error.assert_called_with(
                "No filesystem items to scan because the specified exclude items cancel them out"
            )

        @patch(CALL_TARGET)
        @patch(LOGGER_TARGET)
        @patch.dict(os.environ, {'TEST_SCAN': 'false'})
        def test_filesystem_scan_only_exclude_symlinks(
            self,
            log_mock,
            call_mock,
            disabled,
            yara,
            rules,
            cmd,
            extract_tmp_files,
            create_test_files_fake_yara,
        ):
            # Testing using symlinks for filesystem scan_only and scan_exclude items
            # Symlinks will be skipped, whereas broken symlinks are treated like missing files
            symlink = os.path.join(TEMP_TEST_DIR, 'scan_me/link_file')
            broken_symlink = os.path.join(TEMP_TEST_DIR, 'scan_me/broken_link')
            os.environ['FILESYSTEM_SCAN_ONLY'] = "%s,%s" % (symlink, broken_symlink)
            with pytest.raises(SystemExit):
                mdc = MalwareDetectionClient(None)
                assert mdc.scan_fsobjects == []
            log_mock.info.assert_any_call(
                "Skipping symlink filesystem_scan_only item: '%s'.  Please use non-symlink items",
                symlink,
            )
            log_mock.info.assert_any_call(
                "Skipping missing filesystem_scan_only item: '%s'", broken_symlink
            )
            log_mock.error.assert_called_with(
                "Nothing to scan with filesystem_scan_only option and scan_processes is disabled"
            )

            scan_only = os.path.join(TEMP_TEST_DIR, 'scan_me')
            os.environ['FILESYSTEM_SCAN_ONLY'] = scan_only
            os.environ['FILESYSTEM_SCAN_EXCLUDE'] = "%s,%s" % (symlink, broken_symlink)
            mdc = MalwareDetectionClient(None)
            assert mdc.filesystem_scan_exclude_list == ['/etc/insights-client/signatures']
            assert mdc.scan_fsobjects == [scan_only]
            log_mock.info.assert_any_call(
                "Skipping symlink filesystem_scan_exclude item: '%s'.  Please use non-symlink items",
                symlink,
            )
            log_mock.info.assert_any_call(
                "Skipping missing filesystem_scan_exclude item: '%s'", broken_symlink
            )
            log_mock.info.assert_any_call(
                "Unable to find the items specified for the filesystem_scan_exclude option.  Not excluding any filesystem items"
            )

        @patch(CALL_TARGET)
        @patch(LOGGER_TARGET)
        def test_network_filesystem_mountpoints(
            self,
            log_mock,
            call_mock,
            yara,
            disabled,
            rules,
            cmd,
            extract_tmp_files,
            create_test_files_fake_yara,
        ):
            # Test the exclude_network_filesystem_mountpoints option by 'creating' various mountpoints to exclude
            os.environ['EXCLUDE_NETWORK_FILESYSTEM_MOUNTPOINTS'] = (
                'true'  # Override the default setting for this test class
            )
            scan_me_scan_me = os.path.join(TEMP_TEST_DIR, 'scan_me/scan_me')
            scan_me_too = os.path.join(TEMP_TEST_DIR, 'scan_me_too')
            scan_me_not_mnt = os.path.join(TEMP_TEST_DIR, 'scan_me_not')
            dont_scan_me_mnt = os.path.join(TEMP_TEST_DIR, 'scan_me/dont_scan_me')

            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "test_scan: false" if line.startswith("test_scan:") else line
                line = (
                    line + "remote_rules_location: %s\n" % TEST_RULE_FILE
                    if line.startswith('---')
                    else line
                )
                line = (
                    "filesystem_scan_only: %s" % TEMP_TEST_DIR
                    if line.startswith("filesystem_scan_only:")
                    else line
                )
                print(line)

            # This is the mocked output returned from the findmnt command
            call_mock.return_value = '%s\n%s\n' % (scan_me_not_mnt, dont_scan_me_mnt)

            # Setting exclude_network_filesystem_mountpoints to false means we don't care about excluding mountpoints
            os.environ['EXCLUDE_NETWORK_FILESYSTEM_MOUNTPOINTS'] = 'false'
            os.environ['NETWORK_FILESYSTEM_TYPES'] = ''
            mdc = MalwareDetectionClient(None)
            assert mdc.network_filesystem_mountpoints == []

            # Removing the env var (it'll be true from the config file) but still not having any
            # network_filesystem_types value will generate an error
            del os.environ['EXCLUDE_NETWORK_FILESYSTEM_MOUNTPOINTS']
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_called_with(
                "No value specified for 'network_filesystem_types' option"
            )

            # Ok, now with exclude mountpoints true and a value for types we will produce a list of mountpoints
            os.environ['NETWORK_FILESYSTEM_TYPES'] = 'nfs'
            mdc = MalwareDetectionClient(None)
            assert mdc.network_filesystem_mountpoints == [scan_me_not_mnt, dont_scan_me_mnt]

            # Now we can process all the include and exclude items to build the scan_dict of things to scan
            scan_dict = process_include_exclude_items(
                include_items=mdc.scan_fsobjects,
                exclude_items=mdc.filesystem_scan_exclude_list,
                exclude_mountpoints=mdc.network_filesystem_mountpoints,
            )

            # The exclude_mountpoints will be added to the list of items to exclude
            assert list(scan_dict.keys()) == ['/tmp']
            assert sorted(list(scan_dict['/tmp']['exclude']['items'])) == sorted(
                [scan_me_not_mnt, dont_scan_me_mnt]
            )
            assert all([x in scan_dict['/tmp']['include'] for x in [scan_me_scan_me, scan_me_too]])
            # scan_me dir won't be in the list of include items because it has a sub-item to be excluded
            assert os.path.join(TEMP_TEST_DIR, 'scan_me') not in scan_dict['/tmp']['include']

            # Now make TEMP_TEST_DIR a mountpoint and it will cancel out TEMP_TEST_DIR for scan_only
            call_mock.return_value = '%s\n' % TEMP_TEST_DIR
            mdc = MalwareDetectionClient(None)
            assert mdc.network_filesystem_mountpoints == [TEMP_TEST_DIR]
            scan_dict = process_include_exclude_items(
                include_items=mdc.scan_fsobjects,
                exclude_items=mdc.filesystem_scan_exclude_list,
                exclude_mountpoints=mdc.network_filesystem_mountpoints,
            )
            assert scan_dict == {}
            log_mock.error.assert_called_with(
                "No filesystem items to scan because the specified exclude items cancel them out"
            )

        @patch(CALL_TARGET)
        @patch('os.path.samefile', side_effect=OSError(13, "Permission denied"))
        @patch(LOGGER_TARGET)
        @patch.dict(
            os.environ,
            {
                'EXCLUDE_NETWORK_FILESYSTEM_MOUNTPOINTS': 'true',
                'TEST_SCAN': 'false',
                'REMOTE_RULES_LOCATION': TEST_RULE_FILE,
                'FILESYSTEM_SCAN_ONLY': TEMP_TEST_DIR,
            },
        )
        def test_network_filesystem_permission_denied(
            self,
            log_mock,
            samefile_mock,
            call_mock,
            yara,
            disabled,
            rules,
            cmd,
            extract_tmp_files,
            create_test_files_fake_yara,
        ):
            # Test accessing network filesystem mountpoints that result in permission denied errors
            # Yes, even root can get permission denied trying to access fuse filesystems
            # Should produce the same results as the corresponding test in test_network_filesystem_mountpoints
            scan_me_not_mnt = os.path.join(TEMP_TEST_DIR, 'scan_me_not')
            dont_scan_me_mnt = os.path.join(TEMP_TEST_DIR, 'scan_me/dont_scan_me')
            # This is the mocked output returned from the findmnt command
            call_mock.return_value = '%s\n%s\n' % (scan_me_not_mnt, dont_scan_me_mnt)

            with patch(CONFIG_FILE_TARGET, TEMP_CONFIG_FILE):
                mdc = MalwareDetectionClient(None)
            assert mdc.network_filesystem_mountpoints == [scan_me_not_mnt, dont_scan_me_mnt]

            # process_include_exclude_items uses os.path.samefile, but should gracefully handle the permission denied errors
            scan_dict = process_include_exclude_items(
                include_items=mdc.scan_fsobjects,
                exclude_items=mdc.filesystem_scan_exclude_list,
                exclude_mountpoints=mdc.network_filesystem_mountpoints,
            )
            assert sorted(list(scan_dict['/tmp']['exclude']['items'])) == sorted(
                [scan_me_not_mnt, dont_scan_me_mnt]
            )

        @pytest.mark.skipif(IS_CONTAINER, reason=SKIP_IF_CONTAINER_REASON)
        def test_processes_scan_options(
            self, yara, disabled, rules, cmd, create_test_files_fake_yara
        ):
            # Test the processes scan options
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "test_scan: false" if line.startswith("test_scan:") else line
                line = "scan_filesystem: false" if line.startswith("scan_filesystem:") else line
                line = "scan_processes: true" if line.startswith("scan_processes:") else line
                line = (
                    "processes_scan_only: 1\n" if line.startswith("processes_scan_only:") else line
                )
                line = (
                    "processes_scan_exclude: 1\n"
                    if line.startswith("processes_scan_exclude:")
                    else line
                )
                print(line)
            mdc = MalwareDetectionClient(None)
            assert mdc.scan_pids == ['1']
            assert mdc.processes_scan_exclude_list == ['1']

            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = (
                    "processes_scan_only: 1..10\n"
                    if line.startswith("processes_scan_only:")
                    else line
                )
                line = (
                    "processes_scan_exclude: 1..10\n"
                    if line.startswith("processes_scan_exclude:")
                    else line
                )
                print(line)
            mdc = MalwareDetectionClient(None)
            assert len(mdc.scan_pids) > 1
            assert all([pid in mdc.scan_pids for pid in ['1', '2']])
            assert len(mdc.processes_scan_exclude_list) > 1
            assert all([pid in mdc.processes_scan_exclude_list for pid in ['1', '2']])

            # Test an open ended range, but really just saves typing '1' - scan all processes from 1 to 10
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = (
                    "processes_scan_only: ..10\n"
                    if line.startswith("processes_scan_only:")
                    else line
                )
                line = (
                    "processes_scan_exclude: ..10\n"
                    if line.startswith("processes_scan_exclude:")
                    else line
                )
                print(line)
            mdc = MalwareDetectionClient(None)
            assert len(mdc.scan_pids) > 1
            assert all([int(pid) <= 10 for pid in mdc.scan_pids])
            assert len(mdc.processes_scan_exclude_list) > 1
            assert all([int(pid) <= 10 for pid in mdc.processes_scan_exclude_list])

            # Testing an open ended range - scan all processes from 101 to the max_pid value
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = (
                    "processes_scan_only: 101..\n"
                    if line.startswith("processes_scan_only:")
                    else line
                )
                line = (
                    "processes_scan_exclude: 101..\n"
                    if line.startswith("processes_scan_exclude:")
                    else line
                )
                print(line)
            mdc = MalwareDetectionClient(None)
            assert len(mdc.scan_pids) > 1
            assert all([int(pid) > 100 for pid in mdc.scan_pids])
            assert len(mdc.processes_scan_exclude_list) > 1
            assert all([int(pid) > 100 for pid in mdc.processes_scan_exclude_list])

            # Not invalid ranges, just testing them to confirm they are handled gracefully
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = (
                    "processes_scan_only: 1...10\n"
                    if line.startswith("processes_scan_only:")
                    else line
                )
                line = (
                    "processes_scan_exclude: .1.....10.\n"
                    if line.startswith("processes_scan_exclude:")
                    else line
                )
                print(line)
            mdc = MalwareDetectionClient(None)
            assert len(mdc.scan_pids) > 1
            assert all([int(pid) <= 10 for pid in mdc.scan_pids])
            assert len(mdc.processes_scan_exclude_list) > 1
            assert all([int(pid) <= 10 for pid in mdc.processes_scan_exclude_list])

            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = (
                    "processes_scan_only: systemd\n"
                    if line.startswith("processes_scan_only:")
                    else line
                )
                line = (
                    "processes_scan_exclude: systemd\n"
                    if line.startswith("processes_scan_exclude:")
                    else line
                )
                print(line)
            mdc = MalwareDetectionClient(None)
            assert len(mdc.scan_pids) > 1
            assert '1' in mdc.scan_pids
            assert len(mdc.processes_scan_exclude_list) > 1
            assert '1' in mdc.processes_scan_exclude_list

            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = (
                    "processes_scan_only:\n- 2\n- 3..10\n- systemd"
                    if line.startswith("processes_scan_only:")
                    else line
                )
                line = (
                    "processes_scan_exclude:\n- systemd\n- 2\n- 3..10"
                    if line.startswith("processes_scan_exclude:")
                    else line
                )
                print(line)
            mdc = MalwareDetectionClient(None)
            assert len(mdc.scan_pids) > 1
            assert all([pid in mdc.scan_pids for pid in ['1', '2', '3']])
            assert len(mdc.processes_scan_exclude_list) > 1
            assert all([pid in mdc.processes_scan_exclude_list for pid in ['1', '2', '3']])

        @patch(LOGGER_TARGET)
        def test_processes_scan_options_invalid_or_missing_values(
            self, log_mock, yara, disabled, rules, cmd, create_test_files_fake_yara
        ):
            # Test the processes scan options
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "test_scan: false" if line.startswith("test_scan:") else line
                line = "scan_filesystem: false" if line.startswith("scan_filesystem:") else line
                line = "scan_processes: true" if line.startswith("scan_processes:") else line
                line = (
                    line + "\n- pid2\n- three..10\n- 20.50\n- notsystemd"
                    if line.startswith("processes_scan_only:")
                    else line
                )
                print(line)
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            log_mock.error.assert_any_call(
                "Unable to parse '%s' in to a range of PIDs: %s", 'three..10', ANY
            )
            log_mock.error.assert_any_call(
                "Unable to find the items specified for the processes_scan_only option.  Skipping ..."
            )
            log_mock.error.assert_any_call(
                "Nothing to scan with processes_scan_only option and scan_filesystem is disabled"
            )

            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = line + "\n- 1" if line.startswith("processes_scan_only:") else line
                line = (
                    line + "\n- -1\n- 3..ten\n- 123 systemd"
                    if line.startswith("processes_scan_exclude:")
                    else line
                )
                print(line)
            mdc = MalwareDetectionClient(None)
            assert mdc.scan_pids == ['1']
            assert mdc.processes_scan_exclude_list == []
            log_mock.error.assert_any_call(
                "Unable to parse '%s' in to a range of PIDs: %s", '3..ten', ANY
            )
            log_mock.error.assert_any_call(
                "Unable to find the items specified for the processes_scan_exclude option.  Not excluding any processes."
            )

    @patch(BUILD_YARA_COMMAND_TARGET, return_value=(['lol'], []))
    @patch(GET_RULES_TARGET, return_value=[RULES_FILE])
    @patch(DISABLED_RULES_TARGET)
    @patch(LOAD_CONFIG_TARGET, return_value=CONFIG)
    @patch(LOGGER_TARGET)
    class TestFindYara:

        def test_find_yara_binary(self, log_mock, conf, disabled, rules, cmd):
            # Testing finding yara with correct version
            with patch('os.path.exists', return_value=True):
                with patch(CALL_TARGET, return_value='4.1'):
                    mdc = MalwareDetectionClient(None)
            assert mdc.yara_binary == '/bin/yara'
            assert mdc.yara_version == '4.1'
            cmd.assert_called()

            # 'Find' yara in /usr/bin/yara (fails to 'find' /bin/yara)
            with patch('os.path.exists', side_effect=[False, True]):
                with patch(CALL_TARGET, return_value='4.1'):
                    mdc = MalwareDetectionClient(None)
            assert mdc.yara_binary == '/usr/bin/yara'
            assert mdc.yara_version == '4.1'
            cmd.assert_called()

        def test_find_unsupported_yara(self, log_mock, conf, disabled, rules, cmd):
            # Test finding unsupported yara version
            with patch('os.path.exists', return_value=True):
                with patch(CALL_TARGET, return_value='3.10'):
                    with pytest.raises(SystemExit):
                        mdc = MalwareDetectionClient(None)
                        assert mdc.yara_version == '3.10'
            log_mock.error.assert_called_with(
                "Found /bin/yara with version 3.10, but malware-detection requires version >= 4.1.0\n"
                "Please install a later version of yara."
            )
            cmd.assert_not_called()

        def test_find_invalid_yara(self, log_mock, conf, disabled, rules, cmd):
            # Test finding a binary called yara, but its not yara
            with patch('os.path.exists', return_value=True):
                with patch(CALL_TARGET, return_value='not yara 1.2.3'):
                    with pytest.raises(SystemExit):
                        MalwareDetectionClient(None)
            log_mock.error.assert_called_with(
                "Error getting the version of the specified yara binary %s: %s", "/bin/yara", ANY
            )
            cmd.assert_not_called()

        def test_cant_find_yara(self, log_mock, conf, disabled, rules, cmd):
            # Test can't find yara on the system
            with patch('os.path.exists', return_value=False):
                with pytest.raises(SystemExit):
                    MalwareDetectionClient(None)
            log_mock.error.assert_called_with(
                "Couldn't find yara.  Please ensure the yara package is installed"
            )
            cmd.assert_not_called()

        @patch("os.path.exists", return_value=True)
        @patch(CALL_TARGET)  # mock call to 'yara --version'
        def test_yara_versions(
            self, version_mock, exists_mock, log_mock, conf, disabled, rules, cmd
        ):
            # Test checking the version of yara
            # Invalid versions of yara
            for version in ['4.0.99', '4']:
                version_mock.return_value = version
                with pytest.raises(SystemExit):
                    MalwareDetectionClient(None)
            cmd.assert_not_called()  # We won't get to the build_yara_cmd method because we exit before its called

            # Valid versions of yara
            for version in ['4.1', '4.10.10', '10.100.1000.0', '5']:
                version_mock.return_value = version
                mdc = MalwareDetectionClient(None)
                assert mdc.yara_binary == '/bin/yara'
                assert mdc.yara_version in ['4.1', '4.10.10', '10.100.1000.0', '5']
            cmd.assert_called()

    # Use patch.object, just because I wanted to try using patch.object instead of using patch all the time :shrug:
    @patch(
        NAMEDTMPFILE_TARGET
    )  # Mock NamedTemporaryFile so it doesn't try to create the temporary file
    @patch('os.remove')  # Mock os.remove so it doesn't actually try to remove any existing files
    @patch.object(MalwareDetectionClient, '_parse_exclude_network_filesystem_mountpoints_option')
    @patch.object(
        InsightsConnection, 'get', return_value=Mock(status_code=200, content=b"Rule Content")
    )
    @patch.object(InsightsConnection, 'get_proxies')
    @patch.object(InsightsConnection, '_init_session', return_value=Mock())
    @patch.object(MalwareDetectionClient, '_get_disabled_rules', return_value=[])
    @patch.object(MalwareDetectionClient, '_build_yara_command', return_value=(['lol'], []))
    @patch.object(MalwareDetectionClient, '_load_config', return_value=CONFIG)
    # NOTE: Downloading the malware rules file happens within the malware client code so it's possible to test it here
    # However uploading the results archive is done outside the malware client code so it's not possible to test here
    class TestGetRules:
        """Testing the _get_rules method"""

        @patch('os.path.isdir')
        @patch('os.walk')
        @patch('fnmatch.filter')
        @patch(LOGGER_TARGET)
        @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
        @patch.dict(os.environ, {'RULES_LOCATION': '/test/rules'})
        def test_get_rules_directory_with_matching_files(
                self, yara, log_mock, fnmatch_mock, walk_mock, isdir_mock,
                conf, cmd, disabled, session, proxies, get, findmnt, remove, tmpfile
        ):
            """Test _get_rules when rules_path is a directory containing matching rule files"""
            # Mock directory structure with rule files
            isdir_mock.return_value = True
            walk_mock.return_value = [
                ('/test/rules', ['subdir'], ['rule1.yar', 'rule2.yc', 'other.txt']),
                ('/test/rules/subdir', [], ['rule3.yara', 'readme.md'])
            ]

            # Mock fnmatch.filter to return matching files for each pattern
            def filter_side_effect(filenames, pattern):
                matches = {
                    '*.yar': ['rule1.yar'],
                    '*.yc': ['rule2.yc'],
                    '*.yara': ['rule3.yara']
                }
                return matches.get(pattern, [])

            fnmatch_mock.side_effect = filter_side_effect

            # Create MalwareDetectionClient with mock rules_location
            mdc = MalwareDetectionClient(InsightsConfig())

            # Verify os.walk was called on the correct path
            walk_mock.assert_called_once_with('/test/rules')

            # Verify the correct rule files were found and added
            expected_files = [
                '/test/rules/rule1.yar',
                '/test/rules/rule2.yc',
                '/test/rules/subdir/rule3.yara'
            ]
            for expected_file in expected_files:
                assert expected_file in mdc.rules_files

            # Verify no warning was logged since files were found
            log_mock.warning.assert_not_called()

        @patch('os.path.isdir')
        @patch('os.walk')
        @patch('fnmatch.filter')
        @patch(LOGGER_TARGET)
        @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
        @patch.dict(os.environ, {'RULES_LOCATION': '/test/rules'})
        def test_get_rules_directory_with_no_matching_files(
                self, yara, log_mock, fnmatch_mock, walk_mock, isdir_mock,
                conf, cmd, disabled, session, proxies, get, findmnt, remove, tmpfile
        ):
            """Test _get_rules when rules_path is a directory with no matching rule files"""
            # Mock directory structure with no rule files
            isdir_mock.return_value = True
            walk_mock.return_value = [
                ('/test/rules', ['subdir'], ['readme.txt', 'config.json']),
                ('/test/rules/subdir', [], ['notes.md', 'script.sh'])
            ]

            # Mock fnmatch.filter to return empty lists (no matches)
            fnmatch_mock.return_value = []

            # Create MalwareDetectionClient with mock rules_location
            mdc = MalwareDetectionClient(InsightsConfig())

            # Verify os.walk was called
            walk_mock.assert_called_once_with('/test/rules')

            # Verify warning was logged for no rule files found
            log_mock.warning.assert_called_once_with(
                "No rule files found in directory: /test/rules"
            )

            # Verify no rule files were added (empty list or only remote rules)
            # Note: rules_files might contain remote rules, so we check that no local files were added
            local_rule_files = [f for f in mdc.rules_files if f.__str__().startswith('/test/rules')]
            assert len(local_rule_files) == 0

        @patch('os.path.isdir')
        @patch('os.walk')
        @patch('fnmatch.filter')
        @patch(LOGGER_TARGET)
        @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
        @patch.dict(os.environ, {'RULES_LOCATION': '/test/empty'})
        def test_get_rules_empty_directory(
                self, yara, log_mock, fnmatch_mock, walk_mock, isdir_mock,
                conf, cmd, disabled, session, proxies, get, findmnt, remove, tmpfile
        ):
            """Test _get_rules when rules_path is an empty directory"""
            # Mock empty directory
            isdir_mock.return_value = True
            walk_mock.return_value = [('/test/empty', [], [])]
            fnmatch_mock.return_value = []

            # Create MalwareDetectionClient with mock rules_location
            MalwareDetectionClient(InsightsConfig())

            # Verify os.walk was called
            walk_mock.assert_called_once_with('/test/empty')

            # Verify warning was logged
            log_mock.warning.assert_called_once_with(
                "No rule files found in directory: /test/empty"
            )

            os.environ['USE_REMOTE_RULES'] = 'false'
            # Create MalwareDetectionClient with mock rules_location and remote rules disabled
            with pytest.raises(SystemExit):
                MalwareDetectionClient(InsightsConfig())
            # Verify error was logged for no rule files found
            log_mock.error.assert_called_once_with(
                "No rules have been found. Please enable use_remote_rules or add rules to rules_location directory"
            )

            os.environ['USE_REMOTE_RULES'] = 'true'
            os.environ['REMOTE_RULES_LOCATION'] = '/test/empty'
            # Create MalwareDetectionClient with mock rules_location and remote rules location set to fake value
            with pytest.raises(SystemExit):
                MalwareDetectionClient(InsightsConfig())

            # Verify error was logged for no rule files found
            log_mock.error.assert_called_with("Couldn't find specified rules file: %s", '/test/empty')

        @patch('os.path.isdir')
        @patch('os.walk')
        @patch('fnmatch.filter')
        @patch(LOGGER_TARGET)
        @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
        @patch.dict(os.environ, {'RULES_LOCATION': '/test/mixed'})
        def test_get_rules_directory_mixed_files(
                self, yara, log_mock, fnmatch_mock, walk_mock, isdir_mock,
                conf, cmd, disabled, session, proxies, get, findmnt, remove, tmpfile
        ):
            """Test _get_rules with directory containing mix of rule and non-rule files"""
            # Mock directory with mixed file types
            isdir_mock.return_value = True
            walk_mock.return_value = [
                ('/test/mixed', ['sub1', 'sub2'], ['test.yar', 'readme.txt', 'malware.yc']),
                ('/test/mixed/sub1', [], ['rules.yara', 'config.xml']),
                ('/test/mixed/sub2', [], ['data.json', 'script.py'])
            ]

            # Mock fnmatch.filter to return appropriate matches per pattern
            def filter_side_effect(filenames, pattern):
                file_matches = {
                    ('test.yar', 'readme.txt', 'malware.yc'): {
                        '*.yar': ['test.yar'],
                        '*.yc': ['malware.yc'],
                        '*.yara': []
                    },
                    ('rules.yara', 'config.xml'): {
                        '*.yar': [],
                        '*.yc': [],
                        '*.yara': ['rules.yara']
                    },
                    ('data.json', 'script.py'): {
                        '*.yar': [],
                        '*.yc': [],
                        '*.yara': []
                    }
                }

                for file_tuple, matches in file_matches.items():
                    if set(filenames) == set(file_tuple):
                        return matches.get(pattern, [])
                return []

            fnmatch_mock.side_effect = filter_side_effect

            # Create MalwareDetectionClient with mock rules_location
            mdc = MalwareDetectionClient(InsightsConfig())

            # Verify expected rule files were found
            expected_files = [
                '/test/mixed/test.yar',
                '/test/mixed/malware.yc',
                '/test/mixed/sub1/rules.yara'
            ]

            for expected_file in expected_files:
                assert expected_file in mdc.rules_files

            # Verify no warning was logged since rule files were found
            log_mock.warning.assert_not_called()

        @patch('os.path.isdir')
        @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
        @patch.dict(os.environ, {'RULES_LOCATION': '/test/not_a_dir', 'USE_REMOTE_RULES': 'false'})
        def test_get_rules_not_directory(
                self, yara, isdir_mock, conf, cmd, disabled, session, proxies, get, findmnt, remove, tmpfile
        ):
            """Test _get_rules when rules_path is not a directory (skip directory processing)"""
            # Mock that rules_path is not a directory
            isdir_mock.return_value = False

            # Create MalwareDetectionClient
            # Mock os.path.isfile to return True to test the elif branch
            with patch('os.path.isfile', return_value=True):
                mdc = MalwareDetectionClient(InsightsConfig())

                # Verify isdir was called but os.walk was not called
                isdir_mock.assert_called_once()

                # Since it's not a directory, the directory scanning code should be skipped
                # and the file path should be added directly (tested in elif branch)
                assert '/test/not_a_dir' in mdc.rules_files

        @patch.dict(os.environ, {'TEST_SCAN': 'true'})
        @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
        def test_download_rules_cert_auth(
            self, yara, conf, cmd, disabled, session, proxies, get, findmnt, remove, tmpfile
        ):
            # Test the standard remote_rules_location urls, but will result in cert auth being used to download the rules
            # Test with insights-config None, expect an error when trying to use the insights-config object
            with pytest.raises(SystemExit):
                MalwareDetectionClient(None)
            session.assert_not_called()

            # With default insights config and test scan true ...
            # Expect to use cert auth because no username or password specified and expect to download test-rule.yar
            mdc = MalwareDetectionClient(InsightsConfig())
            assert (
                mdc.remote_rules_location
                == "https://cert.console.redhat.com/api/malware-detection/v1/test-rule.yar"
            )
            # assumes first rule will be remote rule
            assert mdc.rules_files[0].startswith('/tmp')  # rules will be saved into a temp file
            get.assert_called_with(
                "https://cert.console.redhat.com/api/malware-detection/v1/test-rule.yar",
                log_response_text=True,
                verify=True,
                stream=True,
            )

            # With authmethod=CERT, expect 'cert.' to be prefixed to the url
            mdc = MalwareDetectionClient(InsightsConfig(authmethod='CERT'))
            assert (
                mdc.remote_rules_location
                == "https://cert.console.redhat.com/api/malware-detection/v1/test-rule.yar"
            )
            get.assert_called_with(
                "https://cert.console.redhat.com/api/malware-detection/v1/test-rule.yar",
                log_response_text=True,
                verify=True,
                stream=True,
            )

            # With authmethod=BASIC and test scan false ...
            # Expect to still use cert auth because no username or password specified
            os.environ['TEST_SCAN'] = 'false'
            mdc = MalwareDetectionClient(InsightsConfig(authmethod='BASIC'))
            assert (
                mdc.remote_rules_location
                == "https://cert.console.redhat.com/api/malware-detection/v1/signatures.yar"
            )
            get.assert_called_with(
                "https://cert.console.redhat.com/api/malware-detection/v1/signatures.yar",
                log_response_text=False,
                verify=True,
                stream=True,
            )

            mdc = MalwareDetectionClient(InsightsConfig(authmethod='CERT'))
            assert (
                mdc.remote_rules_location
                == "https://cert.console.redhat.com/api/malware-detection/v1/signatures.yar"
            )
            get.assert_called_with(
                "https://cert.console.redhat.com/api/malware-detection/v1/signatures.yar",
                log_response_text=False,
                verify=True,
                stream=True,
            )

        @patch.dict(os.environ, {'TEST_SCAN': 'true'})
        @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
        @patch(LOGGER_TARGET)
        def test_download_rules_basic_auth(
            self,
            log_mock,
            yara,
            conf,
            cmd,
            disabled,
            session,
            proxies,
            get,
            findmnt,
            remove,
            tmpfile,
        ):
            # Test the standard remote_rules_location urls, with basic auth attempting to be used to download the rules
            # Basic auth is used by default, but needs to have a valid username and password for it to work
            # Without a username and password, then cert auth will be used
            expected_rules_url = "https://console.redhat.com/api/malware-detection/v1/test-rule.yar"

            # Test with just a username specified - expect basic auth to be used but fails
            get.return_value = Mock(status_code=401, reason="Unauthorized", text="No can do")
            with pytest.raises(SystemExit):
                MalwareDetectionClient(InsightsConfig(username='user'))
            get.assert_called_with(
                expected_rules_url, log_response_text=True, verify=True, stream=True
            )
            log_mock.error.assert_called_with(
                "Unable to download rules from %s: %s",
                expected_rules_url,
                "401 Unauthorized: No can do",
            )

            # Test with just a password specified - expect basic auth to be used but fails
            with pytest.raises(SystemExit):
                MalwareDetectionClient(InsightsConfig(password='pass'))
            get.assert_called_with(
                expected_rules_url, log_response_text=True, verify=True, stream=True
            )
            log_mock.error.assert_called_with(
                "Unable to download rules from %s: %s",
                expected_rules_url,
                "401 Unauthorized: No can do",
            )

            # Test with 'incorrect' username and/or password - expect basic auth failure
            with pytest.raises(SystemExit):
                MalwareDetectionClient(InsightsConfig(username='user', password='badpass'))
            get.assert_called_with(
                expected_rules_url, log_response_text=True, verify=True, stream=True
            )
            log_mock.error.assert_called_with(
                "Unable to download rules from %s: %s",
                expected_rules_url,
                "401 Unauthorized: No can do",
            )

            # Test with 'correct' username and password - expect basic auth success
            get.return_value = Mock(status_code=200, content=b"Rule Content")
            mdc = MalwareDetectionClient(InsightsConfig(username='user', password='goodpass'))
            assert mdc.remote_rules_location == expected_rules_url
            get.assert_called_with(
                expected_rules_url, log_response_text=True, verify=True, stream=True
            )

        @patch.dict(os.environ, {'TEST_SCAN': 'false'})
        @patch("os.path.exists", return_value=True)  # mock call to os.path.exists in _find_yara
        @patch(CALL_TARGET, return_value='4.2.1')  # mock call to 'yara --version'
        def test_download_rules_versioned_files(
            self,
            version_mock,
            exists_mock,
            conf,
            cmd,
            disabled,
            session,
            proxies,
            get,
            findmnt,
            remove,
            tmpfile,
        ):
            # Test downloading different signatures files depending on the yara version found on the system
            expected_rules_url = (
                "https://cert.console.redhat.com/api/malware-detection/v1/signatures.yar"
            )
            mdc = MalwareDetectionClient(InsightsConfig())
            assert mdc.yara_version == '4.2.1'
            assert mdc.remote_rules_location == expected_rules_url + "?yara_version=4.2.1"
            get.assert_called_with(
                expected_rules_url + "?yara_version=4.2.1",
                log_response_text=False,
                verify=True,
                stream=True,
            )

            version_mock.return_value = '4.1.0'
            mdc = MalwareDetectionClient(InsightsConfig())
            assert mdc.remote_rules_location == expected_rules_url + "?yara_version=4.1.0"
            get.assert_called_with(
                expected_rules_url + "?yara_version=4.1.0",
                log_response_text=False,
                verify=True,
                stream=True,
            )

            version_mock.return_value = '10.100'
            mdc = MalwareDetectionClient(InsightsConfig())
            assert mdc.remote_rules_location == expected_rules_url + "?yara_version=10.100"
            get.assert_called_with(
                expected_rules_url + "?yara_version=10.100",
                log_response_text=False,
                verify=True,
                stream=True,
            )

            # Bypass the _find_yara method so self.yara_version isn't set
            with patch(FIND_YARA_TARGET, return_value=FAKE_YARA):
                mdc = MalwareDetectionClient(InsightsConfig())
            assert not hasattr(mdc, 'yara_version')
            assert mdc.remote_rules_location == expected_rules_url
            get.assert_called_with(
                expected_rules_url, log_response_text=False, verify=True, stream=True
            )

        @patch.dict(os.environ, {'TEST_SCAN': 'false'})
        @patch("os.path.exists", return_value=True)  # mock call to os.path.exists in _find_yara
        @patch(CALL_TARGET, return_value='4.2.0')  # mock call to 'yara --version'
        def test_download_rules_from_stage(
            self,
            version_mock,
            exists_mock,
            conf,
            cmd,
            disabled,
            session,
            proxies,
            get,
            findmnt,
            remove,
            tmpfile,
        ):
            # Test downloading signatures files from the stage environment
            # cert_verify is set to 'False' (string) for stage - check its changed to False (boolean)

            base_url = "cert.console.stage.example.com:443/r/insights"
            expected_rules_url = "https://cert.console.stage.example.com/api/malware-detection/v1/signatures.yar?yara_version=4.2.0"
            mdc = MalwareDetectionClient(InsightsConfig(base_url=base_url, cert_verify='False'))
            assert mdc.yara_version == '4.2.0'
            assert mdc.remote_rules_location == expected_rules_url
            get.assert_called_with(
                expected_rules_url, log_response_text=False, verify=False, stream=True
            )

            for base_url in (
                'cert.console.stage.example.com/r/insights',
                'cert.console.stage.example.com',
            ):
                mdc = MalwareDetectionClient(InsightsConfig(base_url=base_url, cert_verify='false'))
                assert mdc.remote_rules_location == expected_rules_url
                get.assert_called_with(
                    expected_rules_url, log_response_text=False, verify=False, stream=True
                )

            base_url = "https://cert.console.stage.example.com:443/r/insights"
            expected_rules_url = "https://cert.console.stage.example.com:443/api/malware-detection/v1/signatures.yar?yara_version=4.2.0"
            mdc = MalwareDetectionClient(InsightsConfig(base_url=base_url, cert_verify='True'))
            assert mdc.remote_rules_location == expected_rules_url
            get.assert_called_with(
                expected_rules_url, log_response_text=False, verify=True, stream=True
            )

            os.environ['TEST_SCAN'] = 'true'
            base_url = "cloud.stage.example.com"
            expected_rules_url = (
                "https://cloud.stage.example.com/api/malware-detection/v1/test-rule.yar"
            )
            mdc = MalwareDetectionClient(InsightsConfig(base_url=base_url, cert_verify=''))
            assert mdc.remote_rules_location == expected_rules_url
            get.assert_called_with(
                expected_rules_url, log_response_text=True, verify=True, stream=True
            )

        @patch.dict(
            os.environ, {'TEST_SCAN': 'true', 'REMOTE_RULES_LOCATION': 'console.redhat.com/rules.yar'}
        )
        @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
        def test_get_rules_missing_protocol(
            self, yara, conf, cmd, disabled, session, proxies, get, findmnt, remove, tmpfile
        ):
            # Non-standard rules URLS - without https:// at the start and not signatures.yar
            # test-scan true and BASIC auth by default expect test-rule.yar and no 'cert.' in URL
            mdc = MalwareDetectionClient(InsightsConfig(username='user', password='pass'))
            assert mdc.remote_rules_location == "https://console.redhat.com/test-rule.yar"
            get.assert_called_with(
                "https://console.redhat.com/test-rule.yar",
                log_response_text=True,
                verify=True,
                stream=True,
            )

            # test-scan false and CERT auth - expect 'cert.' prefixed to the URL and not test-rule.yar
            os.environ['TEST_SCAN'] = 'false'
            mdc = MalwareDetectionClient(InsightsConfig(authmethod='CERT'))
            assert mdc.remote_rules_location == "https://cert.console.redhat.com/rules.yar"
            get.assert_called_with(
                "https://cert.console.redhat.com/rules.yar",
                log_response_text=False,
                verify=True,
                stream=True,
            )

        @patch.dict(
            os.environ, {'TEST_SCAN': 'false', 'REMOTE_RULES_LOCATION': 'http://localhost/rules.yar'}
        )
        @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
        @patch(LOGGER_TARGET)
        def test_download_failures(
            self,
            log_mock,
            yara,
            conf,
            cmd,
            disabled,
            session,
            proxies,
            get,
            findmnt,
            remove,
            tmpfile,
        ):
            from requests.exceptions import ConnectionError, Timeout

            # Test various problems downloading rules
            expected_rules_url = os.environ['REMOTE_RULES_LOCATION']

            # 404 error - unlikely to occur unless an incorrect remote_rules_location was manually specified
            get.return_value = Mock(status_code=404, reason="Not found", text="Nup")
            with pytest.raises(SystemExit):
                MalwareDetectionClient(InsightsConfig())
            log_mock.error.assert_called_with(
                "Unable to download rules from %s: %s", expected_rules_url, "404 Not found: Nup"
            )
            assert get.call_count == 1

            # Test other errors downloading rules from the backend - these are more likely to occur
            # Firstly handling an error like connection refused (Couldn't connect)
            get.side_effect = [ConnectionError("Couldn't connect"), Timeout("Timeout")]
            with pytest.raises(SystemExit):
                MalwareDetectionClient(InsightsConfig(username='user', password='pass'))
            log_mock.error.assert_called_with(
                "Unable to download rules from %s: %s", expected_rules_url, "Couldn't connect"
            )
            assert get.call_count == 2

            # Then handling a Timeout error
            # Note, because we aren't downloading from console.redhat.com, there won't be 'cert.*' appended to the URL
            with pytest.raises(SystemExit):
                MalwareDetectionClient(InsightsConfig())
            log_mock.error.assert_called_with(
                "Unable to download rules from %s: %s", expected_rules_url, "Timeout"
            )
            assert get.call_count == 3

        @pytest.mark.skipif(
            not (TEST_DOWNLOAD_FAILURE_RETRIES or TEST_ALL),
            reason='test_download_failure_retries is slow due to the inherent delay in the retry logic. '
            'Use TEST_DOWNLOAD_FAILURE_RETRIES=True to enable test',
        )
        @patch.dict(
            os.environ, {'TEST_SCAN': 'false', 'REMOTE_RULES_LOCATION': 'http://localhost/rules.yar'}
        )
        @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
        @patch("os.path.isfile", return_value=True)
        @patch(LOGGER_TARGET)
        def test_download_failure_retries(
            self,
            log_mock,
            isfile,
            yara,
            conf,
            cmd,
            disabled,
            session,
            proxies,
            get,
            findmnt,
            remove,
            tmpfile,
        ):
            from requests.exceptions import ConnectionError, SSLError

            # Testing the download failure retry logic
            expected_rules_url = os.environ['REMOTE_RULES_LOCATION']

            # Status code != 200 will trigger a retry, but only if retries > 1
            get.return_value = Mock(status_code=404, reason="Not found", text="Nup")
            with pytest.raises(SystemExit):
                MalwareDetectionClient(InsightsConfig(retries=1))
            log_mock.error.assert_called_with(
                "Unable to download rules from %s: %s", expected_rules_url, "404 Not found: Nup"
            )
            # Last call to logger.debug will be about downloading rules, not about retrying, which shows retrying wasn't invoked
            log_mock.debug.assert_called_with("Using cert_verify value %s ...", True)
            log_mock.debug.assert_any_call("Downloading rules from: %s", expected_rules_url)

            # Set retries > 1 and now retrying happens
            with pytest.raises(SystemExit):
                MalwareDetectionClient(InsightsConfig(retries=2))
            log_mock.error.assert_called_with(
                "Unable to download rules from %s: %s", expected_rules_url, "404 Not found: Nup"
            )
            # This time, the last call to logger.debug will be about retrying the download
            log_mock.debug.assert_called_with("Trying again in %d seconds ...", 1)

            # Other network errors that raise an exception will trigger a retry
            get.side_effect = ConnectionError("Couldn't connect")
            with pytest.raises(SystemExit):
                MalwareDetectionClient(InsightsConfig(retries=3))
            log_mock.error.assert_called_with(
                "Unable to download rules from %s: %s", expected_rules_url, "Couldn't connect"
            )
            # Because we've set retries to 3, the last attempted retry will wait 2 seconds
            log_mock.debug.assert_called_with("Trying again in %d seconds ...", 2)

            # Certificate verify failed SSL Errors will cause a different CA certificate bundle to be tried
            ca_cert = '/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem'  # default ca cert bundle file to try
            ssl_error = "SSL Error: certificate verify failed"
            get.side_effect = SSLError(ssl_error)
            with pytest.raises(SystemExit):
                MalwareDetectionClient(InsightsConfig(retries=2))
            get.assert_called_with(
                expected_rules_url, log_response_text=False, verify=ca_cert, stream=True
            )
            log_mock.debug.assert_called_with("Trying cert_verify value %s ...", ca_cert)
            log_mock.error.assert_called_with(
                "Unable to download rules from %s: %s", expected_rules_url, ssl_error
            )

            # CA certificate bundles can be specified in the config file or via env vars too
            ca_cert = '/some/cert/file.crt'
            os.environ['CA_CERT'] = (
                ca_cert  # customers can specify their own ca bundle file via config/env vars
            )
            ssl_error = "SslError CERTIFICATE_VERIFY_FAILED"
            get.side_effect = SSLError(ssl_error)
            with pytest.raises(SystemExit):
                MalwareDetectionClient(InsightsConfig(retries=2))
            get.assert_called_with(
                expected_rules_url, log_response_text=False, verify=ca_cert, stream=True
            )
            log_mock.debug.assert_called_with("Trying cert_verify value %s ...", ca_cert)
            log_mock.error.assert_called_with(
                "Unable to download rules from %s: %s", expected_rules_url, ssl_error
            )

        @patch.dict(
            os.environ, {'TEST_SCAN': 'true', 'REMOTE_RULES_LOCATION': '//console.redhat.com/rules.yar'}
        )
        @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
        @patch("os.path.isfile", return_value=True)
        def test_get_rules_location_as_file(
            self, isfile, yara, conf, cmd, disabled, session, proxies, get, findmnt, remove, tmpfile
        ):
            # Test using files for remote_rules_location, esp irregular file names
            # remote_rules_location that starts with a '/' is assumed to be a file, even if its a double '//'
            # Re-writing the rule to be test-rule.yar doesn't apply to local files
            mdc = MalwareDetectionClient(None)
            assert mdc.remote_rules_location == "//console.redhat.com/rules.yar"
            assert mdc.rules_files == ["/etc/insights-client/signatures", "/console.redhat.com/rules.yar"]
            get.assert_not_called()

            # Just to confirm the filename stays the same for regardless of test_rule value
            os.environ['TEST_SCAN'] = 'false'
            mdc = MalwareDetectionClient(None)
            assert mdc.remote_rules_location == "//console.redhat.com/rules.yar"
            assert mdc.rules_files == ["/etc/insights-client/signatures", "/console.redhat.com/rules.yar"]
            get.assert_not_called()

        @patch.dict(os.environ, {'TEST_SCAN': 'true'})
        @patch(LOGGER_TARGET)
        def test_download_rules_via_satellite(
            self, log_mock, conf, cmd, disabled, session, proxies, get, findmnt, remove, tmpfile
        ):
            # Test recognizing and handling of Satellite URLs.
            # Satellite URLs have '/redhat_access/' in their path (so Satellite knows to redirect the query to C.R.C)
            # For malware-detection requests through a Satellite we append the malware-detection path and add https://
            satellite_url = 'satellite.example.com:443/redhat_access/r/insights/platform'
            satellite_cert = '/etc/rhsm/ca/katello-server-ca.pem'
            expected_rules_url_prefix = "https://satellite.example.com:443/redhat_access/r/insights/platform/malware-detection/v1/"
            insights_config = InsightsConfig(
                base_url=satellite_url, verbose=True, cert_verify=satellite_cert
            )

            # Firstly try with test-rule
            expected_rules_url = expected_rules_url_prefix + "test-rule.yar"
            with patch(FIND_YARA_TARGET, return_value=FAKE_YARA):
                mdc = MalwareDetectionClient(insights_config)
            assert mdc.remote_rules_location == expected_rules_url
            get.assert_called_with(
                expected_rules_url, log_response_text=True, verify=satellite_cert, stream=True
            )
            log_mock.debug.assert_called_with("Using cert_verify value %s ...", satellite_cert)
            log_mock.debug.assert_any_call("Downloading rules from: %s", expected_rules_url)

            # Then with the actual rules file
            os.environ['TEST_SCAN'] = 'false'
            expected_rules_url = expected_rules_url_prefix + "signatures.yar"
            with patch(FIND_YARA_TARGET, return_value=FAKE_YARA):
                mdc = MalwareDetectionClient(insights_config)
            assert mdc.remote_rules_location == expected_rules_url
            get.assert_called_with(
                expected_rules_url, log_response_text=False, verify=satellite_cert, stream=True
            )
            log_mock.debug.assert_called_with("Using cert_verify value %s ...", satellite_cert)
            log_mock.debug.assert_any_call("Downloading rules from: %s", expected_rules_url)

            expected_rules_url += '?yara_version=4.1'
            with patch('os.path.exists', return_value=True):
                with patch(CALL_TARGET, return_value='4.1'):
                    mdc = MalwareDetectionClient(insights_config)
            assert mdc.remote_rules_location == expected_rules_url
            get.assert_called_with(
                expected_rules_url, log_response_text=False, verify=satellite_cert, stream=True
            )
            log_mock.debug.assert_called_with("Using cert_verify value %s ...", satellite_cert)
            log_mock.debug.assert_any_call("Downloading rules from: %s", expected_rules_url)

            # Test a Satellite URL with 'cloud.stage.' in its name - expect to still download from Satellite
            # Also test using cert_verify=None in the InsightsConfig and check it changes to cert_verify=True
            satellite_url = (
                'satellite.cloud.stage.example.com:443/redhat_access/r/insights/platform'
            )
            expected_rules_url = "https://satellite.cloud.stage.example.com:443/redhat_access/r/insights/platform/malware-detection/v1/"
            expected_rules_url += 'signatures.yar?yara_version=4.1'
            with patch('os.path.exists', return_value=True):
                with patch(CALL_TARGET, return_value='4.1'):
                    mdc = MalwareDetectionClient(
                        InsightsConfig(base_url=satellite_url, verbose=True, cert_verify=None)
                    )
            assert mdc.remote_rules_location == expected_rules_url
            get.assert_called_with(
                expected_rules_url, log_response_text=False, verify=True, stream=True
            )
            log_mock.debug.assert_called_with("Using cert_verify value %s ...", True)
            log_mock.debug.assert_any_call("Downloading rules from: %s", expected_rules_url)

        @patch.object(builtins, 'sum')
        @patch('os.path.exists')
        @patch(LOGGER_TARGET)
        @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
        @patch.dict(os.environ, {'RULES_LOCATION': '/nonexistent', 'USE_REMOTE_RULES': 'false'})
        def test_get_rules_old_rules_file_removal_with_existing_files(
            self, yara, log_mock, exists_mock, sum_mock,
            conf, cmd, disabled, session, proxies, get, findmnt, remove, tmpfile
        ):
            """Test _get_rules removes old rules files that exist"""
            # Mock glob to return list of old rules files
            old_files = [
                '/tmp/malware-detection_yara_rules.123',
                '/var/tmp/.tmpmdsigs456',
                '/usr/tmp/tmp_malware-detection-client_rules.789'
            ]
            sum_mock.return_value = old_files

            # Mock os.path.exists to return True for all files (they exist)
            exists_mock.return_value = True

            # Mock use_remote_rules to False as to avoid remote rules processing
            with pytest.raises(SystemExit):
                MalwareDetectionClient(InsightsConfig())

            # Verify os.path.exists was called for each old rules file
            assert exists_mock.call_count == 3
            for f in old_files:
                exists_mock.assert_any_call(f)

            # Verify os.remove was called for each existing old rules file
            assert remove.call_count == 3
            for f in old_files:
                remove.assert_any_call(f)

            # Verify debug logging for each removed file
            assert log_mock.debug.call_count == 5  # includes 2 debug logs for env vars
            for f in old_files:
                log_mock.debug.assert_any_call("Removing old rules file %s", f)

    @patch(
        NAMEDTMPFILE_TARGET
    )  # Mock NamedTemporaryFile so it doesn't try to create the temporary file
    @patch('os.remove')  # Mock os.remove so it doesn't actually try to remove any existing files
    @patch(FINDMNT_TARGET)  # Don't run the command
    @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
    @patch(BUILD_YARA_COMMAND_TARGET, return_value=(['lol'], []))
    @patch(LOAD_CONFIG_TARGET, return_value=CONFIG)
    class TestDisabledRules:
        """Testing the _get_disabled_rules method"""

        @patch.dict(os.environ)
        def test_skip_getting_disabled_rules(self, conf, cmd, yara, findmnt, remove, tmpfile):
            # Skip getting disabled rules if doing a test scan
            os.environ['TEST_SCAN'] = 'true'
            with patch(GET_RULES_TARGET, return_value=[RULES_FILE]):
                mdc = MalwareDetectionClient(None)
            assert mdc.disabled_rules == []

            # Skip getting disabled rules if the rules are coming from a file (which is really only for testing anyway)
            os.environ['TEST_SCAN'] = 'false'
            os.environ['REMOTE_RULES_LOCATION'] = '/local/rules/file.yar'
            with patch("os.path.isfile", return_value=True):
                mdc = MalwareDetectionClient(None)
            assert mdc.disabled_rules == []

        @patch.object(InsightsConnection, 'post')
        @patch.object(
            InsightsConnection, 'get', return_value=Mock(status_code=200, content=b"Rule Content")
        )
        @patch.object(InsightsConnection, 'get_proxies')
        @patch.object(InsightsConnection, '_init_session', return_value=Mock())
        @patch.dict(os.environ, {'TEST_SCAN': 'false'})
        def test_get_disabled_rules(
            self, session, proxies, get, post, conf, cmd, yara, findmnt, remove, tmpfile
        ):
            # Test with no disabled rules
            remote_rules_location = (
                "https://cert.console.redhat.com/api/malware-detection/v1/signatures.yar"
            )
            expected_graphql_url = (
                "https://cert.console.redhat.com/api/malware-detection/v1/graphql"
            )
            post.return_value = Mock(
                status_code=200, json=Mock(return_value={'data': {'rulesList': []}})
            )
            mdc = MalwareDetectionClient(InsightsConfig())
            assert mdc.remote_rules_location == remote_rules_location
            assert mdc.graphql_url == expected_graphql_url
            assert post.called_with(expected_graphql_url, ANY, ANY, verify=True, stream=True)
            assert mdc.disabled_rules == []

            # Test with one disabled rule
            post.return_value = Mock(
                status_code=200,
                json=Mock(return_value={'data': {'rulesList': [{'name': 'Rule1'}]}}),
            )
            mdc = MalwareDetectionClient(InsightsConfig())
            assert mdc.disabled_rules == ['rule1']

            # Test with multiple disabled rules and a different remote_rules_location
            remote_rules_location = "http://localhost/rules.yar"
            os.environ['REMOTE_RULES_LOCATION'] = remote_rules_location
            expected_graphql_url = "http://localhost/graphql"
            post.return_value = Mock(
                status_code=200,
                json=Mock(
                    return_value={
                        'data': {
                            'rulesList': [
                                {'name': 'Rule2'},
                                {'name': 'Rule1'},
                                {'name': 'XYZ'},
                                {'name': 'abc'},
                            ]
                        }
                    }
                ),
            )
            mdc = MalwareDetectionClient(InsightsConfig())
            assert mdc.remote_rules_location == remote_rules_location
            assert mdc.graphql_url == expected_graphql_url
            assert post.called_with(expected_graphql_url, ANY, ANY, verify=True, stream=True)
            assert mdc.disabled_rules == ['abc', 'rule1', 'rule2', 'xyz']

        @pytest.mark.skipif(
            not (TEST_GET_DISABLED_RULES_FAILURE or TEST_ALL),
            reason='test_get_disabled_rules_failure_retries is slow due to the inherent delay in the retry logic. '
            'Use TEST_GET_DISABLED_RULES_FAILURE=True to enable test',
        )
        @patch.object(InsightsConnection, 'post')
        @patch.object(
            InsightsConnection, 'get', return_value=Mock(status_code=200, content=b"Rule Content")
        )
        @patch.object(InsightsConnection, 'get_proxies')
        @patch.object(InsightsConnection, '_init_session', return_value=Mock())
        @patch.dict(os.environ, {'TEST_SCAN': 'false', 'REMOTE_RULES_LOCATION': 'localhost/rules.yar'})
        @patch(LOGGER_TARGET)
        def test_get_disabled_rules_failure(
            self, log, session, proxies, get, post, conf, cmd, yara, findmnt, remove, tmpfile
        ):
            from requests.exceptions import ConnectionError, Timeout

            expected_graphql_url = 'https://localhost/graphql'

            # 404 error - unlikely to occur unless an incorrect remote_rules_location was manually specified
            post.return_value = Mock(status_code=404, reason="Not found", text="Nup")
            mdc = MalwareDetectionClient(InsightsConfig(retries=2))
            log.debug.assert_any_call(
                "Unable to get disabled rules list from %s: %s",
                expected_graphql_url,
                "404 Not found: Nup",
            )
            log.debug.assert_any_call("Unable to get disabled rules list.  Skipping ...")
            log.debug.assert_called_with("Disabled rules: %s", [])
            assert post.called_with(expected_graphql_url, ANY, ANY, verify=True, stream=True)
            assert post.call_count == 2
            assert mdc.disabled_rules == []

            # Test other errors downloading rules from the backend - these are more likely to occur
            post.reset_mock()
            post.side_effect = [ConnectionError("Couldn't connect"), Timeout("Timeout")]
            mdc = MalwareDetectionClient(InsightsConfig(retries=3))
            log.debug.assert_any_call(
                "Unable to get disabled rules list from %s: %s",
                expected_graphql_url,
                "Couldn't connect",
            )
            log.debug.assert_any_call(
                "Unable to get disabled rules list from %s: %s", expected_graphql_url, "Timeout"
            )
            log.debug.assert_any_call("Unable to get disabled rules list.  Skipping ...")
            log.debug.assert_called_with("Disabled rules: %s", [])
            assert post.called_with(expected_graphql_url, ANY, ANY, verify=True, stream=True)
            assert post.call_count == 3
            assert mdc.disabled_rules == []

    @patch(GET_RULES_TARGET, return_value=[RULES_FILE])
    @patch(DISABLED_RULES_TARGET)
    @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
    @patch(LOAD_CONFIG_TARGET, return_value=CONFIG)
    class TestBuildYaraCmd:

        @patch('os.path.getsize')
        def test_build_yara_command_success(self, size, conf, yara, disabled, rules):
            expected_yara_cmd = "nice -n 19 {0} -s -w -m -N -a 3600 -p 1 -r -f%s {1}".format(
                FAKE_YARA, RULES_FILE
            )
            size.return_value = 1
            # Use side_effect with 3 'call' values because build_yara_command calls 'call' 3 times ...
            # 1 to get the type of the rules file; 2 to see if the rules files contains valid rules; 3 to call nproc
            # Test with text rules file - file type is 'ascii'
            with patch(CALL_TARGET, side_effect=['ascii', 'ok', '2']) as call_mock:
                mdc = MalwareDetectionClient(None)
                assert call_mock.call_count == 3
            assert ' '.join(mdc.yara_cmd) == expected_yara_cmd % ''

            # Test with 'compiled' rules file - file type is 'Yara 3.x'
            with patch(CALL_TARGET, side_effect=['Yara 3.X', 'ok', '2']) as call_mock:
                mdc = MalwareDetectionClient(None)
                assert call_mock.call_count == 3
            assert ' '.join(mdc.yara_compiled_cmds[0]) == expected_yara_cmd % ' -C'

            # Another test with compiled rules file - file type is 'data'
            with patch(CALL_TARGET, side_effect=['data', 'ok', '2']) as call_mock:
                mdc = MalwareDetectionClient(None)
                assert call_mock.call_count == 3

            assert ' '.join(mdc.yara_compiled_cmds[0]) == expected_yara_cmd % ' -C'

        @patch(LOGGER_TARGET)
        @patch('os.path.getsize')
        def test_build_yara_command_fail(self, size_mock, log_mock, conf, yara, disabled, rules):
            # Test with empty rules file, ie file size is 0
            size_mock.return_value = 0
            with patch(CALL_TARGET, side_effect=['wtf?', 'yikes', '2']) as call_mock:
                with pytest.raises(SystemExit):
                    MalwareDetectionClient(None)
                call_mock.assert_called_once()  # Only 1 call to 'call' before we exit
            log_mock.error.assert_called_with("Rules file %s is empty", RULES_FILE)

            # Test with empty rules files, ie the file type is 'empty'
            size_mock.return_value = 1
            with patch(CALL_TARGET, side_effect=['empty', 'yikes', '2']) as call_mock:
                with pytest.raises(SystemExit):
                    MalwareDetectionClient(None)
                call_mock.assert_called_once()  # Only 1 call to 'call' before we exit
            log_mock.error.assert_called_with("Rules file %s is empty", RULES_FILE)

            # Test with 'invalid' rules file - raise CalledProcessError when running command
            with patch(CALL_TARGET) as call_mock:
                call_mock.side_effect = ['yara', CalledProcessError(1, 'cmd', b'invalid\n'), '2']
                mdc = MalwareDetectionClient(None)
                assert call_mock.call_count == 3  # 3 calls to 'call' before we finish
                assert len(mdc.yara_compiled_cmds) == 1  # assert rules file is still used

            # Test with 'error' rules file - raise CalledProcessError when running command
            with patch(CALL_TARGET) as call_mock:
                call_mock.side_effect = ['yara', CalledProcessError(1, 'cmd', b'error\n'), '2']
                mdc = MalwareDetectionClient(None)
                assert call_mock.call_count == 3  # 3 calls to 'call' before we finish
                assert len(mdc.yara_compiled_cmds) == 0  # assert rules file is removed
            log_mock.error.assert_called_with(
                "Invalid rules file %s: %s", RULES_FILE, 'error\n'
            )

    @patch(CONFIG_FILE_TARGET, TEMP_CONFIG_FILE)
    @patch(BUILD_YARA_COMMAND_TARGET, return_value=(['lol'], []))
    @patch(GET_RULES_TARGET, return_value=[TEST_RULE_FILE])
    @patch(DISABLED_RULES_TARGET)
    @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
    @patch(LOGGER_TARGET)
    @patch.dict(os.environ)
    class TestProcessScanning:

        @pytest.mark.skipif(IS_CONTAINER, reason=SKIP_IF_CONTAINER_REASON)
        def test_scan_processes(
            self, log_mock, yara, disabled, rules, cmd, create_test_files_fake_yara
        ):
            # Test scanning processes to test which processes are going to be scanned
            os.environ['EXCLUDE_NETWORK_FILESYSTEM_MOUNTPOINTS'] = 'false'
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "test_scan: false" if line.startswith("test_scan:") else line
                line = "scan_processes: true" if line.startswith("scan_processes:") else line
                line = "add_metadata: false" if line.startswith("add_metadata:") else line
                print(line)
            mdc = MalwareDetectionClient(None)
            assert mdc.rules_files == [TEST_RULE_FILE]
            assert mdc.do_filesystem_scan is True
            assert mdc.do_process_scan is True
            assert mdc.scan_pids == []
            assert mdc.processes_scan_exclude_list == []

            # Patch the calls to yara so it doesn't actually try to scan any processes
            with patch(CALL_TARGET, return_value=""):
                # have to manually set active cmd:
                all_cmds = mdc.yara_compiled_cmds
                if mdc.yara_cmd:
                    all_cmds.append(mdc.yara_cmd)
                for cmd in all_cmds:
                    mdc.active_cmd = cmd
                mdc.scan_processes()

            assert mdc.processes_scan_exclude_list == [TEST_PID]
            assert len(mdc.scan_pids) > 1
            assert '1' in mdc.scan_pids
            assert TEST_PID not in mdc.scan_pids

            # Exclude some processes via the config file
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = (
                    "processes_scan_exclude: 1\n"
                    if line.startswith("processes_scan_exclude:")
                    else line
                )
                print(line)
            mdc = MalwareDetectionClient(None)
            assert mdc.processes_scan_exclude_list == ['1']
            # Patch the calls to yara so it doesn't actually try to scan any processes
            with patch(CALL_TARGET, return_value=""):
                # have to manually set active cmd:
                all_cmds = mdc.yara_compiled_cmds
                if mdc.yara_cmd:
                    all_cmds.append(mdc.yara_cmd)
                for cmd in all_cmds:
                    mdc.active_cmd = cmd
                mdc.scan_processes()
            assert mdc.processes_scan_exclude_list == ['1', TEST_PID]
            assert len(mdc.scan_pids) > 1
            assert all([x not in mdc.scan_pids for x in ['1', TEST_PID]])

            # Exclude some processes via env vars
            os.environ['PROCESSES_SCAN_EXCLUDE'] = 'systemd,3..10'
            mdc = MalwareDetectionClient(None)
            assert all([pid in mdc.processes_scan_exclude_list for pid in ['1', '3']])
            assert '2' not in mdc.processes_scan_exclude_list
            # Patch the calls to yara so it doesn't actually try to scan any processes
            with patch(CALL_TARGET, return_value=""):
                # have to manually set active cmd:
                all_cmds = mdc.yara_compiled_cmds
                if mdc.yara_cmd:
                    all_cmds.append(mdc.yara_cmd)
                for cmd in all_cmds:
                    mdc.active_cmd = cmd
                mdc.scan_processes()
            assert all([pid in mdc.processes_scan_exclude_list for pid in ['1', '3', TEST_PID]])
            assert len(mdc.scan_pids) > 1
            assert all([x not in mdc.scan_pids for x in ['1', '3', TEST_PID]])
            assert '2' in mdc.scan_pids

        @pytest.mark.skipif(
            not (TEST_PROCESSES_SCAN_SINCE or TEST_ALL),
            reason='test_processes_scan_since is slowish and could potentially fail. '
            'Use TEST_PROCESSES_SCAN_SINCE=True to enable test',
        )
        def test_processes_scan_since(
            self, log_mock, yara, disabled, rules, cmd, create_test_files_fake_yara
        ):
            # Firstly, start the TEST_RULE_SCRIPT process then scan_only that process
            # Expect that we'll find it
            os.system(TEST_RULE_SCRIPT + " &")  # Run the script in the background
            ps_call_output = call([['ps', '-eo', 'pid=', '-o', 'lstart=']])
            os.environ['EXCLUDE_NETWORK_FILESYSTEM_MOUNTPOINTS'] = 'false'
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "test_scan: false" if line.startswith("test_scan:") else line
                line = "scan_filesystem: false" if line.startswith("scan_filesystem:") else line
                line = "scan_processes: true" if line.startswith("scan_processes:") else line
                line = (
                    "processes_scan_only: test-rule"
                    if line.startswith("processes_scan_only:")
                    else line
                )
                line = "add_metadata: false" if line.startswith("add_metadata:") else line
                print(line)
            mdc = MalwareDetectionClient(None)
            assert mdc.rules_files == [TEST_RULE_FILE]
            assert mdc.do_filesystem_scan is False
            assert mdc.do_process_scan is True
            # Match TEST_RULE_SCRIPT process from processes_scan_only
            assert len(mdc.scan_pids) == 1

            # Now find processes started since the last scan, specifically the TEST_RULE_SCRIPT process
            # However expect to not find it because it wasn't started since the last_scan date
            last_scan = time.time()
            last_scan_fmt = datetime.fromtimestamp(last_scan).strftime('%Y-%m-%d %H:%M:%S')
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = (
                    "processes_scan_since: last"
                    if line.startswith("processes_scan_since:")
                    else line
                )
                print(line)
            with patch(
                "insights.specs.datasources.malware_detection.get_scan_since_timestamp",
                return_value=last_scan,
            ):
                mdc = MalwareDetectionClient(None)
            assert mdc.processes_scan_since_dict['timestamp'] == last_scan
            assert mdc.processes_scan_since_dict['datetime'] == last_scan_fmt
            # Still match the TEST_RULE_SCRIPT process from processes_scan_only
            assert len(mdc.scan_pids) == 1
            # But when we run scan_processes() it won't be found because it wasn't started since the last scan
            # Patch the calls to yara so it doesn't actually try to scan any processes
            with patch(CALL_TARGET, return_value=ps_call_output):
                # have to manually set active cmd:
                all_cmds = mdc.yara_compiled_cmds
                if mdc.yara_cmd:
                    all_cmds.append(mdc.yara_cmd)
                for cmd in all_cmds:
                    mdc.active_cmd = cmd
                mdc.scan_processes()
            assert len(mdc.scan_pids) == 0
            log_mock.error.assert_called_with(
                "No processes to scan because none were started since %s", last_scan_fmt
            )

            # Now find processes started since one day ago, specifically the TEST_RULE_SCRIPT process
            # Expect to find it this time it was started since a day ago
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = (
                    "processes_scan_since: 1" if line.startswith("processes_scan_since:") else line
                )
                print(line)
            mdc = MalwareDetectionClient(None)
            # Still match the TEST_RULE_SCRIPT process from processes_scan_only
            assert len(mdc.scan_pids) == 1
            with patch(CALL_TARGET) as call_mock:
                # Patch calls to 'call' within the scan_processes function
                # The first call is to ps to get the process list, which is what we want
                # The second call is to yara, which we want to ignore since yara may not be installed
                call_mock.side_effect = [ps_call_output, ""]
                # have to manually set active cmd:
                all_cmds = mdc.yara_compiled_cmds
                if mdc.yara_cmd:
                    all_cmds.append(mdc.yara_cmd)
                for cmd in all_cmds:
                    mdc.active_cmd = cmd
                mdc.scan_processes()
            # Expect to find the TEST_RULE_SCRIPT process because it was started since a day ago
            assert len(mdc.scan_pids) == 1

            # Start another process and this time there will be a TEST_RULE_SCRIPT process started since last_scan date
            time.sleep(
                2
            )  # Wait a second to ensure the last_scan time is greater than the process start time
            os.system(TEST_RULE_SCRIPT + " &")  # Run the script in the background
            ps_call_output = call([['ps', '-eo', 'pid=', '-o', 'lstart=']])
            with patch(
                "insights.specs.datasources.malware_detection.get_scan_since_timestamp",
                return_value=last_scan,
            ):
                mdc = MalwareDetectionClient(None)
            # Match 2 TEST_RULE_SCRIPT processes from processes_scan_only
            assert len(mdc.scan_pids) == 2
            # But when we run scan_processes() only the latest one will be found
            with patch(CALL_TARGET) as call_mock:
                call_mock.side_effect = [ps_call_output, ""]
                # have to manually set active cmd:
                all_cmds = mdc.yara_compiled_cmds
                if mdc.yara_cmd:
                    all_cmds.append(mdc.yara_cmd)
                for cmd in all_cmds:
                    mdc.active_cmd = cmd
                mdc.scan_processes()
            # Only find the latest TEST_RULE_SCRIPT process
            assert len(mdc.scan_pids) == 1

    @patch(CONFIG_FILE_TARGET, TEMP_CONFIG_FILE)
    @patch('os.remove')  # Mock os.remove so it doesn't actually try to remove any existing files
    @patch(BUILD_YARA_COMMAND_TARGET, return_value=(['lol'], []))
    @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
    @patch(LOGGER_TARGET)
    class TestFilesystemScanning:

        def test_scan_rules_file_with_extra_slashes(
            self, log_mock, yara, cmd, remove, create_test_files_fake_yara
        ):
            # Test scanning RULES_FILE with an extra slash only in the remote_rules_location one
            # Even with the extra slashes in the remote_rules_location there will be rules matched
            # because */rules_compiled.yar and *//rules_compiled.yar are the same file
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "test_scan: false" if line.startswith("test_scan:") else line
                line = (
                    line + "remote_rules_location: %s\n" % TEST_RULE_FILE.replace('/', '//')
                    if line.startswith('---')
                    else line
                )
                line = (
                    "filesystem_scan_only: %s" % TEST_RULE_FILE
                    if line.startswith("filesystem_scan_only:")
                    else line
                )
                line = "add_metadata: false" if line.startswith("add_metadata:") else line
                line = (
                    "exclude_network_filesystem_mountpoints: false"
                    if line.startswith("exclude_network_filesystem_mountpoints:")
                    else line
                )
                print(line)
            mdc = MalwareDetectionClient(None)
            assert mdc.remote_rules_location == TEST_RULE_FILE.replace('/', '//')
            assert mdc.rules_files == [TEST_RULE_FILE]
            assert mdc.scan_fsobjects == [TEST_RULE_FILE]
            with patch(CALL_TARGET) as call_mock:
                # Mock the scan match data from yara
                call_mock.return_value = (
                    'TEST_RedHatInsightsMalwareDetection [author="Red Hat Insights",description="Verifies the Red Hat Insights Malware Detection Client app is present on the system",usage="Verification",date_created="11 June 2021",date_updated="18 October 2021"] %s\n0x4a:$re1: Malware Detection Client'
                    % TEST_RULE_FILE
                )
                # have to manually set active cmd:
                all_cmds = mdc.yara_compiled_cmds
                if mdc.yara_cmd:
                    all_cmds.append(mdc.yara_cmd)
                for cmd in all_cmds:
                    mdc.active_cmd = cmd
                mdc.scan_filesystem()
            rule_match = mdc.host_scan['TEST_RedHatInsightsMalwareDetection']
            assert rule_match['matches'][0]['source'] == TEST_RULE_FILE
            assert rule_match['matches'][0]['string_data'] == "Malware Detection Client"
            assert rule_match['matches'][0]['string_identifier'] == '$re1'
            assert rule_match['matches'][0]['string_offset'] == 74
            assert rule_match['metadata']['author'] == 'Red Hat Insights'
            assert rule_match['metadata']['description'] == 'Verifies the Red Hat Insights Malware Detection Client app is present on the system'
            assert rule_match['metadata']['usage'] == 'Verification'

            log_mock.info.assert_any_call(
                "Matched rule %s in %s %s",
                "TEST_RedHatInsightsMalwareDetection",
                "file",
                TEST_RULE_FILE,
            )

        def test_scan_root_with_extra_slashes(
            self, log_mock, yara, cmd, remove, create_test_files_fake_yara
        ):
            # Testing we handle the situation where items in filesystem_scan_only & scan_exclude contain multiple slashes
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "test_scan: false" if line.startswith("test_scan:") else line
                line = (
                    line + "remote_rules_location: %s\n" % TEST_RULE_FILE
                    if line.startswith('---')
                    else line
                )
                line = line + "- //\n" if line.startswith("filesystem_scan_only:") else line
                line = line + "- //\n" if line.startswith("filesystem_scan_exclude:") else line
                line = (
                    "exclude_network_filesystem_mountpoints: false"
                    if line.startswith("exclude_network_filesystem_mountpoints:")
                    else line
                )
                print(line)
            mdc = MalwareDetectionClient(None)
            assert mdc.scan_fsobjects == ['/']
            assert '/' in mdc.filesystem_scan_exclude_list
            # Chaos monkey - modify scan_fsobjects and filesystem_scan_exclude_list AFTER they have been verified
            # Assert that they still work and cancel each other out
            mdc.scan_fsobjects = ['//']
            mdc.filesystem_scan_exclude_list = ['//']
            mdc.scan_filesystem()
            assert mdc.do_filesystem_scan is False
            log_mock.error.assert_called_with(
                "No filesystem items to scan because the specified exclude items cancel them out"
            )

        @patch('insights.specs.datasources.malware_detection.NamedTemporaryFile')
        @patch(CALL_TARGET, return_value="")
        def test_filesystem_scan_since_tmp_files(
            self,
            call_mock,
            tmp_file_mock,
            log_mock,
            yara,
            cmd,
            remove,
            extract_tmp_files,
            create_test_files_fake_yara,
        ):
            # Set filesystem_scan_only, filesystem_scan_exclude options to some of the tmp files and then 'scan' them
            # Then touch files to test the filesystem_scan_since option and make sure that only the touched files will be scanned
            yara_file_list = os.path.join(TEMP_TEST_DIR, 'yara_file_list')
            scan_me_file = os.path.join(TEMP_TEST_DIR, 'scan_me/scan_me_file')
            scan_me_too_file = os.path.join(TEMP_TEST_DIR, 'scan_me_too/scan_me_too_file')
            filesystem_scan_only = tuple(
                map(lambda x: os.path.join(TEMP_TEST_DIR, x), ['scan_me', 'scan_me_too'])
            )
            filesystem_scan_exclude = tuple(
                map(
                    lambda x: os.path.join(TEMP_TEST_DIR, x),
                    ['scan_me_not', 'scan_me/dont_scan_me', 'scan_me_too/dont_scan_me_too'],
                )
            )

            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = "test_scan: false" if line.startswith("test_scan:") else line
                line = (
                    line + "remote_rules_location: %s\n" % TEST_RULE_FILE
                    if line.startswith('---')
                    else line
                )
                line = (
                    line + '- %s\n- %s' % filesystem_scan_only
                    if line.startswith("filesystem_scan_only:")
                    else line
                )
                line = (
                    line + "- %s\n- %s\n- %s" % filesystem_scan_exclude
                    if line.startswith("filesystem_scan_exclude:")
                    else line
                )
                line = (
                    "exclude_network_filesystem_mountpoints: false"
                    if line.startswith("exclude_network_filesystem_mountpoints:")
                    else line
                )
                print(line)
            mdc = MalwareDetectionClient(None)
            scan_dict = process_include_exclude_items(
                include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
            )
            # Ensure the correct scan include and exclude values are set
            assert list(scan_dict.keys()) == ['/tmp']
            assert sorted(list(scan_dict['/tmp']['exclude']['items'])) == sorted(
                filesystem_scan_exclude
            )

            # Run scan_filesystem, but mock out NamedTemporaryFile so we can use our own file and inspect its contents after
            # Also mock out the call to yara but we don't return anything since we aren't testing it
            with open(yara_file_list, 'w') as f:
                tmp_file_mock.return_value = f
                # have to manually set active cmd:
                all_cmds = mdc.yara_compiled_cmds
                if mdc.yara_cmd:
                    all_cmds.append(mdc.yara_cmd)
                for cmd in all_cmds:
                    mdc.active_cmd = cmd
                mdc.scan_filesystem()
            with open(yara_file_list, 'r') as f:
                contents = f.read().splitlines()
            # Ensure that a number of files are in the list of files passed to yara to scan
            assert len(contents) > 2
            assert all([x in contents for x in [scan_me_file, scan_me_too_file]])

            # With the same filesystem scan_only and scan_exclude values, add filesystem_scan_since: last into the mix and set
            # the last scan time to now.  There should be no matches because no files have been modified since now
            last_scan = time.time()
            last_scan_fmt = datetime.fromtimestamp(last_scan).strftime('%Y-%m-%d %H:%M:%S')
            for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                line = (
                    "filesystem_scan_since: last"
                    if line.startswith("filesystem_scan_since:")
                    else line
                )
                print(line)
            with patch(
                "insights.specs.datasources.malware_detection.get_scan_since_timestamp",
                return_value=last_scan,
            ):
                mdc = MalwareDetectionClient(None)
            assert mdc.filesystem_scan_since_dict['timestamp'] == last_scan
            assert mdc.filesystem_scan_since_dict['datetime'] == last_scan_fmt
            log_mock.info.assert_called_with("Scan for files created/modified since %s%s", ANY, ANY)

            # Run scan_filesystem, but mock out NamedTemporaryFile so we can use our own file and inspect its contents after
            # Also mock out the call to yara, but we don't return anything since we aren't testing it
            with open(yara_file_list, 'w') as f:
                tmp_file_mock.return_value = f
                # have to manually set active cmd:
                all_cmds = mdc.yara_compiled_cmds
                if mdc.yara_cmd:
                    all_cmds.append(mdc.yara_cmd)
                for cmd in all_cmds:
                    mdc.active_cmd = cmd
                mdc.scan_filesystem()
            with open(yara_file_list, 'r') as f:
                contents = f.read().splitlines()
            # Ensure no files were passed to yara to scan because none were modified since 'last_scan'
            assert not contents

            # Try again, keeping the same last_scan time, but this time touch 2 files
            # Confirm that only these 2 files appear in the list of files to be passed to yara
            os.system('touch %s %s' % (scan_me_file, scan_me_too_file))
            with patch(
                "insights.specs.datasources.malware_detection.get_scan_since_timestamp",
                return_value=last_scan,
            ):
                mdc = MalwareDetectionClient(None)

            with open(yara_file_list, 'w') as f:
                tmp_file_mock.return_value = f
                # have to manually set active cmd:
                all_cmds = mdc.yara_compiled_cmds
                if mdc.yara_cmd:
                    all_cmds.append(mdc.yara_cmd)
                for cmd in all_cmds:
                    mdc.active_cmd = cmd
                mdc.scan_filesystem()
            with open(yara_file_list, 'r') as f:
                contents = f.read().splitlines()
            # Ensure the scan_me_file was passed to yara to scan because it was 'modified' since 'last_scan'
            assert len(contents) == 2
            assert contents == [scan_me_file, scan_me_too_file]

            # Touch some files that are excluded from scanning so even though they have been modified, they won't be
            # in the list of files to scan that is passed to yara
            os.system(
                'touch %s %s %s'
                % (
                    os.path.join(TEMP_TEST_DIR, 'scan_me/dont_scan_me/matching_entity'),
                    os.path.join(TEMP_TEST_DIR, 'scan_me_not/matching_entity'),
                    os.path.join(
                        TEMP_TEST_DIR, "scan_me_too/dont_scan_me_too/'another matching_entity'"
                    ),
                )
            )
            with patch(
                "insights.specs.datasources.malware_detection.get_scan_since_timestamp",
                return_value=last_scan,
            ):
                mdc = MalwareDetectionClient(None)

            with open(yara_file_list, 'w') as f:
                tmp_file_mock.return_value = f
                # have to manually set active cmd:
                all_cmds = mdc.yara_compiled_cmds
                if mdc.yara_cmd:
                    all_cmds.append(mdc.yara_cmd)
                for cmd in all_cmds:
                    mdc.active_cmd = cmd
                mdc.scan_filesystem()
            with open(yara_file_list, 'r') as f:
                contents = f.read().splitlines()
            # Ensure both scan_me_file and scan_me_too_file were passed to yara because both were modified since last_scan
            assert len(contents) == 2
            assert contents == [scan_me_file, scan_me_too_file]

        @patch(LOAD_CONFIG_TARGET, return_value=CONFIG)
        @patch.dict(os.environ)
        def test_rule_n_glob_files_excluded(
            self, conf, log_mock, yara, cmd, remove, extract_tmp_files, create_test_files_fake_yara
        ):
            # Fake a scan but make sure we are excluding the rules file and globbed files (they are supposed to be
            # insights log files, but that's hard to mock).
            # Also test we are not excluding ones we actually want to scan
            glob_files = [
                os.path.join(TEMP_TEST_DIR, 'scan_me', f) for f in ['new_file', 'old_file']
            ]
            os.environ['TEST_SCAN'] = 'false'
            os.environ['EXCLUDE_NETWORK_FILESYSTEM_MOUNTPOINTS'] = 'false'
            os.environ['REMOTE_RULES_LOCATION'] = TEST_RULE_FILE
            os.environ['FILESYSTEM_SCAN_ONLY'] = "%s,%s" % (
                TEMP_TEST_DIR,
                glob_files[1],
            )  # we actually want to scan glob_files[1]
            mdc = MalwareDetectionClient(None)
            assert mdc.rules_files == [TEST_RULE_FILE]
            assert mdc.scan_fsobjects == [TEMP_TEST_DIR, glob_files[1]]

            # Patch the call to glob so it returns a specific list of files
            # Patch the calls for running yara and have it return no matches
            with patch(
                "insights.specs.datasources.malware_detection.glob", return_value=glob_files
            ):
                with patch(CALL_TARGET, return_value=""):
                    # have to manually set active cmd:
                    all_cmds = mdc.yara_compiled_cmds
                    if mdc.yara_cmd:
                        all_cmds.append(mdc.yara_cmd)
                    for cmd in all_cmds:
                        mdc.active_cmd = cmd
                    mdc.scan_filesystem()
            assert mdc.rules_files[0] in mdc.filesystem_scan_exclude_list
            assert glob_files[0] in mdc.filesystem_scan_exclude_list
            # Make sure glob_files[1] isn't excluded because we actually want to scan that file
            assert glob_files[1] not in mdc.filesystem_scan_exclude_list

            # This time patch glob so it returns an empty list, ie simulating no extra files to exclude
            mdc = MalwareDetectionClient(None)
            with patch("insights.specs.datasources.malware_detection.glob", return_value=[]):
                with patch(CALL_TARGET, return_value=""):
                    # have to manually set active cmd:
                    all_cmds = mdc.yara_compiled_cmds
                    if mdc.yara_cmd:
                        all_cmds.append(mdc.yara_cmd)
                    for cmd in all_cmds:
                        mdc.active_cmd = cmd
                    mdc.scan_filesystem()
            assert mdc.rules_files[0] in mdc.filesystem_scan_exclude_list
            # None of the glob files should be excluded this time
            assert all([f not in mdc.filesystem_scan_exclude_list for f in glob_files])

    @patch(FINDMNT_TARGET)
    @patch(BUILD_YARA_COMMAND_TARGET, return_value=(['lol'], []))
    @patch(GET_RULES_TARGET, return_value=RULES_FILE)
    @patch(DISABLED_RULES_TARGET)
    @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
    @patch(LOAD_CONFIG_TARGET, return_value=CONFIG)
    @patch.dict(os.environ, {'TEST_SCAN': 'false'})
    class TestFilesystemIncludeExcludeProcessing:

        def test_process_include_exclude_items_simple(
            self, conf, yara, disabled, rules, cmd, findmnt
        ):
            # Test the process_include_exclude_items function with simple modified include and exclude items
            # Simple in that the include and exclude files are modified in such a way that
            # directory listings aren't required get the list of included files
            # Add a single toplevel directory to the include file - expect only a single directory to scan
            mdc = MalwareDetectionClient(None)
            mdc.scan_fsobjects = ['/etc']
            mdc.filesystem_scan_exclude_list = []
            scan_dict = process_include_exclude_items(
                include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
            )
            assert list(scan_dict.keys()) == ['/etc']
            assert 'include' not in scan_dict['/etc']
            assert 'exclude' not in scan_dict['/etc']

            # Add some extra subdirectories to scan
            mdc.scan_fsobjects.extend(['/var/lib', '/var/log'])
            scan_dict = process_include_exclude_items(
                include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
            )
            assert sorted(scan_dict.keys()) == ['/etc', '/var']
            assert sorted(list(scan_dict['/var']['include'])) == ['/var/lib', '/var/log']
            assert 'exclude' not in scan_dict['/var']

            # Add some extra directories to exclude that won't impact the already included directories
            mdc.filesystem_scan_exclude_list.extend(['/tmp', '/var/run'])
            scan_dict = process_include_exclude_items(
                include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
            )
            assert sorted(scan_dict.keys()) == ['/etc', '/var']
            assert sorted(scan_dict['/var']['include']) == ['/var/lib', '/var/log']
            assert scan_dict['/var']['exclude']['items'] == ['/var/run']

            # Exclude /var which will remove it from the list of directories to scan
            mdc.filesystem_scan_exclude_list.append('/var')
            scan_dict = process_include_exclude_items(
                include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
            )
            assert list(scan_dict.keys()) == ['/etc']

            # Exclude /etc which means there will be no directories to scan
            mdc.filesystem_scan_exclude_list.append('/etc')
            scan_dict = process_include_exclude_items(
                include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
            )
            assert scan_dict == {}

        def test_process_include_exclude_items_complex(
            self, conf, yara, disabled, rules, cmd, findmnt
        ):
            # Test the process function with modified include and exclude files that will require more complex
            # processing to generate the list of items to be scanned
            # Because we are including items in /var/lib, we only need to list the contents of the /var/lib directory
            # We don't need to list the contents of the /var directory
            mdc = MalwareDetectionClient(None)
            mdc.scan_fsobjects = ['/var/lib', '/var/log']
            mdc.filesystem_scan_exclude_list = [
                '/var/lib/systemd',
                '/var/lib/misc/',
                '/var/log/wtmp',
            ]

            scan_dict = process_include_exclude_items(
                include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
            )
            assert list(scan_dict.keys()) == ['/var']
            assert sorted(scan_dict['/var']['exclude']['items']) == [
                '/var/lib/misc',
                '/var/lib/systemd',
                '/var/log/wtmp',
            ]
            # The exclude items shouldn't be in the include items
            # Nor should other items that aren't in the explicitly included items
            assert all(
                [
                    x not in scan_dict['/var']['include']
                    for x in [
                        '/var/lib/misc',
                        '/var/lib/systemd',
                        '/var/log/wtmp',
                        '/var/cache',
                        '/var/lib',
                        '/var/log',
                        '/var/tmp',
                        '/tmp',
                    ]
                ]
            )
            # In 'include' will be items that are in the same directory as the excluded items, eg /var/log/lastlog
            # but not the excluded items, eg /var/log/wtmp
            # Some of these directories may not exist on the test system, but if they do they will be included
            maybe_dirs = list(
                filter(
                    lambda path: os.path.exists(path),
                    ['/var/lib/dbus', '/var/lib/pam', '/var/lib/rpm', '/var/log/lastlog'],
                )
            )
            assert all([x in scan_dict['/var']['include'] for x in maybe_dirs])

            # Change the include directory to /var
            # Now immediate child directories of /var will be in the include list, eg /var/cache and /var/tmp
            # Because now we have to list the contents of the /var and /var/lib directories
            mdc.scan_fsobjects.append('/var')
            scan_dict = process_include_exclude_items(
                include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
            )
            assert list(scan_dict.keys()) == ['/var']
            assert sorted(scan_dict['/var']['exclude']['items']) == [
                '/var/lib/misc',
                '/var/lib/systemd',
                '/var/log/wtmp',
            ]
            assert all(
                [
                    x not in scan_dict['/var']['include']
                    for x in [
                        '/var/lib/misc',
                        '/var/lib/systemd',
                        '/var/log/wtmp',
                        '/var/lib',
                        '/var/log',
                        '/tmp',
                    ]
                ]
            )
            # Some of these directories may not exist on the test system, but if they do they will be included
            maybe_dirs = list(
                filter(
                    lambda path: os.path.exists(path),
                    [
                        '/var/cache',
                        '/var/tmp',
                        '/var/lib/dbus',
                        '/var/lib/pam',
                        '/var/lib/rpm',
                        '/var/log/lastlog',
                    ],
                )
            )
            assert all([x in scan_dict['/var']['include'] for x in maybe_dirs])

    @patch(FINDMNT_TARGET)
    @patch(BUILD_YARA_COMMAND_TARGET, return_value=(['lol'], []))
    @patch(GET_RULES_TARGET, return_value=RULES_FILE)
    @patch(DISABLED_RULES_TARGET, return_value=[])
    @patch(FIND_YARA_TARGET, return_value=FAKE_YARA)
    @patch(LOAD_CONFIG_TARGET, return_value=CONFIG)
    @patch.dict(os.environ, {'TEST_SCAN': 'false'})
    class TestParseScanOutput:

        def test_contrived_scan_output(self, conf, yara, disabled, rules, cmd, findmnt):
            # Parse the CONTRIVED_SCAN_OUTPUT to find actual rule matches amongst malformed output lines
            mdc = MalwareDetectionClient(None)
            assert mdc.disabled_rules == []
            mdc.add_metadata = False
            mdc.parse_scan_output(CONTRIVED_SCAN_OUTPUT)

            # Expect 5 rule matches and 8 source/file matches across the 5 rules
            # 1 source/file match for rule 'this', 3 source/file matches for rule 'Rule',
            # 2 for 'another_matching_rule', 1 for 'Iyamtho' and 1 for 'n_m3_t00'
            assert len(mdc.host_scan) == 5  # 5 rules matched
            assert mdc.matches == 8  # 8 sources/files matched across the 5 rules

            # 1 matching source/file and 1 matching string for rule 'this'
            rule_match = mdc.host_scan['this']['matches']
            rule_sources = set(map(lambda x: x['source'], rule_match))
            assert len(rule_sources) == 1
            assert len(rule_match) == 1
            assert 'e-r-r-o-r s-c-a-n-n-i-n-g' in rule_match[0]['source']
            assert rule_match[0]['string_data'] == "matches 'this' rule"
            assert rule_match[0]['string_identifier'] == '$match'
            assert rule_match[0]['string_offset'] == 291

            # 3 source/file matches and 14 matching strings across those sources for rule 'Rule'
            rule_match = mdc.host_scan['Rule']['matches']
            rule_sources = set(map(lambda x: x['source'], rule_match))
            assert len(rule_sources) == 3
            assert len(rule_match) == 14
            assert rule_match[0]['source'] == MATCHING_ENTITY_FILE
            assert rule_match[0]['string_data'] == 'string match in the file "matching_entity"'
            assert rule_match[0]['string_identifier'] == '$match0'
            assert rule_match[0]['string_offset'] == 21
            assert rule_match[1]['source'] == MATCHING_ENTITY_FILE
            assert rule_match[1]['string_data'] == "another string match in matching_entity"
            assert rule_match[1]['string_identifier'] == '$match1'
            assert rule_match[1]['string_offset'] == 83
            assert rule_match[2]['source'] == MATCHING_ENTITY_FILE
            assert (
                rule_match[2]['string_data']
                == 'string with different types of quotes \'here\' and "here"'
            )
            assert rule_match[2]['string_identifier'] == '$match2'
            assert rule_match[2]['string_offset'] == 230

            # Rule matches for ANOTHER_MATCHING_ENTITY_FILE (which has a space in the filename)
            assert rule_match[3]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert (
                rule_match[3]['string_data']
                == "string match containing error scanning but it's ok because its not in a rule line"
            )
            assert rule_match[3]['string_identifier'] == '$match3'
            assert rule_match[3]['string_offset'] == 2
            assert rule_match[4]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[4]['string_data'] == "contains ="
            assert rule_match[4]['string_identifier'] == '$grep1'
            assert rule_match[4]['string_offset'] == 97
            assert rule_match[6]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[6]['string_data'] == "contains .+"
            assert rule_match[6]['string_identifier'] == '$grep2'
            assert rule_match[6]['string_offset'] == 153
            assert rule_match[8]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[8]['string_data'] == 'contains "'
            assert rule_match[8]['string_identifier'] == '$grep3'
            assert rule_match[8]['string_offset'] == 213
            assert rule_match[9]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[9]['string_data'] == "contains '"
            assert rule_match[9]['string_identifier'] == '$grep4'
            assert rule_match[9]['string_offset'] == 241
            assert rule_match[10]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[10]['string_data'] == 'contains ()[]'
            assert rule_match[10]['string_identifier'] == '$grep5'
            assert rule_match[10]['string_offset'] == 269
            assert rule_match[11]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[11]['string_data'] == 'contains {'
            assert rule_match[11]['string_identifier'] == '$grep6'
            assert rule_match[11]['string_offset'] == 299
            assert rule_match[12]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[12]['string_data'] == 'contains ^$'
            assert rule_match[12]['string_identifier'] == '$grep7'
            assert rule_match[12]['string_offset'] == 327

            assert rule_match[13]['source'].startswith('matching_entity_3')
            assert rule_match[13]['string_data'] == ''
            assert rule_match[13]['string_identifier'] == ''
            assert rule_match[13]['string_offset'] == -1

            # 2 source matches and 4 matching strings for 'another_matching_rule'
            rule_match = mdc.host_scan['another_matching_rule']['matches']
            rule_sources = set(map(lambda x: x['source'], rule_match))
            assert len(rule_sources) == 2
            assert len(rule_match) == 4
            assert rule_match[2]['source'].endswith(
                'snap/signal-desktop/350/opt/Signal/resources/app.asar'
            )
            assert rule_match[2]['string_data'] == '#!/bin/sh'
            assert rule_match[2]['string_identifier'] == '$s0'
            assert rule_match[2]['string_offset'] == 60783814
            assert rule_match[3]['source'] == '1234567'
            assert rule_match[3]['string_data'] == '#!/bin/sh'
            assert rule_match[3]['string_identifier'] == '$s0'
            assert rule_match[3]['string_offset'] == 0

            # 1 matching source and 1 matching string for 'Iyamtho'
            rule_match = mdc.host_scan['Iyamtho']['matches']
            assert len(rule_match) == 1
            assert rule_match[0]['source'] == "yep"
            assert rule_match[0]['string_data'] == ''
            assert rule_match[0]['string_identifier'] == ''
            assert rule_match[0]['string_offset'] == -1

            # 1 matching source and 1 matching string for 'n_m3_t00'
            rule_match = mdc.host_scan['n_m3_t00']['matches']
            assert len(rule_match) == 1
            assert rule_match[0]['source'] == "damn   straight"
            assert rule_match[0]['string_data'] == ''
            assert rule_match[0]['string_identifier'] == ''
            assert rule_match[0]['string_offset'] == -1

        def test_contrived_scan_output_metadata(
            self, conf, yara, disabled, rules, cmd, findmnt, create_test_files_fake_yara
        ):
            # Again, parse the CONTRIVED_SCAN_OUTPUT to find actual rule matches amongst malformed output lines,
            # but this time check the expected metadata values too

            # Again, need to populate rules_file_location with any rule, but its not relevant for the tests
            mdc = MalwareDetectionClient(None)
            mdc.parse_scan_output(CONTRIVED_SCAN_OUTPUT)

            # Matches and metadata for MATCHING_ENTITY_FILE
            rule_match = mdc.host_scan['Rule']['matches']
            assert rule_match[0]['source'] == MATCHING_ENTITY_FILE
            assert rule_match[0]['string_offset'] == 21
            metadata = rule_match[0]['metadata']
            assert metadata['source_type'] == 'file'
            assert metadata['file_type'] == 'ASCII text'
            assert metadata['mime_type'] == 'text/plain; charset=us-ascii'
            assert metadata['md5sum'] == '9dd5c5e00d28520dc9da3c509c0db2a0'
            assert metadata['line_number'] == 1
            assert metadata['line'] == urlencode(
                'This line contains a string match in the file "matching_entity"'
            )

            # Testing displaying long lines
            assert rule_match[1]['source'] == MATCHING_ENTITY_FILE
            assert rule_match[1]['string_offset'] == 83
            metadata = rule_match[1]['metadata']
            assert metadata['source_type'] == 'file'
            assert metadata['file_type'] == 'ASCII text'
            assert metadata['mime_type'] == 'text/plain; charset=us-ascii'
            assert metadata['md5sum'] == '9dd5c5e00d28520dc9da3c509c0db2a0'
            assert metadata['line_number'] == 2
            assert metadata['line'] == urlencode(
                'This line contains another string match in matching_entity and it is very long for testing the ellipses that are added o...'
            )

            # Testing matching/displaying a mixture of quote types in the string_data
            assert rule_match[2]['source'] == MATCHING_ENTITY_FILE
            assert rule_match[2]['string_offset'] == 230
            metadata = rule_match[2]['metadata']
            assert metadata['source_type'] == 'file'
            assert metadata['file_type'] == 'ASCII text'
            assert metadata['mime_type'] == 'text/plain; charset=us-ascii'
            assert metadata['md5sum'] == '9dd5c5e00d28520dc9da3c509c0db2a0'
            assert metadata['line_number'] == 4
            assert metadata['line'] == urlencode(
                """And this line contains a string with different types of quotes 'here' and "here" and its long too but not long enough"""
            )

            # Rule match metadata for ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[3]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[3]['string_offset'] == 2
            metadata = rule_match[3]['metadata']
            assert metadata['source_type'] == 'file'
            assert metadata['file_type'] == 'ASCII text'
            assert metadata['mime_type'] == 'text/plain; charset=us-ascii'
            assert metadata['md5sum'] == '64764d295e92ffeec36d3fcd646a3af4'
            assert metadata['line_number'] == 3
            assert metadata['line'] == urlencode(
                "string match containing error scanning but it's ok because its not in a rule line"
            )

            assert rule_match[4]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[4]['string_offset'] == 97
            metadata = rule_match[4]['metadata']
            assert metadata['md5sum'] == '64764d295e92ffeec36d3fcd646a3af4'
            assert metadata['line_number'] == 7
            assert metadata['line'] == urlencode("This line contains = char")

            assert rule_match[5]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[5]['string_offset'] == 123
            metadata = rule_match[5]['metadata']
            assert metadata['line_number'] == 8
            assert metadata['line'] == urlencode("This line contains = char too")

            assert rule_match[6]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[6]['string_offset'] == 153
            metadata = rule_match[6]['metadata']
            assert metadata['line_number'] == 9
            assert metadata['line'] == urlencode("This line contains .+ chars")

            assert rule_match[8]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[8]['string_offset'] == 213
            metadata = rule_match[8]['metadata']
            assert metadata['line_number'] == 11
            assert metadata['line'] == urlencode('This line contains "" chars')

            assert rule_match[9]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[9]['string_offset'] == 241
            metadata = rule_match[9]['metadata']
            assert metadata['line_number'] == 12
            assert metadata['line'] == urlencode("This line contains '' chars")

            assert rule_match[10]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[10]['string_offset'] == 269
            metadata = rule_match[10]['metadata']
            assert metadata['line_number'] == 13
            assert metadata['line'] == urlencode("This line contains ()[] chars")

            assert rule_match[11]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[11]['string_offset'] == 299
            metadata = rule_match[11]['metadata']
            assert metadata['line_number'] == 14
            assert metadata['line'] == urlencode("This line contains {} chars")

            assert rule_match[12]['source'] == ANOTHER_MATCHING_ENTITY_FILE
            assert rule_match[12]['string_offset'] == 327
            metadata = rule_match[12]['metadata']
            assert metadata['line_number'] == 15
            assert metadata['line'] == urlencode("This line contains ^$ chars")

            # Testing a missing file - expect minimal metadata because we can't know the other values
            assert (
                rule_match[13]['source']
                == "matching_entity_3, but without any string matches - yes that's ok"
            )
            metadata = rule_match[13]['metadata']
            assert metadata['source_type'] == 'file'
            assert all(
                [key not in ['file_type', 'md5sum', 'line_number'] for key in metadata.keys()]
            )

            # Testing a missing file for another rule - again, expect minimal metadata because we can't find out more info
            rule_match = mdc.host_scan['another_matching_rule']['matches']
            assert rule_match[2]['source'].endswith(
                'snap/signal-desktop/350/opt/Signal/resources/app.asar'
            )
            metadata = rule_match[2]['metadata']
            assert metadata['source_type'] == 'file'
            assert all(
                [key not in ['file_type', 'md5sum', 'line_number'] for key in metadata.keys()]
            )

            # Testing a missing process - again, expect minimal metadata because we can't find out more info
            assert rule_match[3]['source'] == '1234567'
            metadata = rule_match[3]['metadata']
            assert metadata['source_type'] == 'process'
            assert all(
                [
                    key not in ['process_name', 'file_type', 'md5sum', 'line_number']
                    for key in metadata.keys()
                ]
            )

        @patch(LOGGER_TARGET)
        def test_contrived_scan_output_with_disabled_rules(
            self, log_mock, conf, yara, disabled, rules, cmd, findmnt
        ):
            # Parse the CONTRIVED_SCAN_OUTPUT but with a number of disabled rules
            # Disable 3 of the rules in CONTRIVED_SCAN_OUTPUT, expect 2 matching rules
            disabled.return_value = ['this', 'rule', 'another_matching_rule']
            mdc = MalwareDetectionClient(None)
            assert mdc.disabled_rules == ['this', 'rule', 'another_matching_rule']
            mdc.parse_scan_output(CONTRIVED_SCAN_OUTPUT)
            assert mdc.matches == 2
            assert all(
                list(
                    map(lambda x: x not in ['this', 'Rule', 'another_matching_rule'], mdc.host_scan)
                )
            )
            assert all(list(map(lambda x: x in ['Iyamtho', 'n_m3_t00'], mdc.host_scan)))
            log_mock.debug.assert_any_call(
                'Skipping matches for disabled rule %s in %s %s', 'this', 'file', ANY
            )
            log_mock.debug.assert_any_call(
                'Skipping matches for disabled rule %s in %s %s', 'Rule', 'file', ANY
            )
            log_mock.info.assert_any_call('Matched rule %s in %s %s', 'Iyamtho', 'file', ANY)
            log_mock.info.assert_any_call('Matched rule %s in %s %s', 'n_m3_t00', 'file', ANY)

            # disable all rules except 'Rule', because it isn't excluded
            log_mock.debug.reset_mock()
            log_mock.info.reset_mock()
            disabled.return_value = ['this', 'rool', 'another_matching_rule', 'iyamtho', 'n_m3_t00']
            mdc = MalwareDetectionClient(None)
            mdc.parse_scan_output(CONTRIVED_SCAN_OUTPUT)
            assert list(mdc.host_scan) == ['Rule']
            assert mdc.matches == 3
            sources = set(map(lambda x: x['source'], mdc.host_scan['Rule']['matches']))
            assert len(sources) == 3
            log_mock.debug.assert_any_call(
                'Skipping matches for disabled rule %s in %s %s', 'Iyamtho', 'file', ANY
            )
            log_mock.info.assert_any_call('Matched rule %s in %s %s', 'Rule', 'file', ANY)

        def test_random_output(self, conf, yara, disabled, rules, cmd, findmnt):
            mdc = MalwareDetectionClient(None)
            assert mdc.disabled_rules == []
            mdc.parse_scan_output(RANDOM_OUTPUT)
            assert mdc.matches == 2
            rule_match = mdc.host_scan['Lorem']['matches']
            assert rule_match[0]['source'].startswith('ipsum dolor')
            assert rule_match[0]['string_data'] == ''
            assert rule_match[0]['string_identifier'] == ''
            assert rule_match[0]['string_offset'] == -1
            rule_match = mdc.host_scan['Dictum']['matches']
            assert rule_match[0]['source'].startswith('at tempor')
            assert rule_match[0]['string_data'] == ''
            assert rule_match[0]['string_identifier'] == ''
            assert rule_match[0]['string_offset'] == -1

        @patch(LOGGER_TARGET)
        def test_random_output_with_disabled_rules(
            self, log_mock, conf, yara, disabled, rules, cmd, findmnt
        ):
            # Test various disabled rules with parsing RANDOM_OUTPUT
            # Disable 'Lorem' rule (and others) and expect just to match the 'Dictum' rule
            disabled.return_value = ['random', 'rules', 'and', 'lorem']
            mdc = MalwareDetectionClient(None)
            mdc.parse_scan_output(RANDOM_OUTPUT)
            assert mdc.matches == 1
            assert list(mdc.host_scan) == ['Dictum']
            log_mock.debug.assert_any_call(
                'Skipping matches for disabled rule %s in %s %s', 'Lorem', 'file', ANY
            )

            # Disable both 'Lorem' and 'Dictum' rules and expect to have no matches
            log_mock.debug.reset_mock()
            disabled.return_value = ['dictum', 'lorem']
            mdc = MalwareDetectionClient(None)
            mdc.parse_scan_output(RANDOM_OUTPUT)
            assert mdc.matches == 0
            assert mdc.host_scan == {}
            log_mock.debug.assert_any_call(
                'Skipping matches for disabled rule %s in %s %s', 'Dictum', 'file', ANY
            )
            log_mock.debug.assert_any_call(
                'Skipping matches for disabled rule %s in %s %s', 'Lorem', 'file', ANY
            )

            # Disable rules other than 'Lorem' and 'Dictum' and expect match both those rules
            disabled.return_value = ['dictumm', 'lore', 'and', 'other', 'rules']
            mdc = MalwareDetectionClient(None)
            mdc.parse_scan_output(RANDOM_OUTPUT)
            assert mdc.matches == 2
            assert sorted(mdc.host_scan) == ['Dictum', 'Lorem']
            log_mock.info.assert_any_call('Matched rule %s in %s %s', 'Dictum', 'file', ANY)
            log_mock.info.assert_any_call('Matched rule %s in %s %s', 'Lorem', 'file', ANY)

        def test_parse_metadata_enhanced(self, conf, yara, disabled, rules, cmd, findmnt):
            """Test _parse_metadata_enhanced with various scenarios"""
            mdc = MalwareDetectionClient(None)

            # Test cases: (input, expected_output)
            test_cases = [
                # Empty and whitespace
                ("", {}),
                ("   ", {}),

                # Simple key-value pairs
                ("key=value", {"key": "value"}),
                ("key1=value1,key2=value2", {"key1": "value1", "key2": "value2"}),

                # Spaces around equals sign
                ("key = value", {"key": "value"}),
                ("key =value", {"key": "value"}),
                ("key= value", {"key": "value"}),

                # Quoted values
                ('key="quoted value"', {"key": "quoted value"}),
                ("key='quoted value'", {"key": "quoted value"}),

                # Quoted keys
                ('"key"=value', {"key": "value"}),
                ("'key'=value", {"key": "value"}),

                # Values with commas in quotes
                ('key="value,with,commas"', {"key": "value,with,commas"}),

                # Numbers
                ("age=25", {"age": 25}),
                ("price=19.99", {"price": 19.99}),

                # Booleans
                ("active=true", {"active": True}),
                ("disabled=false", {"disabled": False}),
                ("enabled=True", {"enabled": True}),
                ("hidden=False", {"hidden": False}),

                # Duplicate keys (last value wins)
                ("key=first,key=second", {"key": "second"}),

                # Mixed types
                ('name="John",age=30,active=true', {"name": "John", "age": 30, "active": True}),

                # Multiple spaces
                ("key1  =  value1  ,  key2  =  value2", {"key1": "value1", "key2": "value2"}),

                # Invalid parts (no equals sign) should be skipped
                ("valid=value,invalid,another=value2", {"valid": "value", "another": "value2"}),

                # Empty values
                ("key=", {"key": ""}),

                # Complex quoted values
                ('title="Hello, World!",count=5', {"title": "Hello, World!", "count": 5}),
            ]

            for input_text, expected in test_cases:
                result = mdc._parse_metadata_enhanced(input_text)
                assert result == expected, "Failed for input: {!r}. Got: {!r}, Expected: {!r}".format(
                    input_text, result, expected
                )

        def test_parse_value_enhanced(self, conf, yara, disabled, rules, cmd, findmnt):
            """Test that _parse_value_enhanced parsing is done correctly"""
            mdc = MalwareDetectionClient(None)
            assert not mdc._parse_value_enhanced("")
            assert mdc._parse_value_enhanced("true")
            assert not mdc._parse_value_enhanced("false")
            assert mdc._parse_value_enhanced("null") is None
            assert mdc._parse_value_enhanced("4") == 4
            assert mdc._parse_value_enhanced("4.5") == 4.5
            assert mdc._parse_value_enhanced("lol") == "lol"

        def test_smart_split_comma(self, conf, yara, disabled, rules, cmd, findmnt):
            """Test _smart_split_comma with various scenarios"""
            mdc = MalwareDetectionClient(None)

            # Test cases: (input, expected_output)
            test_cases = [
                ("apple,banana,cherry", ["apple", "banana", "cherry"]),
                ("apple,'banana,orange',cherry", ["apple", "'banana,orange'", "cherry"]),
                ('apple,"banana,orange",cherry', ["apple", '"banana,orange"', "cherry"]),
                (r'apple,"banana\",orange",cherry', ["apple", r'"banana\",orange"', "cherry"]),
                ("", []),
                ("apple", ["apple"]),
                ('"first,part","second,part",third', ['"first,part"', '"second,part"', "third"]),
                ("apple,,cherry", ["apple", "", "cherry"]),
                ("apple,banana,", ["apple", "banana"]),
                ("""'single,comma',"double,comma",plain""", ["'single,comma'", '"double,comma"', "plain"]),
                ('''"He said 'hello, world'",next''', ['''"He said 'hello, world'"''', "next"]),
            ]

            for input_text, expected in test_cases:
                result = mdc._smart_split_comma(input_text)
                assert result == expected, "Failed for input: {!r}".format(input_text)


################################################################################################################
# The following section involves tests that are run only if yara is installed on the system
# They are included to ensure future versions of yara maintain the same behaviour expected by malware-detection
################################################################################################################
REAL_YARA = None
try:
    REAL_YARA = '/bin/yara' if os.path.isfile('/bin/yara') else str(call('which yara')).strip()
except CalledProcessError:
    # Double check its not in /usr/local/bin/yara
    REAL_YARA = '/usr/local/bin/yara' if os.path.isfile('/usr/local/bin/yara') else None

if REAL_YARA:
    REAL_YARA_VERSION = str(call('%s --version' % REAL_YARA)).strip()

    def find_test_item(item_name):
        for path, dirs, files in os.walk(os.getcwd()):
            if item_name in dirs or item_name in files:
                return os.path.join(path, item_name)
        return ""

    def tld(path):
        """
        Return the top level directory for 'path' argument
        """
        full_path = os.path.abspath(path)
        if os.path.samefile(full_path, '/'):
            return full_path
        parent_dirs = []
        get_parent_dirs(full_path, parent_dirs)
        return parent_dirs[0]

    # Location of this test file (might be a better way of determining this!)
    MALWARE_TEST_FILE = find_test_item('test_malware_detection.py')
    # Miscellaneous rules files and other text files used in the tests
    COMPILED_RULES_FILE = os.path.join(TEMP_TEST_DIR, 'compiled_rules.%s.yar' % REAL_YARA_VERSION)
    RULE_RULE_FILE = os.path.join(TEMP_TEST_DIR, 'rule rule.yar')
    RULE_METADATA_TEST_FILE = os.path.join(TEMP_TEST_DIR, 'rule_metadata_test.yar')
    TEST_RULE_SCRIPT = os.path.join(TEMP_TEST_DIR, 'test-rule_process_match.sh')

    @pytest.fixture
    def create_test_files_real_yara():
        # Write the test files to the temp directory
        if not os.path.exists(TEMP_TEST_DIR):
            os.mkdir(TEMP_TEST_DIR)
        with open(TEMP_CONFIG_FILE, 'w') as tcf:
            tcf.write(DEFAULT_MALWARE_CONFIG)
        test_files = [
            (MATCHING_ENTITY_FILE, MATCHING_ENTITY_FILE_CONTENTS),
            (ANOTHER_MATCHING_ENTITY_FILE, ANOTHER_MATCHING_ENTITY_FILE_CONTENTS),
            (TEST_RULE_FILE, TEST_RULE_FILE_CONTENTS),
            (TEST_RULE_SCRIPT, TEST_RULE_SCRIPT_CONTENTS),
            (RULE_RULE_FILE, RULE_RULE_FILE_CONTENTS),
            (RULE_METADATA_TEST_FILE, RULE_METADATA_TEST_FILE_CONTENTS),
        ]
        for test_file, contents in test_files:
            if not os.path.exists(test_file):
                with open(test_file, 'w') as f:
                    f.write(contents)
        os.chmod(TEST_RULE_SCRIPT, 0o755)
        if REAL_YARA:
            with open(COMPILED_RULES_FILE, 'wb') as crf:
                crf.write(TEST_RULE1)
                crf.write(TEST_RULE2)
            os.system("cat {0} | yarac - {0}".format(COMPILED_RULES_FILE))
        yield
        os.system('rm -rf %s' % TEMP_TEST_DIR)

    @pytest.mark.skipif(IS_RHEL6, reason=SKIP_IF_RHEL6_REASON)
    class TestsUtilizingRealYara:

        @patch(CONFIG_FILE_TARGET, TEMP_CONFIG_FILE)
        class TestMalwareDetectionOptions:

            @patch(GET_RULES_TARGET, return_value=[TEST_RULE_FILE])
            @patch(DISABLED_RULES_TARGET, return_value=[])
            def test_running_default_options(
                self, disabled, rules, create_test_files_real_yara, caplog
            ):
                # However, with the default malware options, test_scan is true, so some of the options values
                # will be different from those of the default options.
                # For example, do_filesystem_scan AND do_process_scan are both True when doing a test scan
                logger.setLevel('DEBUG')
                test_pid = str(os.getpid())

                mdc = MalwareDetectionClient(None)
                assert mdc.yara_binary == REAL_YARA
                assert mdc.rules_files == [TEST_RULE_FILE]
                assert mdc.disabled_rules == []
                assert mdc.do_process_scan is True
                if os.path.isfile(TEMP_CONFIG_FILE):
                    assert mdc.scan_fsobjects == [TEMP_CONFIG_FILE]
                    assert mdc.do_filesystem_scan is True
                else:
                    assert mdc.do_filesystem_scan is False
                assert mdc.scan_pids == [test_pid]
                assert mdc.filesystem_scan_since_dict == {'timestamp': None}
                assert mdc.filesystem_scan_exclude_list == []
                assert mdc.network_filesystem_mountpoints == []
                assert mdc.scan_timeout == 3600
                assert mdc.nice_value == 19
                assert mdc.rules_location == '/etc/insights-client/signatures'
                assert mdc.use_remote_rules is True
                yara_cmd = ' '.join(mdc.yara_cmd)
                assert yara_cmd == "nice -n 19 %s -s -N -a 3600 -p %s -r -f %s" % (
                    REAL_YARA,
                    CPUS,
                    mdc.rules_file,
                )
                if os.path.isfile(TEMP_CONFIG_FILE):
                    assert (
                        "Performing a test scan of %s and the current process (PID %s)"
                        % (TEMP_CONFIG_FILE, test_pid)
                        in caplog.text
                    )
                else:
                    assert (
                        "Performing a test scan of the current process (PID %s)" % test_pid
                        in caplog.text
                    )
                assert "Using yara binary: %s" % REAL_YARA in caplog.text

                # Remainder of test may not work in QE Jenkins environment because config file may not exist and
                # can't scan the pytest process
                if os.path.exists(TEMP_CONFIG_FILE):
                    mdc.scan_filesystem()
                    assert (
                        "Scanning specified files in %s ..." % tld(TEMP_CONFIG_FILE) in caplog.text
                    )
                    assert (
                        "Matched rule TEST_RedHatInsightsMalwareDetection in file %s"
                        % TEMP_CONFIG_FILE
                        in caplog.text
                    )

                mdc.scan_processes()
                assert "Scanning process %s ..." % test_pid in caplog.text
                assert (
                    "Matched rule TEST_RedHatInsightsMalwareDetection in process %s" % test_pid
                    in caplog.text
                )

            @patch(GET_RULES_TARGET, return_value=COMPILED_RULES_FILE)
            @patch(DISABLED_RULES_TARGET, return_value=[])
            def test_running_modified_options(self, disabled, rules, create_test_files_real_yara):
                # Disable test_scan and the malware options should mostly be same as what's in the mdc object
                logger.setLevel('DEBUG')
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    print(line)

                mdc = MalwareDetectionClient(None)
                assert mdc.rules_file == COMPILED_RULES_FILE
                assert mdc.disabled_rules == []
                assert mdc.yara_binary == REAL_YARA
                assert mdc.do_filesystem_scan is True
                assert mdc.do_process_scan is False
                assert mdc.scan_fsobjects == []
                assert mdc.scan_pids == []
                assert mdc.filesystem_scan_since_dict == {'timestamp': None, 'datetime': None}
                assert all(
                    [
                        d in mdc.filesystem_scan_exclude_list
                        for d in ['/proc', '/sys', '/mnt', '/media']
                    ]
                )

            def test_scan_only_option(self, create_test_files_real_yara, caplog):
                # Test various combinations of scan_only and the scan_filesystem and scan_processes options
                # Firstly, test the default option values
                logger.setLevel('INFO')
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = (
                        line + "remote_rules_location: %s\n" % COMPILED_RULES_FILE
                        if line.startswith('---')
                        else line
                    )
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    print(line)
                mdc = MalwareDetectionClient(None)
                assert mdc.do_filesystem_scan is True
                assert mdc.do_process_scan is False
                assert mdc.scan_fsobjects == []
                assert mdc.scan_pids == []

                # Add scan_only for a process - expect to exit because we can't scan processes because do_process_scan is false
                caplog.clear()
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = (
                        "processes_scan_only: 1"
                        if line.startswith("processes_scan_only:")
                        else line
                    )
                    print(line)
                MalwareDetectionClient(None)
                assert (
                    "Skipping processes_scan_only option because scan_processes is false"
                    in caplog.text
                )

                # Enable process scanning and now the scan_only value should be used
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "scan_processes: true" if line.startswith("scan_processes:") else line
                    print(line)
                mdc = MalwareDetectionClient(None)
                assert mdc.scan_pids == ['1']

                # Add directories and processes and expect all to be scanned
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = (
                        "filesystem_scan_only:\n- /tmp"
                        if line.startswith("filesystem_scan_only:")
                        else line
                    )
                    line = (
                        "processes_scan_only:\n- 1"
                        if line.startswith("processes_scan_only:")
                        else line
                    )
                    print(line)
                mdc = MalwareDetectionClient(None)
                assert mdc.scan_fsobjects == ['/tmp']
                assert mdc.scan_pids == ['1']

                # Disable filesystem scanning and only expect the process to be scanned
                caplog.clear()
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "scan_filesystem: false" if line.startswith("scan_filesystem:") else line
                    print(line)
                mdc = MalwareDetectionClient(None)
                assert (
                    "Skipping filesystem_scan_only option because scan_filesystem is false"
                    in caplog.text
                )
                assert mdc.scan_pids == ['1']

                # Disable both filesystem and process scanning and expect an error as there is nothing to scan
                caplog.clear()
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "scan_processes: false" if line.startswith("scan_processes:") else line
                    print(line)
                with pytest.raises(SystemExit) as exc_info:
                    MalwareDetectionClient(None)
                assert (
                    "Both scan_filesystem and scan_processes are disabled.  Nothing to scan."
                    in caplog.text
                )
                assert exc_info.value.code == constants.sig_kill_bad

            def test_invalid_option_values(self, create_test_files_real_yara, caplog):
                # Check the malware client app behaves in a predictable way if the user specifies invalid option values
                # Invalid value for nice - should be an integer
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    line = "nice_value: nineteen" if line.startswith("nice_value") else line
                    print(line)
                with pytest.raises(SystemExit) as exc_info:
                    MalwareDetectionClient(None)
                assert (
                    "Problem setting configuration option nice_value: invalid literal"
                    in caplog.text
                )
                assert exc_info.value.code == constants.sig_kill_bad

                # Missing colon for nice_value option
                caplog.clear()
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "nice_value 19\n" if line.startswith("nice_value") else line
                    print(line)
                with pytest.raises(SystemExit) as exc_info:
                    MalwareDetectionClient(None)
                assert (
                    "Error encountered loading the malware-detection app config file" in caplog.text
                )
                assert exc_info.value.code == constants.sig_kill_bad

                # Bad list items for scan_only, mixing single item and list items
                caplog.clear()
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "nice_value: 19\n" if line.startswith("nice_value") else line
                    line = (
                        "filesystem_scan_only: /bad\n- /bad"
                        if line.startswith("filesystem_scan_only:")
                        else line
                    )
                    print(line)
                with pytest.raises(SystemExit) as exc_info:
                    MalwareDetectionClient(None)
                assert (
                    "Error encountered loading the malware-detection app config file" in caplog.text
                )
                assert exc_info.value.code == constants.sig_kill_bad

                # Bad list items for scan_only, not a list item
                caplog.clear()
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = (
                        "filesystem_scan_only:"
                        if line.startswith("filesystem_scan_only:")
                        else line
                    )
                    line = "/bad" if line.startswith("- /bad") else line
                    print(line)
                with pytest.raises(SystemExit) as exc_info:
                    MalwareDetectionClient(None)
                assert (
                    "Error encountered loading the malware-detection app config file" in caplog.text
                )
                assert exc_info.value.code == constants.sig_kill_bad

                # Bad list items for scan_only, not enough spaces
                caplog.clear()
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "-/bad" if line.startswith("/bad") else line
                    print(line)
                with pytest.raises(SystemExit) as exc_info:
                    MalwareDetectionClient(None)
                assert (
                    "Error encountered loading the malware-detection app config file" in caplog.text
                )
                assert exc_info.value.code == constants.sig_kill_bad

                # Bad list items for scan_only, using tabs instead of spaces
                caplog.clear()
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "\t- /bad" if line.startswith("-/bad") else line
                    print(line)
                with pytest.raises(SystemExit) as exc_info:
                    MalwareDetectionClient(None)
                assert (
                    "Error encountered loading the malware-detection app config file" in caplog.text
                )
                assert exc_info.value.code == constants.sig_kill_bad

            @patch.dict(os.environ)
            def test_using_env_vars(self, create_test_files_real_yara, caplog):
                # Set certain option values via environment variables
                env_var_list = [
                    ('REMOTE_RULES_LOCATION', COMPILED_RULES_FILE),
                    ('TEST_SCAN', 'false'),
                    ('SCAN_FILESYSTEM', 'YES'),
                    ('SCAN_PROCESSES', 'hello'),  # will be interpreted as false
                    ('FILESYSTEM_SCAN_ONLY', '/tmp'),
                    ('FILESYSTEM_SCAN_EXCLUDE', '/tmp'),
                    ('FILESYSTEM_SCAN_SINCE', '2'),
                    ('SCAN_TIMEOUT', '1800'),
                ]
                for key, value in env_var_list:
                    os.environ[key] = value

                logger.setLevel('INFO')
                mdc = MalwareDetectionClient(None)
                assert mdc.yara_binary == REAL_YARA
                assert mdc.rules_file == COMPILED_RULES_FILE
                assert mdc.test_scan is False
                assert mdc.do_filesystem_scan is True
                assert mdc.do_process_scan is False
                assert mdc.scan_fsobjects == ['/tmp']
                assert mdc.filesystem_scan_exclude_list == ['/tmp']
                assert mdc.filesystem_scan_since_dict['timestamp'] < time.time() - (2 * 86400)
                assert mdc.scan_timeout == 1800
                # Not env vars, but just checking they have the expected values
                assert mdc.nice_value == 19
                assert mdc.cpu_thread_limit == CPUS
                assert mdc.rules_location == '/etc/insights-client/signatures'
                assert mdc.use_remote_rules is True

                # Start a filesystem scan and expect scan_only and scan_exclude to cancel each other out
                result = mdc.scan_filesystem()
                assert (
                    "No filesystem items to scan because the specified exclude items cancel them out"
                    in caplog.text
                )
                assert result is False

                # Test when SCAN_ONLY and SCAN_EXCLUDE values are comma separated
                for key, value in [
                    ('FILESYSTEM_SCAN_ONLY', '/tmp,/,/var/tmp'),
                    ('FILESYSTEM_SCAN_EXCLUDE', '/home,/,/fred,barney'),
                ]:
                    os.environ[key] = value
                mdc = MalwareDetectionClient(None)
                assert mdc.scan_fsobjects == ['/tmp', '/', '/var/tmp']
                assert mdc.filesystem_scan_exclude_list == ['/home', '/']
                assert mdc.test_scan is False

            @patch(GET_RULES_TARGET, return_value=COMPILED_RULES_FILE)
            @patch(DISABLED_RULES_TARGET)
            @patch.dict(os.environ)
            def test_invalid_env_vars(self, disabled, rules, caplog):
                # Set options to invalid values
                logger.setLevel('INFO')
                os.environ['TEST_SCAN'] = 'false'
                os.environ['NICE_VALUE'] = 'nineteen'
                with pytest.raises(SystemExit) as exc_info:
                    MalwareDetectionClient(None)
                assert (
                    "Problem parsing environment variable NICE_VALUE: invalid literal"
                    in caplog.text
                )
                assert exc_info.value.code == constants.sig_kill_bad

                caplog.clear()
                os.environ['NICE_VALUE'] = '19'
                os.environ['FILESYSTEM_SCAN_SINCE'] = 'blast'
                with pytest.raises(SystemExit) as exc_info:
                    MalwareDetectionClient(None)
                assert "Unknown value 'blast' for filesystem_scan_since option" in caplog.text
                assert exc_info.value.code == constants.sig_kill_bad

            @patch(GET_RULES_TARGET, return_value=TEST_RULE_FILE)
            @patch(DISABLED_RULES_TARGET)
            def test_successful_find_yara_binary(
                self, disabled, rules, create_test_files_real_yara, caplog
            ):
                logger.setLevel('DEBUG')
                # Find yara using the system search path
                # If its already on the system, then malware will find that one, otherwise the test yara will be found
                mdc = MalwareDetectionClient(None)
                assert mdc.yara_binary == REAL_YARA
                assert "Using yara binary: %s" % REAL_YARA in caplog.text

                # Finding yara binary by using the yara_binary option
                # NOTE: this option has been removed from the config file, so specifying it has no effect
                caplog.clear()
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = (
                        "yara_binary: %s" % REAL_YARA if line.startswith("yara_binary:") else line
                    )
                    print(line)
                mdc = MalwareDetectionClient(None)
                assert mdc.yara_binary == REAL_YARA
                assert "Using yara binary: %s" % REAL_YARA in caplog.text

            def test_failing_find_yara_binary(self, create_test_files_real_yara, caplog):
                # Testing failing to find yara

                # Test with no yara found on system.  If yara is present on the system, then skip this test
                if not REAL_YARA:
                    with pytest.raises(SystemExit) as exc_info:
                        MalwareDetectionClient(None)
                    assert (
                        "Couldn't find yara.  Please ensure the yara package is installed"
                        in caplog.text
                    )
                    assert exc_info.value.code == constants.sig_kill_bad

                caplog.clear()
                with patch('os.path.exists', return_value=False):
                    with pytest.raises(SystemExit) as exc_info:
                        MalwareDetectionClient(None)
                assert (
                    "Couldn't find yara.  Please ensure the yara package is installed"
                    in caplog.text
                )
                assert exc_info.value.code == constants.sig_kill_bad

            @patch(GET_RULES_TARGET, return_value=TEST_RULE_FILE)
            @patch(DISABLED_RULES_TARGET)
            @patch(CALL_TARGET)
            def test_find_yara_version(self, call, disabled, rules, caplog):
                # Test checking the version of yara

                # Invalid versions of yara
                for version in ['4.0.99', '4']:
                    call.return_value = version
                    with pytest.raises(SystemExit) as exc_info:
                        MalwareDetectionClient(None)
                    assert (
                        "Found %s with version %s, but malware-detection requires version >= %s\n"
                        % (REAL_YARA, version, MIN_YARA_VERSION)
                        in caplog.text
                    )
                    assert exc_info.value.code == constants.sig_kill_bad
                    caplog.clear()

                # Valid versions of yara
                for version in ['4.1', '10.0.0']:
                    call.return_value = version
                    with patch(BUILD_YARA_COMMAND_TARGET, return_value="yara"):
                        mdc = MalwareDetectionClient(None)
                    assert mdc.yara_binary

            @patch(GET_RULES_TARGET, return_value=COMPILED_RULES_FILE)
            @patch(DISABLED_RULES_TARGET)
            def test_scanning_compiled_rules_file(
                self, disabled, rules, create_test_files_real_yara, caplog
            ):
                # Scan the compiled rules file ... with the compiled rules file
                logger.setLevel('INFO')
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    line = (
                        "filesystem_scan_only: %s" % COMPILED_RULES_FILE
                        if line.startswith("filesystem_scan_only:")
                        else line
                    )
                    print(line)

                mdc = MalwareDetectionClient(None)
                assert mdc.scan_fsobjects == [COMPILED_RULES_FILE]
                mdc.scan_filesystem()
                assert (
                    "Scan only the specified filesystem item: ['%s']" % COMPILED_RULES_FILE
                    in caplog.text
                )
                assert (
                    "Scanning specified files in %s ..." % tld(COMPILED_RULES_FILE) in caplog.text
                )
                assert (
                    "Matched rule MalwareDetectionClientRule in file %s" % COMPILED_RULES_FILE
                    in caplog.text
                )
                assert (
                    "Matched rule MiscellaneousStringsRule in file %s" % COMPILED_RULES_FILE
                    in caplog.text
                )
                # Matched 2 rules
                assert mdc.matches == 2
                # Matched rule strings 3 times
                string_matches = sum([len(mdc.host_scan[x]) for x in mdc.host_scan])
                assert string_matches == 3

                rule_match = mdc.host_scan['MalwareDetectionClientRule']
                assert rule_match[0]['source'] == COMPILED_RULES_FILE
                assert rule_match[0]['string_data'] == 'MalwareDetectionClient'
                assert rule_match[0]['string_identifier'] == '$text1'
                assert rule_match[0]['string_offset'] <= 544

                rule_match = mdc.host_scan['MiscellaneousStringsRule']
                assert rule_match[0]['source'] == COMPILED_RULES_FILE
                assert rule_match[0]['string_data'] == 'sent"'
                assert rule_match[0]['string_identifier'] == '$string1'
                assert rule_match[0]['string_offset'] == 601
                assert rule_match[1]['source'] == COMPILED_RULES_FILE
                assert (
                    rule_match[1]['string_data'] == r'ata_sff\\x00bioset\\x00bond0\\x00cifsd\\x00'
                )
                assert rule_match[1]['string_identifier'] == '$string2'
                assert rule_match[1]['string_offset'] == 616
                assert mdc.scan_processes() is False

            @patch(GET_RULES_TARGET, return_value=COMPILED_RULES_FILE)
            @patch(DISABLED_RULES_TARGET)
            def test_scanning_malware_test_file(self, disabled, rules, create_test_files_real_yara):
                # Scan this test_malware_detection.py file
                # Expect to match both rules in COMPILED_RULES_FILE, but only 1 string match per rule, due to using
                #   yara -f/fast scan option which stops searching for a matching string after finding it once
                logger.setLevel('INFO')
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    line = (
                        "filesystem_scan_only: %s" % MALWARE_TEST_FILE
                        if line.startswith("filesystem_scan_only:")
                        else line
                    )
                    print(line)

                mdc = MalwareDetectionClient(None)
                assert mdc.scan_fsobjects == [MALWARE_TEST_FILE]
                assert mdc.string_match_limit == 10

                mdc.scan_filesystem()
                assert mdc.matches == 2  # Matches the two rules
                rule_match = mdc.host_scan['MalwareDetectionClientRule']
                assert len(rule_match) == 1  # But only 1 string match per rule
                assert rule_match[0]['source'] == MALWARE_TEST_FILE
                assert rule_match[0]['string_data'] == 'MalwareDetectionClient'
                assert rule_match[0]['string_identifier'] == '$text1'
                # First match is near the start of the file
                assert rule_match[0]['string_offset'] < 1000

                # Increase string_match_limit and scan again.
                # Expect to find just one string match per rule again because of the yara fast scan option
                # (Should we still bother using string_match_limit now that we use fast scan option?)
                mdc = MalwareDetectionClient(None)
                assert mdc.scan_fsobjects == [MALWARE_TEST_FILE]
                mdc.string_match_limit = 100
                mdc.scan_filesystem()
                rule_match = mdc.host_scan['MalwareDetectionClientRule']
                assert len(rule_match) == 1

            @patch(GET_RULES_TARGET, return_value=COMPILED_RULES_FILE)
            @patch(DISABLED_RULES_TARGET)
            def test_scan_only_root(self, disabled, rules, create_test_files_real_yara, caplog):
                # Nothing special about root when parsing the scan_only option
                # There is no parsing of root to individual toplevel directories until running scan_filesystem
                logger.setLevel('DEBUG')
                scan_only = '/'
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    line = (
                        "filesystem_scan_only: %s" % scan_only
                        if line.startswith("filesystem_scan_only:")
                        else line
                    )
                    print(line)

                mdc = MalwareDetectionClient(None)
                assert mdc.scan_fsobjects == [scan_only]
                assert "Scan only the specified filesystem item: ['%s']" % scan_only in caplog.text

                # This is called by scan_filesystem to convert '/' into its top level subdirectories
                scan_dict = process_include_exclude_items(
                    include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
                )
                assert (
                    "Found root directory in list of items to scan.  Ignoring the other items ..."
                    in caplog.text
                )
                assert all([x in list(scan_dict.keys()) for x in INCLUDED_TLDS])
                assert '/' not in list(scan_dict.keys())

                # Multiple directories aren't consolidated until later
                caplog.clear()
                scan_only = ['/', '/tmp', '/home']
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = (
                        "filesystem_scan_only: %s" % scan_only
                        if line.startswith("filesystem_scan_only:")
                        else line
                    )
                    print(line)

                mdc = MalwareDetectionClient(None)
                assert mdc.scan_fsobjects == scan_only
                assert "Scan only the specified filesystem items: %s" % scan_only in caplog.text
                scan_dict = process_include_exclude_items(
                    include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
                )
                assert (
                    "Found root directory in list of items to scan.  Ignoring the other items ..."
                    in caplog.text
                )
                assert all([x in list(scan_dict.keys()) for x in INCLUDED_TLDS])
                assert '/' not in list(scan_dict.keys())

            @patch(GET_RULES_TARGET, return_value=COMPILED_RULES_FILE)
            @patch(DISABLED_RULES_TARGET)
            def test_scan_exclude_root(self, disabled, rules, create_test_files_real_yara, caplog):
                # Nothing special about root when parsing the scan_exclude option
                # There is no parsing of root to individual toplevel directories until running scan_filesystem
                logger.setLevel('DEBUG')
                # Add '/' to the list of scan_exclude items.  Add it directly after the scan_exclude: line
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    line = line + "\n- /" if line.startswith("filesystem_scan_exclude:") else line
                    print(line)
                mdc = MalwareDetectionClient(None)
                assert '/' in mdc.filesystem_scan_exclude_list

                # When scan_filesystem is run, '/' will be expanded into toplevel directories that cancel out everything
                process_include_exclude_items(
                    include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
                )
                assert (
                    "Found root directory in the exclude list.  Expanding it to all toplevel directories ..."
                    in caplog.text
                )
                assert (
                    "No filesystem items to scan because the specified exclude items cancel them out"
                    in caplog.text
                )

            @patch(GET_RULES_TARGET, return_value=COMPILED_RULES_FILE)
            @patch(DISABLED_RULES_TARGET)
            def test_scan_only_scan_exclude_nullify(
                self, disabled, rules, create_test_files_real_yara, caplog
            ):
                # Testing scan_only and scan_exclude items such that the exclude items nullify all the scan_only items
                # In which case there will be nothing to scan
                logger.setLevel('INFO')
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    line = (
                        line + "\n- /var/log\n- /usr/lib/systemd\n- /tmp"
                        if line.startswith("filesystem_scan_only:")
                        else line
                    )
                    line = (
                        line + "\n- /tmp/\n- /usr/lib/\n- /var/log"
                        if line.startswith("filesystem_scan_exclude:")
                        else line
                    )
                    print(line)

                mdc = MalwareDetectionClient(None)
                assert mdc.scan_fsobjects == ['/var/log', '/usr/lib/systemd', '/tmp']
                assert all(
                    [
                        x in mdc.filesystem_scan_exclude_list
                        for x in ['/tmp', '/usr/lib', '/var/log']
                    ]
                )
                result = mdc.scan_filesystem()
                assert (
                    "No filesystem items to scan because the specified exclude items cancel them out"
                    in caplog.text
                )
                assert result is False

                # Both scan_only and scan_exclude contain root
                caplog.clear()
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = line + "\n- /" if line.startswith("filesystem_scan_only:") else line
                    line = line + "\n- /" if line.startswith("filesystem_scan_exclude:") else line
                    print(line)
                mdc = MalwareDetectionClient(None)
                assert mdc.scan_fsobjects == ['/', '/var/log', '/usr/lib/systemd', '/tmp']
                assert all(
                    [
                        x in mdc.filesystem_scan_exclude_list
                        for x in ['/', '/tmp', '/usr/lib', '/var/log']
                    ]
                )
                result = mdc.scan_filesystem()
                assert (
                    "No filesystem items to scan because the specified exclude items cancel them out"
                    in caplog.text
                )
                assert result is False

            @patch.dict(os.environ, {'TEST_SCAN': 'false'})
            @patch(GET_RULES_TARGET)
            @patch(DISABLED_RULES_TARGET)
            def test_rules_file_types(self, disabled, rules, create_test_files_real_yara, caplog):
                # Testing if the rules file is compiled or not
                logger.setLevel('DEBUG')
                rules.return_value = TEST_RULE_FILE
                mdc = MalwareDetectionClient(None)
                assert '-C' not in mdc.yara_cmd
                assert 'Compiled rules: False' in caplog.text

                caplog.clear()
                rules.return_value = COMPILED_RULES_FILE
                mdc = MalwareDetectionClient(None)
                assert '-C' in mdc.yara_cmd
                assert 'Compiled rules: True' in caplog.text

            def test_bad_rules_files(self, create_test_files_real_yara, caplog):
                # Tests with non-rules/problematic files
                with open(TEMP_CONFIG_FILE, 'a') as tcf:
                    # Append remote_rules_location: and test_rule: to the bottom of the config file
                    tcf.write("remote_rules_location: %s\ntest_scan: false\n" % REAL_YARA)
                with pytest.raises(SystemExit) as exc_info:
                    MalwareDetectionClient(None)
                assert "Unable to use rules file %s" % REAL_YARA in caplog.text
                assert exc_info.value.code == constants.sig_kill_bad

                # Missing rules file
                caplog.clear()
                missing_file = os.path.join(TEMP_TEST_DIR, 'missing')
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = (
                        "remote_rules_location: %s" % missing_file
                        if line.startswith("remote_rules_location:")
                        else line
                    )
                    print(line)
                with pytest.raises(SystemExit) as exc_info:
                    MalwareDetectionClient(None)
                assert "Couldn't find specified rules file: %s" % missing_file in caplog.text
                assert exc_info.value.code == constants.sig_kill_bad

                # Specify directory instead of a file
                caplog.clear()
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = (
                        "remote_rules_location: %s" % TEMP_TEST_DIR
                        if line.startswith("remote_rules_location:")
                        else line
                    )
                    print(line)
                with pytest.raises(SystemExit) as exc_info:
                    MalwareDetectionClient(None)
                assert "Couldn't find specified rules file: %s" % TEMP_TEST_DIR in caplog.text
                assert exc_info.value.code == constants.sig_kill_bad

            def test_files_with_extra_slashes_1(self, create_test_files_real_yara, caplog):
                # Test behaviour when remote_rules_location file and scan_only files have extra slashes in them
                logger.setLevel('INFO')
                # Test scanning COMPILED_RULES_FILE with an extra slash only in the remote_rules_location one
                # Even with the extra slashes in the remote_rules_location there will be rules matched
                # because */rules_compiled.yar and *//rules_compiled.yar are the same file
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    line = (
                        line + "remote_rules_location: %s\n" % COMPILED_RULES_FILE.replace('/', '//')
                        if line.startswith('---')
                        else line
                    )
                    line = (
                        "filesystem_scan_only: %s" % COMPILED_RULES_FILE
                        if line.startswith("filesystem_scan_only:")
                        else line
                    )
                    print(line)
                mdc = MalwareDetectionClient(None)
                assert mdc.remote_rules_location == COMPILED_RULES_FILE.replace('/', '//')
                assert mdc.rules_file == COMPILED_RULES_FILE
                assert mdc.scan_fsobjects == [COMPILED_RULES_FILE]
                mdc.scan_filesystem()
                rule_match = mdc.host_scan['MalwareDetectionClientRule']
                assert rule_match[0]['source'] == COMPILED_RULES_FILE
                # Will match the rules file because it is the same file
                assert (
                    "Matched rule MalwareDetectionClientRule in file %s" % COMPILED_RULES_FILE
                    in caplog.text
                )
                assert (
                    "Matched rule MiscellaneousStringsRule in file %s" % COMPILED_RULES_FILE
                    in caplog.text
                )

            def test_files_with_extra_slashes_2(self, create_test_files_real_yara, caplog):
                # Test behaviour when remote_rules_location file and scan_only files have extra slashes in them
                # Replace slashes in both remote_rules_location and scan_only with double slashes
                logger.setLevel('INFO')
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    if line.startswith('---'):
                        line = line + "remote_rules_location: %s\n" % COMPILED_RULES_FILE.replace(
                            '/', '//'
                        )
                    if line.startswith("filesystem_scan_only:"):
                        line = line + "- %s\n- %s" % (
                            TEMP_TEST_DIR.replace('/', '//'),
                            MALWARE_TEST_FILE.replace('/', '//'),
                        )
                    print(line)
                mdc = MalwareDetectionClient(None)
                assert mdc.remote_rules_location == COMPILED_RULES_FILE.replace('/', '//')
                assert mdc.rules_file == COMPILED_RULES_FILE
                assert mdc.scan_fsobjects == [TEMP_TEST_DIR, MALWARE_TEST_FILE]
                mdc.scan_filesystem()
                rule_match = mdc.host_scan['MalwareDetectionClientRule']
                assert len(rule_match) == 1
                assert rule_match[0]['source'] == MALWARE_TEST_FILE
                assert (
                    "Matched rule MalwareDetectionClientRule in file %s" % MALWARE_TEST_FILE
                    in caplog.text
                )
                assert (
                    "Matched rule MiscellaneousStringsRule in file %s" % MALWARE_TEST_FILE
                    in caplog.text
                )
                # Won't match anything in the rules file because we are not specifically scanning that
                assert (
                    "Matched rule MalwareDetectionClientRule in file %s" % COMPILED_RULES_FILE
                    not in caplog.text
                )
                assert (
                    "Matched rule MiscellaneousStringsRule in file %s" % COMPILED_RULES_FILE
                    not in caplog.text
                )

            def test_files_with_extra_slashes_3(self, create_test_files_real_yara, caplog):
                # Test behaviour when remote_rules_location file and scan_only files have extra slashes in them
                # For completeness (silliness?), try with even more slashes in the file names
                logger.setLevel('INFO')
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    if line.startswith('---'):
                        line = line + "remote_rules_location: %s" % COMPILED_RULES_FILE.replace(
                            '/', '////'
                        )
                    if line.startswith("filesystem_scan_only:"):
                        line = line + "- %s\n- %s" % (
                            TEMP_TEST_DIR.replace('/', '///'),
                            MALWARE_TEST_FILE.replace('/', '///'),
                        )
                    print(line)
                mdc = MalwareDetectionClient(None)
                assert mdc.remote_rules_location == COMPILED_RULES_FILE.replace('/', '////')
                assert mdc.rules_file == COMPILED_RULES_FILE
                assert mdc.scan_fsobjects == [TEMP_TEST_DIR, MALWARE_TEST_FILE]
                mdc.scan_filesystem()
                rule_match = mdc.host_scan['MalwareDetectionClientRule']
                assert rule_match[0]['source'] == MALWARE_TEST_FILE
                assert (
                    "Matched rule MalwareDetectionClientRule in file %s" % MALWARE_TEST_FILE
                    in caplog.text
                )
                assert (
                    "Matched rule MiscellaneousStringsRule in file %s" % MALWARE_TEST_FILE
                    in caplog.text
                )
                # Won't match anything in the rules file because we are not specifically scanning that
                assert (
                    "Matched rule MalwareDetectionClientRule in file %s" % COMPILED_RULES_FILE
                    not in caplog.text
                )
                assert (
                    "Matched rule MiscellaneousStringsRule in file %s" % COMPILED_RULES_FILE
                    not in caplog.text
                )

            def test_files_with_extra_slashes_4(self, create_test_files_real_yara, caplog):
                # Test behaviour when remote_rules_location file and scan_only files have extra slashes in them

                # Chaos monkey! - add extra slashes to rules_file and scan_fsobjects AFTER they have been verified
                # This is a contrived test and shouldn't happen in normal code flow but done to make sure malware behaves well
                # Testing passing double slashes directly into mdc just before mdc.scan_filesystems, eg
                # mdc.scan_fsobjects = ['//home//bob']; mdc.scan_filesystem()
                logger.setLevel('INFO')
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    line = (
                        line + "remote_rules_location: %s" % COMPILED_RULES_FILE
                        if line.startswith("---")
                        else line
                    )
                    line = (
                        line + "- %s\n- %s" % (TEMP_TEST_DIR, MALWARE_TEST_FILE)
                        if line.startswith("filesystem_scan_only:")
                        else line
                    )
                    print(line)
                mdc = MalwareDetectionClient(None)
                assert mdc.rules_file == COMPILED_RULES_FILE
                assert mdc.scan_fsobjects == [TEMP_TEST_DIR, MALWARE_TEST_FILE]
                # Modify rules_file and scan_fsobjects AFTER they have been verified
                mdc.rules_file = COMPILED_RULES_FILE.replace('/', '//')
                mdc.scan_fsobjects = [
                    TEMP_TEST_DIR.replace('/', '//'),
                    MALWARE_TEST_FILE.replace('/', '//'),
                ]
                mdc.scan_filesystem()
                assert "Scanning specified files in %s" % tld(TEMP_TEST_DIR) in caplog.text
                rule_match = mdc.host_scan['MalwareDetectionClientRule']
                assert rule_match[0]['source'] == MALWARE_TEST_FILE
                rule_match = mdc.host_scan['MiscellaneousStringsRule']
                assert rule_match[0]['source'] == MALWARE_TEST_FILE
                assert (
                    "Matched rule MalwareDetectionClientRule in file %s" % MALWARE_TEST_FILE
                    in caplog.text
                )
                assert (
                    "Matched rule MiscellaneousStringsRule in file %s" % MALWARE_TEST_FILE
                    in caplog.text
                )

            def test_scan_exclude_with_extra_slashes_1(self, create_test_files_real_yara, caplog):
                # Testing we handle the situation where items in scan_exclude contain multiple slashes!
                logger.setLevel('INFO')
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    line = (
                        line + "remote_rules_location: %s" % COMPILED_RULES_FILE
                        if line.startswith("---")
                        else line
                    )
                    line = line + "- //\n" if line.startswith("filesystem_scan_exclude:") else line
                    print(line)
                mdc = MalwareDetectionClient(None)
                result = mdc.scan_filesystem()
                assert (
                    "No filesystem items to scan because the specified exclude items cancel them out"
                    in caplog.text
                )
                assert result is False

            def test_scan_exclude_with_extra_slashes_2(self, create_test_files_real_yara, caplog):
                # Testing we handle the situation where items in scan_exclude contain multiple slashes!
                # Chaos monkey! - add extra slashes to scan_exclude items AFTER they have been verified
                # This is a contrived test and shouldn't happen in normal code flow but done to make sure malware behaves well
                logger.setLevel('INFO')
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    line = (
                        line + "remote_rules_location: %s" % COMPILED_RULES_FILE
                        if line.startswith("---")
                        else line
                    )
                    line = (
                        "filesystem_scan_only: %s" % TEMP_TEST_DIR
                        if line.startswith("filesystem_scan_only:")
                        else line
                    )
                    line = (
                        line + "- %s\n" % TEMP_TEST_DIR
                        if line.startswith("filesystem_scan_exclude:")
                        else line
                    )
                    print(line)
                mdc = MalwareDetectionClient(None)
                assert mdc.scan_fsobjects == [TEMP_TEST_DIR]
                assert TEMP_TEST_DIR in mdc.filesystem_scan_exclude_list
                # Modify rules_file and scan_fsobjects AFTER they have been verified
                mdc.filesystem_scan_exclude_list = [TEMP_TEST_DIR.replace('/', '//')]
                result = mdc.scan_filesystem()
                assert (
                    "No filesystem items to scan because the specified exclude items cancel them out"
                    in caplog.text
                )
                assert result is False

            def test_scanning_pytest_process(self, create_test_files_real_yara, caplog):
                # Do a test scan to scan the python pytest process whilst its running the tests
                logger.setLevel('INFO')
                test_pid = str(os.getpid())
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = (
                        line + "remote_rules_location: %s" % COMPILED_RULES_FILE
                        if line.startswith("---")
                        else line
                    )
                    print(line)
                mdc = MalwareDetectionClient(None)
                assert "and the current process (PID %s)" % test_pid in caplog.text
                assert mdc.do_filesystem_scan is True
                assert mdc.do_process_scan is True

                mdc.scan_processes()
                assert "Scanning process %s ..." % test_pid in caplog.text
                assert (
                    "Matched rule MalwareDetectionClientRule in process %s" % test_pid
                    in caplog.text
                )
                assert (
                    "Matched rule MiscellaneousStringsRule in process %s" % test_pid in caplog.text
                )
                # Matches both rules in the PID
                assert mdc.matches == 2
                rule_match = mdc.host_scan['MalwareDetectionClientRule']
                assert len(rule_match) == 1
                assert rule_match[0]['source'] == test_pid
                assert rule_match[0]['string_data'] == 'MalwareDetectionClient'
                assert rule_match[0]['string_identifier'] == '$text1'
                assert rule_match[0]['string_offset'] > 0
                metadata = rule_match[0]['metadata']
                assert metadata['source_type'] == 'process'
                # The process running the tests should have any of these strings in its name
                assert any([s in metadata['process_name'] for s in ('python', 'pytest', 'tests')])
                # Check that file related metadata keys are not present
                assert all(
                    [key not in ['file_type', 'md5sum', 'line_number'] for key in metadata.keys()]
                )

            def test_filenames_containing_spaces(self, create_test_files_real_yara, caplog):
                # Check that filenames with spaces in them are handled ok
                logger.setLevel('DEBUG')
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    line = (
                        line + "remote_rules_location: %s" % RULE_RULE_FILE
                        if line.startswith("---")
                        else line
                    )
                    line = (
                        "filesystem_scan_only: %s" % ANOTHER_MATCHING_ENTITY_FILE
                        if line.startswith("filesystem_scan_only:")
                        else line
                    )
                    print(line)
                mdc = MalwareDetectionClient(None)
                assert (
                    "Scan only the specified filesystem item: ['%s']" % ANOTHER_MATCHING_ENTITY_FILE
                    in caplog.text
                )
                assert "Using specified rules file: %s" % RULE_RULE_FILE in caplog.text
                assert (
                    "Yara command: ['nice', '-n', '19', '%s', '-s', '-N', '-a', '3600', '-p', '%s', '-r', '-f', '%s']"
                    % (REAL_YARA, CPUS, RULE_RULE_FILE)
                    in caplog.text
                )
                mdc.scan_filesystem()
                rule_match = mdc.host_scan['Rule']
                assert len(rule_match) == 8

        @patch(CONFIG_FILE_TARGET, TEMP_CONFIG_FILE)
        class TestLineNumberMetadata:

            @patch(GET_RULES_TARGET, return_value=RULE_RULE_FILE)
            @patch(DISABLED_RULES_TARGET)
            def test_rule_rule_scan_another_matching_entity(
                self, disabled, rules, create_test_files_real_yara
            ):
                mdc = MalwareDetectionClient(None)
                mdc.scan_fsobjects = [ANOTHER_MATCHING_ENTITY_FILE]
                mdc.scan_filesystem()

                # 8 matching strings for 'Rule' in 'another matching_entity' file
                rule_match = mdc.host_scan['Rule']
                assert len(rule_match) == 8

                assert rule_match[0]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert (
                    rule_match[0]['string_data']
                    == "string match containing error scanning but it's ok because its not in a rule line"
                )
                assert rule_match[0]['string_identifier'] == '$match3'
                assert rule_match[0]['string_offset'] == 2
                metadata = rule_match[0]['metadata']
                assert metadata['source_type'] == 'file'
                assert metadata['file_type'] == 'ASCII text'
                assert metadata['mime_type'] == 'text/plain; charset=us-ascii'
                assert metadata['md5sum'] == '64764d295e92ffeec36d3fcd646a3af4'
                assert metadata['line_number'] == 3
                assert metadata['line'] == urlencode(
                    "string match containing error scanning but it's ok because its not in a rule line"
                )

                assert rule_match[1]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[1]['string_data'] == "contains ="
                assert rule_match[1]['string_identifier'] == '$grep1'
                assert rule_match[1]['string_offset'] == 97
                metadata = rule_match[1]['metadata']
                assert metadata['md5sum'] == '64764d295e92ffeec36d3fcd646a3af4'
                assert metadata['line_number'] == 7
                assert metadata['line'] == urlencode("This line contains = char")

                assert rule_match[2]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[2]['string_data'] == "contains .+"
                assert rule_match[2]['string_identifier'] == '$grep2'
                assert rule_match[2]['string_offset'] == 153
                metadata = rule_match[2]['metadata']
                assert metadata['line_number'] == 9
                assert metadata['line'] == urlencode("This line contains .+ chars")

                assert rule_match[3]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[3]['string_data'] == 'contains "'
                assert rule_match[3]['string_identifier'] == '$grep3'
                assert rule_match[3]['string_offset'] == 213
                metadata = rule_match[3]['metadata']
                assert metadata['line_number'] == 11
                assert metadata['line'] == urlencode('This line contains "" chars')

                assert rule_match[4]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[4]['string_data'] == "contains '"
                assert rule_match[4]['string_identifier'] == '$grep4'
                assert rule_match[4]['string_offset'] == 241
                metadata = rule_match[4]['metadata']
                assert metadata['line_number'] == 12
                assert metadata['line'] == urlencode("This line contains '' chars")

                assert rule_match[5]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[5]['string_data'] == 'contains ()[]'
                assert rule_match[5]['string_identifier'] == '$grep5'
                assert rule_match[5]['string_offset'] == 269
                metadata = rule_match[5]['metadata']
                assert metadata['line_number'] == 13
                assert metadata['line'] == urlencode("This line contains ()[] chars")

                assert rule_match[6]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[6]['string_data'] == 'contains {'
                assert rule_match[6]['string_identifier'] == '$grep6'
                assert rule_match[6]['string_offset'] == 299
                metadata = rule_match[6]['metadata']
                assert metadata['line_number'] == 14
                assert metadata['line'] == urlencode("This line contains {} chars")

                assert rule_match[7]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[7]['string_data'] == 'contains ^$'
                assert rule_match[7]['string_identifier'] == '$grep7'
                assert rule_match[7]['string_offset'] == 327
                metadata = rule_match[7]['metadata']
                assert metadata['line_number'] == 15
                assert metadata['line'] == urlencode("This line contains ^$ chars")

            @patch(GET_RULES_TARGET, return_value=RULE_METADATA_TEST_FILE)
            @patch(DISABLED_RULES_TARGET)
            def test_rule_metadata_test_scanning_itself(
                self, disabled, rules, create_test_files_real_yara
            ):
                # Taking some complicated rule strings to make sure grepping for line numbers is working correctly
                mdc = MalwareDetectionClient(None)
                mdc.scan_fsobjects = [RULE_METADATA_TEST_FILE]
                mdc.scan_filesystem()

                rule_match = mdc.host_scan['MetadataTestRule']
                assert len(rule_match) == 9

                assert rule_match[0]['source'] == RULE_METADATA_TEST_FILE
                assert (
                    rule_match[0]['string_data']
                    == r'echo -e "[-] Ping \\033[31m${host_name}\\033[0m bad"'
                )
                assert rule_match[0]['string_identifier'] == '$s1'
                assert rule_match[0]['string_offset'] == 392
                metadata = rule_match[0]['metadata']
                assert metadata['source_type'] == 'file'
                assert metadata['file_type'] == 'UTF-8 Unicode text'
                assert metadata['mime_type'] == 'text/plain; charset=utf-8'
                assert metadata['md5sum'] == '6ea8306ed20cede50ea10964dddc8d47'
                assert metadata['line_number'] == 9
                assert metadata['line'] == urlencode(
                    r'Testing $s1 = "echo -e "[-] Ping \\033[31m${host_name}\\033[0m bad" ascii fullword'
                )

                assert rule_match[1]['source'] == RULE_METADATA_TEST_FILE
                assert rule_match[1]['string_data'] == '"${user_name}"@"${host_name}" -p "${port}'
                assert rule_match[1]['string_identifier'] == '$s2'
                assert rule_match[1]['string_offset'] == 477
                metadata = rule_match[1]['metadata']
                assert metadata['line_number'] == 10
                assert metadata['line'] == urlencode(
                    'Testing $s2 = ""${user_name}"@"${host_name}" -p "${port}" ascii fullword'
                )

                assert rule_match[2]['source'] == RULE_METADATA_TEST_FILE
                assert rule_match[2]['string_data'] == """'$password' &" <<< GMANcode27'"""
                assert rule_match[2]['string_identifier'] == '$s3'
                assert rule_match[2]['string_offset'] == 554
                metadata = rule_match[2]['metadata']
                assert metadata['line_number'] == 11
                assert metadata['line'] == urlencode(
                    """Testing $s3 = "'$password' &" <<< GMANcode27'" ascii fullword"""
                )

                assert rule_match[3]['source'] == RULE_METADATA_TEST_FILE
                assert rule_match[3]['string_data'] == "for ssh_creds in ${allThreads[@]}; do"
                assert rule_match[3]['string_identifier'] == '$s4'
                assert rule_match[3]['string_offset'] == 119
                metadata = rule_match[3]['metadata']
                assert metadata['line_number'] == 4
                assert metadata['line'] == urlencode(
                    'Testing $s4 = "for ssh_creds in ${allThreads[@]}; do" ascii fullword'
                )

                assert rule_match[4]['source'] == RULE_METADATA_TEST_FILE
                assert rule_match[4]['string_data'] == '"text=$MSG" "$MSG_URL$id&"'
                assert rule_match[4]['string_identifier'] == '$s5'
                assert rule_match[4]['string_offset'] == 620
                metadata = rule_match[4]['metadata']
                assert metadata['line_number'] == 12
                assert metadata['line'] == urlencode(
                    'Testing $s5 = ""text=$MSG" "$MSG_URL$id&"" ascii fullword'
                )

                # Cannot match line_numbers for this rule due to non-ascii chars
                assert rule_match[5]['source'] == RULE_METADATA_TEST_FILE
                assert rule_match[5]['string_data'] == r"--exclude=\\*.\\xE2\\x98\\xA2 -l"
                assert rule_match[5]['string_identifier'] == '$s6'
                assert rule_match[5]['string_offset'] == 682
                metadata = rule_match[5]['metadata']
                assert all([key not in ['line_number', 'line'] for key in metadata.keys()])

                assert rule_match[6]['source'] == RULE_METADATA_TEST_FILE
                assert rule_match[6]['string_data'] == r"--include=\\*.{txt,sh,exe}"
                assert rule_match[6]['string_identifier'] == '$s7'
                assert rule_match[6]['string_offset'] == 731
                metadata = rule_match[6]['metadata']
                assert metadata['line_number'] == 14
                assert metadata['line'] == urlencode(
                    r'Testing $s7 = "--include=\*.{txt,sh,exe}" ascii fullword'
                )

                assert rule_match[7]['source'] == RULE_METADATA_TEST_FILE
                assert rule_match[7]['string_data'] == "allThreads=($1)"
                assert rule_match[7]['string_identifier'] == '$s8'
                assert rule_match[7]['string_offset'] == 192
                metadata = rule_match[7]['metadata']
                assert metadata['line_number'] == 5
                assert metadata['line'] == urlencode(
                    'Testing $s8 = "allThreads=($1)" ascii fullword'
                )

                assert rule_match[8]['source'] == RULE_METADATA_TEST_FILE
                assert rule_match[8]['string_data'] == "$(host): encrypt files. Done."
                assert rule_match[8]['string_identifier'] == '$s9'
                assert rule_match[8]['string_offset'] == 243
                metadata = rule_match[8]['metadata']
                assert metadata['line_number'] == 6
                assert metadata['line'] == urlencode(
                    'Testing $s9 = "$(host): encrypt files. Done." ascii fullword'
                )

        @patch(CONFIG_FILE_TARGET, TEMP_CONFIG_FILE)
        # Rule used is irrelevant for these tests because we aren't actually scanning, just parsing canned yara output
        @patch(GET_RULES_TARGET, return_value=TEST_RULE_FILE)
        @patch(DISABLED_RULES_TARGET, return_value=[])
        class TestParseScanOutput:

            def test_contrived_scan_output(
                self, disabled, rules, create_test_files_real_yara, caplog
            ):
                # Parse the CONTRIVED_SCAN_OUTPUT to find actual rule matches amongst malformed output lines
                # The remote_rules_location is irrelevant to the test but it just makes the initialization of
                # MalwareDetectionClient easier if done this way (and its simpler than using @patch ... maybe)
                logger.setLevel(
                    'DEBUG'
                )  # Make sure the logging level is set *before* performing the scan
                mdc = MalwareDetectionClient(None)
                mdc.add_metadata = False
                mdc.parse_scan_output(CONTRIVED_SCAN_OUTPUT)

                # Expect 5 rule matches and 8 source/file matches across the 5 rules
                # 1 source/file match for rule 'this', 3 source/file matches for rule 'Rule',
                # 2 for 'another_matching_rule', 1 for 'Iyamtho' and 1 for 'n_m3_t00'
                assert len(mdc.host_scan) == 5  # 5 rules matched
                assert mdc.matches == 8  # 8 sources/files matched across the 5 rules

                # 1 matching source/file and 1 matching string for rule 'this'
                rule_match = mdc.host_scan['this']
                rule_sources = set(map(lambda x: x['source'], rule_match))
                assert len(rule_sources) == 1
                assert len(rule_match) == 1
                assert 'e-r-r-o-r s-c-a-n-n-i-n-g' in rule_match[0]['source']
                assert rule_match[0]['string_data'] == "matches 'this' rule"
                assert rule_match[0]['string_identifier'] == '$match'
                assert rule_match[0]['string_offset'] == 291

                # 3 source/file matches and 14 matching strings across those sources for rule 'Rule'
                rule_match = mdc.host_scan['Rule']
                rule_sources = set(map(lambda x: x['source'], rule_match))
                assert len(rule_sources) == 3
                assert len(rule_match) == 14
                assert rule_match[0]['source'] == MATCHING_ENTITY_FILE
                assert rule_match[0]['string_data'] == 'string match in the file "matching_entity"'
                assert rule_match[0]['string_identifier'] == '$match0'
                assert rule_match[0]['string_offset'] == 21
                assert rule_match[1]['source'] == MATCHING_ENTITY_FILE
                assert rule_match[1]['string_data'] == "another string match in matching_entity"
                assert rule_match[1]['string_identifier'] == '$match1'
                assert rule_match[1]['string_offset'] == 83
                assert rule_match[2]['source'] == MATCHING_ENTITY_FILE
                assert (
                    rule_match[2]['string_data']
                    == 'string with different types of quotes \'here\' and "here"'
                )
                assert rule_match[2]['string_identifier'] == '$match2'
                assert rule_match[2]['string_offset'] == 230

                # Rule matches for ANOTHER_MATCHING_ENTITY_FILE (which has a space in the filename)
                assert rule_match[3]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert (
                    rule_match[3]['string_data']
                    == "string match containing error scanning but it's ok because its not in a rule line"
                )
                assert rule_match[3]['string_identifier'] == '$match3'
                assert rule_match[3]['string_offset'] == 2
                assert rule_match[4]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[4]['string_data'] == "contains ="
                assert rule_match[4]['string_identifier'] == '$grep1'
                assert rule_match[4]['string_offset'] == 97
                assert rule_match[6]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[6]['string_data'] == "contains .+"
                assert rule_match[6]['string_identifier'] == '$grep2'
                assert rule_match[6]['string_offset'] == 153
                assert rule_match[8]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[8]['string_data'] == 'contains "'
                assert rule_match[8]['string_identifier'] == '$grep3'
                assert rule_match[8]['string_offset'] == 213
                assert rule_match[9]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[9]['string_data'] == "contains '"
                assert rule_match[9]['string_identifier'] == '$grep4'
                assert rule_match[9]['string_offset'] == 241
                assert rule_match[10]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[10]['string_data'] == 'contains ()[]'
                assert rule_match[10]['string_identifier'] == '$grep5'
                assert rule_match[10]['string_offset'] == 269
                assert rule_match[11]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[11]['string_data'] == 'contains {'
                assert rule_match[11]['string_identifier'] == '$grep6'
                assert rule_match[11]['string_offset'] == 299
                assert rule_match[12]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[12]['string_data'] == 'contains ^$'
                assert rule_match[12]['string_identifier'] == '$grep7'
                assert rule_match[12]['string_offset'] == 327

                assert rule_match[13]['source'].startswith('matching_entity_3')
                assert rule_match[13]['string_data'] == ''
                assert rule_match[13]['string_identifier'] == ''
                assert rule_match[13]['string_offset'] == -1

                # 2 source matches and 4 matching strings for 'another_matching_rule'
                rule_match = mdc.host_scan['another_matching_rule']
                rule_sources = set(map(lambda x: x['source'], rule_match))
                assert len(rule_sources) == 2
                assert len(rule_match) == 4
                assert rule_match[2]['source'].endswith(
                    'snap/signal-desktop/350/opt/Signal/resources/app.asar'
                )
                assert rule_match[2]['string_data'] == '#!/bin/sh'
                assert rule_match[2]['string_identifier'] == '$s0'
                assert rule_match[2]['string_offset'] == 60783814
                assert rule_match[3]['source'] == '1234567'
                assert rule_match[3]['string_data'] == '#!/bin/sh'
                assert rule_match[3]['string_identifier'] == '$s0'
                assert rule_match[3]['string_offset'] == 0

                # 1 matching source and 1 matching string for 'Iyamtho'
                rule_match = mdc.host_scan['Iyamtho']
                assert len(rule_match) == 1
                assert rule_match[0]['source'] == " yep"
                assert rule_match[0]['string_data'] == ''
                assert rule_match[0]['string_identifier'] == ''
                assert rule_match[0]['string_offset'] == -1

                # 1 matching source and 1 matching string for 'n_m3_t00'
                rule_match = mdc.host_scan['n_m3_t00']
                assert len(rule_match) == 1
                assert rule_match[0]['source'] == "damn   straight"
                assert rule_match[0]['string_data'] == ''
                assert rule_match[0]['string_identifier'] == ''
                assert rule_match[0]['string_offset'] == -1

                assert (
                    "Error parsing string match '0x1badoffset:$s1: skip this line': " in caplog.text
                )
                assert (
                    "Error parsing string match '0x2error scanning skip/this/line/too: need more colons': "
                    in caplog.text
                )

            def test_contrived_scan_output_metadata(
                self, disabled, rules, create_test_files_real_yara, caplog
            ):
                # Again, parse the CONTRIVED_SCAN_OUTPUT to find actual rule matches amongst malformed output lines,
                # but this time check the expected metadata values too

                # Again, need to populate rules_file_location with any rule, but its not relevant for the tests
                logger.setLevel('DEBUG')
                mdc = MalwareDetectionClient(None)
                mdc.parse_scan_output(CONTRIVED_SCAN_OUTPUT)

                # Matches and metadata for MATCHING_ENTITY_FILE
                # assert "grep -Ebon  -e 'another string match in matching_entity'" in caplog.text
                rule_match = mdc.host_scan['Rule']
                assert rule_match[0]['source'] == MATCHING_ENTITY_FILE
                assert rule_match[0]['string_offset'] == 21
                metadata = rule_match[0]['metadata']
                assert metadata['source_type'] == 'file'
                assert metadata['file_type'] == 'ASCII text'
                assert metadata['mime_type'] == 'text/plain; charset=us-ascii'
                assert metadata['md5sum'] == '9dd5c5e00d28520dc9da3c509c0db2a0'
                assert metadata['line_number'] == 1
                assert metadata['line'] == urlencode(
                    'This line contains a string match in the file "matching_entity"'
                )

                # Testing displaying long lines
                assert rule_match[1]['source'] == MATCHING_ENTITY_FILE
                assert rule_match[1]['string_offset'] == 83
                metadata = rule_match[1]['metadata']
                assert metadata['source_type'] == 'file'
                assert metadata['file_type'] == 'ASCII text'
                assert metadata['mime_type'] == 'text/plain; charset=us-ascii'
                assert metadata['md5sum'] == '9dd5c5e00d28520dc9da3c509c0db2a0'
                assert metadata['line_number'] == 2
                assert metadata['line'] == urlencode(
                    'This line contains another string match in matching_entity and it is very long for testing the ellipses that are added o...'
                )

                # Testing matching/displaying a mixture of quote types in the string_data
                assert rule_match[2]['source'] == MATCHING_ENTITY_FILE
                assert rule_match[2]['string_offset'] == 230
                metadata = rule_match[2]['metadata']
                assert metadata['source_type'] == 'file'
                assert metadata['file_type'] == 'ASCII text'
                assert metadata['mime_type'] == 'text/plain; charset=us-ascii'
                assert metadata['md5sum'] == '9dd5c5e00d28520dc9da3c509c0db2a0'
                assert metadata['line_number'] == 4
                assert metadata['line'] == urlencode(
                    """And this line contains a string with different types of quotes 'here' and "here" and its long too but not long enough"""
                )

                # Rule match metadata for ANOTHER_MATCHING_ENTITY_FILE
                # assert "[]' -e 'contains =' -e 'contains '\"'\"''" in caplog.text
                assert rule_match[3]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[3]['string_offset'] == 2
                metadata = rule_match[3]['metadata']
                assert metadata['source_type'] == 'file'
                assert metadata['file_type'] == 'ASCII text'
                assert metadata['mime_type'] == 'text/plain; charset=us-ascii'
                assert metadata['md5sum'] == '64764d295e92ffeec36d3fcd646a3af4'
                assert metadata['line_number'] == 3
                assert metadata['line'] == urlencode(
                    "string match containing error scanning but it's ok because its not in a rule line"
                )

                assert rule_match[4]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[4]['string_offset'] == 97
                metadata = rule_match[4]['metadata']
                assert metadata['md5sum'] == '64764d295e92ffeec36d3fcd646a3af4'
                assert metadata['line_number'] == 7
                assert metadata['line'] == urlencode("This line contains = char")

                assert rule_match[5]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[5]['string_offset'] == 123
                metadata = rule_match[5]['metadata']
                assert metadata['line_number'] == 8
                assert metadata['line'] == urlencode("This line contains = char too")

                assert rule_match[6]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[6]['string_offset'] == 153
                metadata = rule_match[6]['metadata']
                assert metadata['line_number'] == 9
                assert metadata['line'] == urlencode("This line contains .+ chars")

                assert rule_match[8]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[8]['string_offset'] == 213
                metadata = rule_match[8]['metadata']
                assert metadata['line_number'] == 11
                assert metadata['line'] == urlencode('This line contains "" chars')

                assert rule_match[9]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[9]['string_offset'] == 241
                metadata = rule_match[9]['metadata']
                assert metadata['line_number'] == 12
                assert metadata['line'] == urlencode("This line contains '' chars")

                assert rule_match[10]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[10]['string_offset'] == 269
                metadata = rule_match[10]['metadata']
                assert metadata['line_number'] == 13
                assert metadata['line'] == urlencode("This line contains ()[] chars")

                assert rule_match[11]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[11]['string_offset'] == 299
                metadata = rule_match[11]['metadata']
                assert metadata['line_number'] == 14
                assert metadata['line'] == urlencode("This line contains {} chars")

                assert rule_match[12]['source'] == ANOTHER_MATCHING_ENTITY_FILE
                assert rule_match[12]['string_offset'] == 327
                metadata = rule_match[12]['metadata']
                assert metadata['line_number'] == 15
                assert metadata['line'] == urlencode("This line contains ^$ chars")

                # Testing a missing file - expect minimal metadata because we can't know the other values
                assert (
                    rule_match[13]['source']
                    == "matching_entity_3, but without any string matches - yes that's ok"
                )
                metadata = rule_match[13]['metadata']
                assert metadata['source_type'] == 'file'
                assert all(
                    [key not in ['file_type', 'md5sum', 'line_number'] for key in metadata.keys()]
                )

                # Testing a missing file for another rule - again, expect minimal metadata because we can't find out more info
                rule_match = mdc.host_scan['another_matching_rule']
                assert rule_match[2]['source'].endswith(
                    'snap/signal-desktop/350/opt/Signal/resources/app.asar'
                )
                metadata = rule_match[2]['metadata']
                assert metadata['source_type'] == 'file'
                assert all(
                    [key not in ['file_type', 'md5sum', 'line_number'] for key in metadata.keys()]
                )

                # Testing a missing process - again, expect minimal metadata because we can't find out more info
                assert rule_match[3]['source'] == '1234567'
                metadata = rule_match[3]['metadata']
                assert metadata['source_type'] == 'process'
                assert all(
                    [
                        key not in ['process_name', 'file_type', 'md5sum', 'line_number']
                        for key in metadata.keys()
                    ]
                )

            def test_contrived_scan_output_with_disabled_rules(
                self, disabled, rules, create_test_files_real_yara, caplog
            ):
                # Parse the CONTRIVED_SCAN_OUTPUT but with a number of disabled rules
                # Disable 3 of the rules in CONTRIVED_SCAN_OUTPUT, expect 2 matching rules
                logger.setLevel('DEBUG')
                disabled.return_value = ['this', 'rule', 'another_matching_rule']
                mdc = MalwareDetectionClient(None)
                assert mdc.disabled_rules == ['this', 'rule', 'another_matching_rule']
                mdc.parse_scan_output(CONTRIVED_SCAN_OUTPUT)
                assert mdc.matches == 2
                assert all(
                    list(
                        map(
                            lambda x: x not in ['this', 'Rule', 'another_matching_rule'],
                            mdc.host_scan,
                        )
                    )
                )
                assert all(list(map(lambda x: x in ['Iyamtho', 'n_m3_t00'], mdc.host_scan)))
                assert 'Skipping matches for disabled rule this in file' in caplog.text
                assert 'Skipping matches for disabled rule Rule in file' in caplog.text
                assert (
                    'Skipping matches for disabled rule another_matching_rule in file'
                    in caplog.text
                )
                assert 'Matched rule Iyamtho in file' in caplog.text
                assert 'Matched rule n_m3_t00 in file' in caplog.text

                # disable all rules except 'Rule', because it isn't excluded
                caplog.clear()
                disabled.return_value = [
                    'this',
                    'rool',
                    'another_matching_rule',
                    'iyamtho',
                    'n_m3_t00',
                ]
                mdc = MalwareDetectionClient(None)
                mdc.parse_scan_output(CONTRIVED_SCAN_OUTPUT)
                assert list(mdc.host_scan) == ['Rule']
                assert mdc.matches == 3
                sources = set(map(lambda x: x['source'], mdc.host_scan['Rule']))
                assert len(sources) == 3
                assert 'Skipping matches for disabled rule Rule in file' not in caplog.text
                assert 'Matched rule Rule in file' in caplog.text

            def test_error_scan_output(self, disabled, rules, create_test_files_real_yara, caplog):
                logger.setLevel('DEBUG')
                mdc = MalwareDetectionClient(None)
                mdc.parse_scan_output(ERROR_SCAN_OUTPUT)

                assert mdc.matches == 0
                assert mdc.host_scan == {}
                assert (
                    'error scanning /var/lib/snapd//snap/core/10859/dev/core: could not open file'
                    in caplog.text
                )

            def test_error4_scan_output(self, disabled, rules, create_test_files_real_yara, caplog):
                logger.setLevel('DEBUG')
                mdc = MalwareDetectionClient(None)
                mdc.parse_scan_output(ERROR4_SCAN_OUTPUT)

                assert mdc.matches == 0
                assert mdc.host_scan == {}
                assert (
                    'error scanning /var/lib/snapd/snap/core/10859/dev/core: error: 4'
                    in caplog.text
                )

            def test_random_output(self, disabled, rules, create_test_files_real_yara):
                mdc = MalwareDetectionClient(None)
                mdc.parse_scan_output(RANDOM_OUTPUT)

                assert mdc.matches == 2
                rule_match = mdc.host_scan['Lorem']
                assert rule_match[0]['source'].startswith('ipsum dolor')
                assert rule_match[0]['string_data'] == ''
                assert rule_match[0]['string_identifier'] == ''
                assert rule_match[0]['string_offset'] == -1
                rule_match = mdc.host_scan['Dictum']
                assert rule_match[0]['source'].startswith('at tempor')
                assert rule_match[0]['string_data'] == ''
                assert rule_match[0]['string_identifier'] == ''
                assert rule_match[0]['string_offset'] == -1

            def test_random_output_with_disabled_rules(
                self, disabled, rules, create_test_files_real_yara, caplog
            ):
                # Test various disabled rules with parsing RANDOM_OUTPUT
                # Disable 'Lorem' rule (and others) and expect just to match the 'Dictum' rule
                logger.setLevel('DEBUG')
                disabled.return_value = ['random', 'rules', 'and', 'lorem']
                mdc = MalwareDetectionClient(None)
                mdc.parse_scan_output(RANDOM_OUTPUT)
                assert mdc.matches == 1
                assert list(mdc.host_scan) == ['Dictum']
                assert 'Skipping matches for disabled rule Lorem in file' in caplog.text

                # Disable both 'Lorem' and 'Dictum' rules and expect to have no matches
                caplog.clear()
                disabled.return_value = ['dictum', 'lorem']
                mdc = MalwareDetectionClient(None)
                mdc.parse_scan_output(RANDOM_OUTPUT)
                assert mdc.matches == 0
                assert mdc.host_scan == {}
                assert 'Skipping matches for disabled rule Dictum in file' in caplog.text
                assert 'Skipping matches for disabled rule Lorem in file' in caplog.text

                # Disable rules other than 'Lorem' and 'Dictum' and expect match both those rules
                caplog.clear()
                disabled.return_value = ['dictumm', 'lore', 'and', 'other', 'rules']
                mdc = MalwareDetectionClient(None)
                mdc.parse_scan_output(RANDOM_OUTPUT)
                assert mdc.matches == 2
                assert sorted(mdc.host_scan) == ['Dictum', 'Lorem']
                assert 'Skipping matches for disabled rule Dictum in file' not in caplog.text
                assert 'Skipping matches for disabled rule Lorem in file' not in caplog.text
                assert 'Matched rule Dictum in file' in caplog.text
                assert 'Matched rule Lorem in file' in caplog.text

        @patch(CONFIG_FILE_TARGET, TEMP_CONFIG_FILE)
        class TestFilesystemIncludeExcludeProcessing:

            @patch(GET_RULES_TARGET, return_value=COMPILED_RULES_FILE)
            @patch(DISABLED_RULES_TARGET)
            @patch.dict(os.environ, {'TEST_SCAN': 'false'})
            def test_process_include_exclude_items_simple(
                self, disabled, rules, create_test_files_real_yara, caplog
            ):
                # Test the process_include_exclude_items function with simple modified include and exclude items
                # Simple in that the include and exclude files are modified in such a way that
                # directory listings aren't required get the list of included files
                # Add a single toplevel directory to the include file - expect only a single directory to scan
                logger.setLevel('INFO')
                mdc = MalwareDetectionClient(None)
                mdc.scan_fsobjects = ['/etc']
                mdc.filesystem_scan_exclude_list = []
                scan_dict = process_include_exclude_items(
                    include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
                )
                assert list(scan_dict.keys()) == ['/etc']
                assert 'include' not in scan_dict['/etc']
                assert 'exclude' not in scan_dict['/etc']

                # Add some extra subdirectories to scan
                mdc.scan_fsobjects.extend(['/var/lib', '/var/log'])
                scan_dict = process_include_exclude_items(
                    include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
                )
                assert sorted(scan_dict.keys()) == ['/etc', '/var']
                assert sorted(list(scan_dict['/var']['include'])) == ['/var/lib', '/var/log']
                assert 'exclude' not in scan_dict['/var']

                # Add some extra directories to exclude that won't impact the already included directories
                mdc.filesystem_scan_exclude_list.extend(['/tmp', '/var/run'])
                scan_dict = process_include_exclude_items(
                    include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
                )
                assert sorted(scan_dict.keys()) == ['/etc', '/var']
                assert sorted(scan_dict['/var']['include']) == ['/var/lib', '/var/log']
                assert scan_dict['/var']['exclude']['items'] == ['/var/run']

                # Exclude /var which will remove it from the list of directories to scan
                mdc.filesystem_scan_exclude_list.append('/var')
                scan_dict = process_include_exclude_items(
                    include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
                )
                assert list(scan_dict.keys()) == ['/etc']

                # Exclude /etc which means there will be no directories to scan
                caplog.clear()
                mdc.filesystem_scan_exclude_list.append('/etc')
                process_include_exclude_items(
                    include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
                )
                assert (
                    "No filesystem items to scan because the specified exclude items cancel them out"
                    in caplog.text
                )
                assert list(scan_dict.keys()) == ['/etc']

            @patch(GET_RULES_TARGET, return_value=COMPILED_RULES_FILE)
            @patch(DISABLED_RULES_TARGET)
            @patch.dict(os.environ, {'TEST_SCAN': 'false'})
            def test_process_include_exclude_files_complex(
                self, disabled, rules, create_test_files_real_yara
            ):
                # Test the process function with modified include and exclude files that will require more complex
                # processing to generate the list of items to be scanned
                # Because we are including items in /var/lib, we only need to list the contents of the /var/lib directory
                # We don't need to list the contents of the /var directory
                mdc = MalwareDetectionClient(None)
                mdc.scan_fsobjects = ['/var/lib', '/var/log']
                mdc.filesystem_scan_exclude_list = [
                    '/var/lib/systemd',
                    '/var/lib/misc/',
                    '/var/log/wtmp',
                ]

                scan_dict = process_include_exclude_items(
                    include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
                )
                assert list(scan_dict.keys()) == ['/var']
                assert sorted(scan_dict['/var']['exclude']['items']) == [
                    '/var/lib/misc',
                    '/var/lib/systemd',
                    '/var/log/wtmp',
                ]
                # The exclude items shouldn't be in the include items
                # Nor should other items that aren't in the explicitly included items
                assert all(
                    [
                        x not in scan_dict['/var']['include']
                        for x in [
                            '/var/lib/misc',
                            '/var/lib/systemd',
                            '/var/log/wtmp',
                            '/var/cache',
                            '/var/lib',
                            '/var/log',
                            '/var/tmp',
                            '/tmp',
                        ]
                    ]
                )
                # In 'include' will be items that are in the same directory as the excluded items, eg /var/log/lastlog
                # but not the excluded items, eg /var/log/wtmp
                # Some of these directories may not exist on the test system, but if they do they will be included
                maybe_dirs = list(
                    filter(
                        lambda path: os.path.exists(path),
                        ['/var/lib/dbus', '/var/lib/pam', '/var/lib/rpm', '/var/log/lastlog'],
                    )
                )
                assert all([x in scan_dict['/var']['include'] for x in maybe_dirs])

                # Change the include directory to /var
                # Now immediate child directories of /var will be in the include list, eg /var/cache and /var/tmp
                # Because now we have to list the contents of the /var and /var/lib directories
                mdc.scan_fsobjects.append('/var')
                scan_dict = process_include_exclude_items(
                    include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
                )
                assert list(scan_dict.keys()) == ['/var']
                assert sorted(scan_dict['/var']['exclude']['items']) == [
                    '/var/lib/misc',
                    '/var/lib/systemd',
                    '/var/log/wtmp',
                ]
                assert all(
                    [
                        x not in scan_dict['/var']['include']
                        for x in [
                            '/var/lib/misc',
                            '/var/lib/systemd',
                            '/var/log/wtmp',
                            '/var/lib',
                            '/var/log',
                            '/tmp',
                        ]
                    ]
                )
                # Some of these directories may not exist on the test system, but if they do they will be included
                maybe_dirs = list(
                    filter(
                        lambda path: os.path.exists(path),
                        [
                            '/var/cache',
                            '/var/tmp',
                            '/var/lib/dbus',
                            '/var/lib/pam',
                            '/var/lib/rpm',
                            '/var/log/lastlog',
                        ],
                    )
                )
                assert all([x in scan_dict['/var']['include'] for x in maybe_dirs])

            @pytest.mark.skipif(
                not os.path.exists('/usr/local/libexec'), reason="No /usr/local/libexec"
            )
            @patch(GET_RULES_TARGET, return_value=COMPILED_RULES_FILE)
            @patch(DISABLED_RULES_TARGET)
            @patch.dict(os.environ, {'TEST_SCAN': 'false'})
            def test_process_include_exclude_files_similar_names(
                self, disabled, rules, create_test_files_real_yara
            ):
                # Now test including/excluding items that have similar names, eg /usr/local/lib and /usr/local/libexec
                # /usr/local has sub directories /usr/local/lib and /usr/local/libexec (ie similar names)
                # If we exclude /usr/local/lib then /usr/local/libexec should still be included
                mdc = MalwareDetectionClient(None)
                mdc.scan_fsobjects = ['/usr/local']
                mdc.filesystem_scan_exclude_list.append('/usr/local/lib')

                scan_dict = process_include_exclude_items(
                    include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
                )
                assert list(scan_dict.keys()) == ['/usr']
                assert list(scan_dict['/usr']['exclude']['items']) == ['/usr/local/lib']
                # Ensure /usr/local/lib is NOT in the list of items to scan
                assert all(
                    [
                        x not in scan_dict['/usr']['include']
                        for x in ['/usr', '/usr/lib', '/usr/local', '/usr/local/lib']
                    ]
                )
                # But ensure /usr/local/libexec IS in the list of items to scan
                assert all(
                    [
                        x in scan_dict['/usr']['include']
                        for x in ['/usr/local/bin', '/usr/local/share', '/usr/local/libexec']
                    ]
                )

                # Add /usr/local/libexec as an item to exclude and ensure both /usr/local/lib and libexec are excluded now
                mdc.filesystem_scan_exclude_list.append('/usr/local/libexec')
                scan_dict = process_include_exclude_items(
                    include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
                )
                assert list(scan_dict.keys()) == ['/usr']
                assert sorted(scan_dict['/usr']['exclude']['items']) == [
                    '/usr/local/lib',
                    '/usr/local/libexec',
                ]
                # Ensure /usr/local/lib and /usr/local/libexec are both not in the list of items to scan
                assert all(
                    [
                        x not in scan_dict['/usr']['include']
                        for x in [
                            '/usr',
                            '/usr/lib',
                            '/usr/local',
                            '/usr/local/lib',
                            '/usr/local/libexec',
                        ]
                    ]
                )
                assert all(
                    [
                        x in scan_dict['/usr']['include']
                        for x in ['/usr/local/bin', '/usr/local/share']
                    ]
                )

                # Test including /usr/local/lib and excluding an item from it
                # Confirm that only items from /usr/local/lib are included and NOT from any other directory
                usr_local_lib = sorted(
                    filter(
                        lambda x: not os.path.islink(x),
                        map(lambda x: '/usr/local/lib/' + x, os.listdir('/usr/local/lib')),
                    )
                )
                assert len(usr_local_lib) > 2
                excluded_item1 = usr_local_lib[0]
                excluded_item2 = usr_local_lib[1]
                mdc.scan_fsobjects = ['/usr/local/lib']
                mdc.filesystem_scan_exclude_list = [excluded_item1, excluded_item2]
                scan_dict = process_include_exclude_items(
                    include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
                )
                assert list(scan_dict.keys()) == ['/usr']
                assert sorted(scan_dict['/usr']['exclude']['items']) == [
                    excluded_item1,
                    excluded_item2,
                ]
                # Ensure only /usr/local/lib items are included (except the first 2 in the directory)
                assert all(
                    [
                        x not in scan_dict['/usr']['include']
                        for x in [
                            '/usr',
                            '/usr/local',
                            '/usr/local/lib',
                            '/usr/local/libexec',
                            excluded_item1,
                            excluded_item2,
                        ]
                    ]
                )
                assert all([x in scan_dict['/usr']['include'] for x in usr_local_lib[2:]])

            @patch(GET_RULES_TARGET, return_value=RULE_RULE_FILE)
            @patch(DISABLED_RULES_TARGET)
            def test_scan_tmp_files(
                self, disabled, rules, create_test_files_real_yara, extract_tmp_files, caplog
            ):
                # Scan the files in the tmp file and set scan_only, scan_exclude to same values as above
                logger.setLevel('INFO')
                scan_me_file = os.path.join(TEMP_TEST_DIR, 'scan_me/scan_me_file')
                scan_me_too_file = os.path.join(TEMP_TEST_DIR, 'scan_me_too/scan_me_too_file')
                scan_only = tuple(
                    map(lambda x: os.path.join(TEMP_TEST_DIR, x), ['scan_me', 'scan_me_too'])
                )
                scan_exclude = tuple(
                    map(
                        lambda x: os.path.join(TEMP_TEST_DIR, x),
                        ['scan_me_not', 'scan_me/dont_scan_me', 'scan_me_too/dont_scan_me_too'],
                    )
                )
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    line = (
                        line + '- %s\n- %s' % scan_only
                        if line.startswith("filesystem_scan_only:")
                        else line
                    )
                    line = (
                        line + "- %s\n- %s\n- %s" % scan_exclude
                        if line.startswith("filesystem_scan_exclude:")
                        else line
                    )
                    print(line)

                mdc = MalwareDetectionClient(None)
                scan_dict = process_include_exclude_items(
                    include_items=mdc.scan_fsobjects, exclude_items=mdc.filesystem_scan_exclude_list
                )
                assert list(scan_dict.keys()) == ['/tmp']
                assert sorted(list(scan_dict['/tmp']['exclude']['items'])) == sorted(scan_exclude)

                mdc.scan_filesystem()
                assert mdc.matches == 2  # There were only 2 files the matched the rule
                rule_match = mdc.host_scan['Rule']
                assert len(rule_match) == 11  # There were 11 strings matched in the 2 files
                # Asserting the names of the 2 files that were matched
                sources = set([rm['source'] for rm in rule_match])
                assert len(sources) == 2
                assert scan_me_file in sources
                assert scan_me_too_file in sources

                # With the same scan_only and scan_exclude values, add scan_since: last into the mix and set the last scan
                # time to now.  There should be no matches because no files have been modified since now
                caplog.clear()
                last_scan = time.time()
                last_scan_fmt = datetime.fromtimestamp(last_scan).strftime('%Y-%m-%d %H:%M:%S')
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = (
                        "filesystem_scan_since: last"
                        if line.startswith("filesystem_scan_since:")
                        else line
                    )
                    print(line)
                with patch(
                    "insights.specs.datasources.malware_detection.get_scan_since_timestamp",
                    return_value=last_scan,
                ):
                    mdc = MalwareDetectionClient(None)
                assert mdc.filesystem_scan_since_dict['timestamp'] == last_scan
                assert mdc.filesystem_scan_since_dict['datetime'] == last_scan_fmt
                assert (
                    "Scan for files created/modified since last successful scan on %s"
                    % last_scan_fmt
                    in caplog.text
                )
                mdc.scan_filesystem()
                assert mdc.matches == 0  # There were no files modified since 'last_scan'

                # Try again, keeping the same last_scan time, but this time touch one of the matching files
                # Should get a match on the touched file because it has been 'modified' since the last scan
                caplog.clear()
                os.system('touch %s' % scan_me_file)
                with patch(
                    "insights.specs.datasources.malware_detection.get_scan_since_timestamp",
                    return_value=last_scan,
                ):
                    mdc = MalwareDetectionClient(None)
                mdc.scan_filesystem()
                assert mdc.matches == 1  # There was 1 file modified since last_scan
                rule_match = mdc.host_scan['Rule']
                assert len(rule_match) == 8  # There were 8 strings matched in the 2 files
                assert rule_match[0]['source'] == scan_me_file

                # Touch another of the matching files, keeping the same last_scan time.
                # Expect to get 2 matching files this time
                caplog.clear()
                os.system('touch %s' % scan_me_too_file)
                with patch(
                    "insights.specs.datasources.malware_detection.get_scan_since_timestamp",
                    return_value=last_scan,
                ):
                    mdc = MalwareDetectionClient(None)
                mdc.scan_filesystem()
                assert (
                    mdc.matches == 2
                )  # There were 2 files modified since last_scan the matched the rule
                rule_match = mdc.host_scan['Rule']
                assert len(rule_match) == 11  # There were 11 strings matched in the 2 files
                # Asserting the names of the 2 files that were matched
                sources = set([rm['source'] for rm in rule_match])
                assert len(sources) == 2
                assert scan_me_file in sources
                assert scan_me_too_file in sources

        @patch(NAMEDTMPFILE_TARGET)
        @patch.object(
            InsightsConnection, 'get', return_value=Mock(status_code=200, content=b"Rule Content")
        )
        @patch.object(InsightsConnection, 'get_proxies')
        @patch.object(InsightsConnection, '_init_session', return_value=Mock())
        @patch(DISABLED_RULES_TARGET, return_value=[])
        @patch(CONFIG_FILE_TARGET, TEMP_CONFIG_FILE)
        class TestGetRules:

            @patch.dict(os.environ)
            def test_get_regular_rules_location_urls(
                self, disabled, init_session, get_proxies, get, tmpfile, caplog
            ):
                # Test the standard remote_rules_location urls, depending on whether test rule or cert auth is set
                # With test scan true, expect to download test-rule.yar
                test_rule_url = (
                    "https://cert.console.redhat.com/api/malware-detection/v1/test-rule.yar"
                )
                signatures_url = (
                    "https://cert.console.redhat.com/api/malware-detection/v1/signatures.yar?yara_version=%s"
                    % REAL_YARA_VERSION
                )
                logger.setLevel('DEBUG')
                os.environ['TEST_SCAN'] = 'true'
                with pytest.raises(SystemExit):
                    MalwareDetectionClient(InsightsConfig())
                get.assert_called_with(
                    test_rule_url, log_response_text=True, verify=True, stream=True
                )
                assert test_rule_url in caplog.text

                caplog.clear()
                with pytest.raises(SystemExit):
                    MalwareDetectionClient(InsightsConfig(authmethod='CERT'))
                get.assert_called_with(
                    test_rule_url, log_response_text=True, verify=True, stream=True
                )
                assert test_rule_url in caplog.text

                # With authmethod=BASIC and test scan false ...
                # Expect to still use cert auth because no username or password specified
                caplog.clear()
                os.environ['TEST_SCAN'] = 'false'
                with pytest.raises(SystemExit):
                    MalwareDetectionClient(InsightsConfig(authmethod='BASIC'))
                get.assert_called_with(
                    signatures_url, log_response_text=False, verify=True, stream=True
                )
                assert signatures_url in caplog.text

                caplog.clear()
                with pytest.raises(SystemExit):
                    MalwareDetectionClient(InsightsConfig(authmethod='CERT'))
                get.assert_called_with(
                    signatures_url, log_response_text=False, verify=True, stream=True
                )
                assert signatures_url in caplog.text

            def test_get_irregular_rules_location_urls(
                self,
                disabled,
                init_session,
                get_proxies,
                get,
                tmpfile,
                create_test_files_real_yara,
                caplog,
            ):
                # Non-standard rules URLs
                # Without https:// at the start and not signatures.yar
                logger.setLevel('DEBUG')
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = (
                        line + "remote_rules_location: console.redhat.com/rules.yar"
                        if line.startswith("---")
                        else line
                    )
                    print(line)
                with pytest.raises(SystemExit):
                    MalwareDetectionClient(InsightsConfig())
                get.assert_called_with(
                    "https://cert.console.redhat.com/test-rule.yar",
                    log_response_text=True,
                    verify=True,
                    stream=True,
                )
                assert "https://cert.console.redhat.com/test-rule.yar" in caplog.text

                caplog.clear()
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    print(line)
                with pytest.raises(SystemExit):
                    MalwareDetectionClient(InsightsConfig(authmethod='CERT'))
                get.assert_called_with(
                    "https://cert.console.redhat.com/rules.yar",
                    log_response_text=False,
                    verify=True,
                    stream=True,
                )
                assert "https://cert.console.redhat.com/rules.yar" in caplog.text

            def test_get_rules_location_files(
                self,
                disabled,
                init_session,
                get_proxies,
                get,
                tmpfile,
                create_test_files_real_yara,
                caplog,
            ):
                # Test using files for remote_rules_location, esp irregular file names
                # Re-writing the rule to be test-rule.yar doesn't apply to local files
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = (
                        line + "remote_rules_location: //console.redhat.com/rules.yar"
                        if line.startswith("---")
                        else line
                    )
                    print(line)
                with pytest.raises(SystemExit):
                    MalwareDetectionClient(None)
                assert (
                    "Couldn't find specified rules file: /console.redhat.com/rules.yar"
                    in caplog.text
                )

                caplog.clear()
                for line in fileinput.FileInput(TEMP_CONFIG_FILE, inplace=1):
                    line = "test_scan: false" if line.startswith("test_scan:") else line
                    print(line)
                with pytest.raises(SystemExit):
                    MalwareDetectionClient(None)
                assert (
                    "Couldn't find specified rules file: /console.redhat.com/rules.yar"
                    in caplog.text
                )


MATCHING_ENTITY_FILE_CONTENTS = """
This line contains a string match in the file "matching_entity"
This line contains another string match in matching_entity and it is very long for testing the ellipses that are added onto very long lines

And this line contains a string with different types of quotes 'here' and "here" and its long too but not long enough
""".lstrip()

ANOTHER_MATCHING_ENTITY_FILE_CONTENTS = """


string match containing error scanning but it's ok because its not in a rule line



This line contains = char
This line contains = char too
This line contains .+ chars
This line contains .+ chars too
This line contains "" chars
This line contains '' chars
This line contains ()[] chars
This line contains {} chars
This line contains ^$ chars
This line contains ^$ chars too
The previous line and this one too are ignored as they are beyond the default 10 string match limit
"""[
    1:
]  # Ignore first newline char otherwise it changes the md5sum

TEST_RULE_FILE_CONTENTS = """
rule TEST_RedHatInsightsMalwareDetection
// Verifies the Red Hat Insights Malware Detection Client app is present on the system
{
    strings:
        $re1 = /Malware ?Detection ?Client/
    condition:
        $re1
}
""".lstrip()

TEST_RULE_SCRIPT_CONTENTS = """
#!/bin/sh
# As a process this will match the TEST_RedHatInsightsMalwareDetection rule
echo "Malware Detection Client"
sleep 3
""".lstrip()

CONTRIVED_SCAN_OUTPUT = """
error scanning this line, it will be skipped
this line also contains error scanning so it will be skipped too
0x1:$string1: skip me coz the rule line had an error
0x11:$string2: skip me too
0x111:$string3: skip me three
this [] line doesn't contain e-r-r-o-r s-c-a-n-n-i-n-g so will be considered a scan match, even though it shouldn't
0x123:$match: matches 'this' rule
BadFormat
0x1:$skipme: the previous line will fail because it doesn't follow the 'rule matching_entity' format
Rule:matching_entity
0x1:$alsoskipme: the previous line also fails because it has a ":" instead of a " " between rule & matching_entity
Rule [] %s
0x15:$match0: string match in the file "matching_entity"
0x53:$match1: another string match in matching_entity
0xe6:$match2: string with different types of quotes 'here' and "here"
Rule [] %s
0x2:$match3: string match containing error scanning but it's ok because its not in a rule line
0x61:$grep1: contains =
0x7b:$grep1: contains =
0x99:$grep2: contains .+
0xb5:$grep2: contains .+
0xd5:$grep3: contains "
0xf1:$grep4: contains '
0x10d:$grep5: contains ()[]
0x12b:$grep6: contains {
0x147:$grep7: contains ^$
0x163:$grep7: contains ^$
0x179:$ignored: The previous line and this one too are ignored as they are beyond the default 10 string match limit
Rule [] matching_entity_3, but without any string matches - yes that's ok
more error scanning this line
another_matching_rule [] /var/lib/snapd/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/signal-desktop/350/opt/Signal/resources/app.asar
0x212f197:$s0: #!/bin/sh
0x2130313:$s0: #!/bin/sh
0x39f7cc6:$s0: #!/bin/sh
another_matching_rule [] /var/lib/snapd/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859error scanning /dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/signal-desktop/350/opt/Signal/resources/app2.asar
0x212f197:$s0: #!/bin/sh
0x2130313:$s0: #!/bin/sh
0x39f7cc6:$s0: #!/bin/sh
another_matching_rule [] 1234567
0x0:$s0: #!/bin/sh
0x1badoffset:$s1: skip this line
0x2error scanning skip/this/line/too: need more colons
badoffset_but_notarule:$s2: a bad offset line that looks a bit like a rule line but really isn't
007 isn't considered a valid rule name
666neither is this
_me neither
 same here
nor: I
Iyamtho [] yep
n_m3_t00 [] damn   straight
error scanning /var/lib/snapd/snap/core/10859/dev/core: error: 4
error scanning /var/lib/snapd/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/core: error: 4
error scanning /var/lib/snapd//cookie/snap.gnome-3-28-1804: could not open file
error scanning /var/lib/snapd//device/private-keys-v1/_53ir43FCxbgdSyj8NriGt9gfonABhzHHhsGnGhvjqpK_hwdIcP0ScJpKppzEhps: could not open file
""" % (
    MATCHING_ENTITY_FILE,
    ANOTHER_MATCHING_ENTITY_FILE,
)

RANDOM_OUTPUT = """
Lorem [] ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
Dictum [] at tempor commodo ullamcorper a lacus vestibulum sed. Non odio euismod lacinia at quis risus sed.
"""

# Base64 representation of a tgz file containing simple files to be extracted into /tmp to be used for scanning
# It is decoded and unzipped in the extract_tmp_files fixture
SCAN_FILES_BASE64 = """
H4sIAD+feGEAA+1a247TMBDtc79iVCEVhARxrk888A+8IajSxt1Ym8YlcXZVIf6dcbZG2WzTkm5t
ijpnH3KxHbt7PPaZGdertFxs+MeJRXiIJIn0lSWR170aTFjIgpixJAyCicdY5IcTiGwOyqCpVVoB
TDZ5o/Ij9U6V/6eo9/yX/HGxFgW30YcmOI7DEfwHvh9NwLMxmD6I/5Z/WWRXxT/ziX8XMPxvxZbb
mgBn8B8kyQRiC2N5AeK/5T+TpVrUdsTA+P0/ZFFE+78LHOR/k6pVLsq7BS+VULvX9nHK/iMv6PGf
+Fid1n8H+JKLGgpRcljhDEhFWUMKtaqQfWinAYgSVM5B7w0w682M2fRQ+1Jig+rFV3ptsV4GQgG2
f+DVDgqJldeyAsVrpRvqXnlRiG3Na3xIFaQVhzTLeAbYl+w00wOop9PP+EV15Ac9CpVDJtZrXuEY
QO22+GW5hh+NxE5hjqPm83ZcM3072w+xfupESQnLRgH+vKcXvJTNXT791xS+Csbkl5W85+UC/3H3
F+/jaf1PRqz/MYtw//dtDsrgxu3f8G9p628xfv8P8I/2fxfo86/XvYVYpJsL9jFe/4dJRP6fE/T4
t+ICtvyHw/wnQdTn34ti4t8FptNnMm2vmfQLXlUoxvS8aB+18hFqjnrpHpZ8lTY1b6WRVkMo7lKo
GhSIWnhNEQdk4SdY5Wk1XKL11aHSD+/b4vpY2VDb2Wy47Xw+XPb23ddvw6U/fw2XfX9zumw/Wg7b
ij8I2ezrpUa9SnzQYlPLXXFXygoFb6olMN+175Z8J9u6HDK+TptCAfOey+1CbIT6G2Vq7F9rLJvx
n1H6z2e4AqD+s+mUGty4/Zt1H83YWg5ovP7zQ4/yP07Q5d9WDugM/cc8yv84QZd/WzmgM/j3EtL/
TtDl31YO6Az+/YjyP07Q5f/PPX/g1QXVwPj9P/KZR/u/Cwzzb8L4r9fdp+z/hf/v4zWh9d8FyP8n
/9/YvyUP+4z8bxSxgOzfBSj/S/lfY/+2YoCj438sCWKd/7W1IHVx4/Zv+MfJfV3xPzr/5QRd/rvh
9ktOiDP8v4QlxL8LHOX/Qi7geP/PDyn/6wbk/5H/Z8z9qvI/ZP9O0OX/qvI/5P87QZf/a8r/xB7l
f1ygy7+t49Zn+P9ee/6Hzn9bR5d/W7GA8f5fGOv1n/w/+xjg/6LbQLv+Hzn/eyD+j7e0/7sAxf8p
/m/s/2ri/34Y+EHv/O8lA1Jd3Lj9EwgEAoFAIBBuC78BaSEregBQAAA=""".replace(
    '\n', ''
)

TEST_RULE1 = b"""
rule MalwareDetectionClientRule
{
    strings:
        $text1 = "MalwareDetectionClient" fullword

    condition:
        $text1
}
"""

TEST_RULE2 = b"""
rule MiscellaneousStringsRule
// sent""
// ata_sff\\\\x00bioset\\\\x00bond0\\\\x00cifsd\\\\x00
{
    strings:
        $string1 = "sent\\"" fullword
        $string2 = "ata_sff\\\\x00bioset\\\\x00bond0\\\\x00cifsd\\\\x00"

    condition:
         any of them
}
"""

RULE_RULE_FILE_CONTENTS = r"""
rule Rule
/*
   Strings to trigger matches in the tests/matching_entity and tests/another\\ matching_entity files
   Output from this rule against those files should look like CONTRIVED_SCAN_OUTPUT in tests/test_parse_scan_output.py
   This file is also used in tests for testing rules files with spaces in them
*/
{
    meta:
        description = "Strings to trigger matches in the test/*matching_entity files"

    strings:
        $match0 = "string match in the file \\"matching_entity\\""
        $match1 = "another string match in matching_entity"
        $match2 = "string with different types of quotes 'here' and \\"here\\""
        $match3 = "string match containing error scanning but it's ok because its not in a rule line"

        $grep1 = "contains ="
        $grep2 = "contains .+"
        $grep3 = "contains \\""
        $grep4 = "contains '"
        $grep5 = "contains ()[]"
        $grep6 = "contains {"
        $grep7 = "contains ^$"

    condition:
        any of them
}
"""

RULE_METADATA_TEST_FILE_CONTENTS = r"""rule MetadataTestRule
/*
    // These rule strings will match the rule strings below unmodified ...
    Testing $s4 = "for ssh_creds in ${allThreads[@]}; do" ascii fullword
    Testing $s8 = "allThreads=($1)" ascii fullword
    Testing $s9 = "$(host): encrypt files. Done." ascii fullword

    // These rule strings with some modifications match the rule strings below ...
    Testing $s1 = "echo -e \"[-] Ping \\033[31m${host_name}\\033[0m bad" ascii fullword
    Testing $s2 = ""${user_name}"@"${host_name}" -p "${port}" ascii fullword
    Testing $s3 = "'$password' &" <<< GMANcode27'" ascii fullword
    Testing $s5 = ""text=$MSG" "$MSG_URL$id&"" ascii fullword
    Testing $s6 = "--exclude=\*.☢ -l" fullword
    Testing $s7 = "--include=\*.{txt,sh,exe}" ascii fullword
*/
{
    strings:
        $s1 = "echo -e \\"[-] Ping \\\\033[31m${host_name}\\\\033[0m bad\\"" ascii fullword
        $s2 = "\\"${user_name}\\"@\\"${host_name}\\" -p \\"${port}" ascii fullword
        $s3 = "'$password' &\\" <<< GMANcode27'" ascii fullword
        $s4 = "for ssh_creds in ${allThreads[@]}; do" ascii fullword
        $s5 = "\\"text=$MSG\\" \\"$MSG_URL$id&\\"" ascii fullword
        $s6 = "--exclude=\\\\*.☢ -l" fullword
        $s7 = "--include=\\\\*.{txt,sh,exe}" ascii fullword
        $s8 = "allThreads=($1)" ascii fullword
        $s9 = "$(host): encrypt files. Done." ascii fullword

    condition:
        any of them
}
"""

ERROR_SCAN_OUTPUT = """
error scanning /var/lib/snapd//snap/core/10859/dev/core: could not open file
error scanning /var/lib/snapd//snap/core/10859/dev/fd/3/cookie/snap.core: could not open file
error scanning /var/lib/snapd//cookie/snap.gnome-3-28-1804: could not open file
error scanning /var/lib/snapd//device/private-keys-v1/_53ir43FCxbgdSyj8NriGt9gfonABhzHHhsGnGhvjqpK_hwdIcP0ScJpKppzEhps: could not open file
"""

ERROR4_SCAN_OUTPUT = """
error scanning /var/lib/snapd/snap/core/10859/dev/core: error: 4
error scanning /var/lib/snapd/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/core: error: 4
error scanning /var/lib/snapd/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/core: error: 4
error scanning /var/lib/snapd/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/fd/3/snap/core/10859/dev/core: error: 4
"""
