import os
import re
import time
import json
import yaml
from sys import exit
import logging
from glob import glob
from datetime import datetime
from tempfile import NamedTemporaryFile
try:
    # python 2
    from urllib import quote as urlencode
    from urlparse import urlparse, urlunparse
except ImportError:
    # python 3
    from urllib.parse import urlparse, urlunparse, quote as urlencode

from insights.client.connection import InsightsConnection
from insights.client.constants import InsightsConstants as constants
from insights.client.utilities import (
    generate_machine_id, write_data_to_file, get_time
)
from insights.util.subproc import call, CalledProcessError

logger = logging.getLogger(__name__)
MIN_YARA_VERSION = "4.1.0"
MALWARE_APP_URL = 'https://console.redhat.com/insights/malware'
MALWARE_CONFIG_FILE = os.path.join(constants.default_conf_dir, "malware-detection-config.yml")
LAST_FILESYSTEM_SCAN_FILE = os.path.join(constants.default_conf_dir, '.last_malware-detection_filesystem_scan')
LAST_PROCESSES_SCAN_FILE = os.path.join(constants.default_conf_dir, '.last_malware-detection_processes_scan')
DEFAULT_MALWARE_CONFIG = """
# Configuration file for the Red Hat Insights Malware Detection Client app
# File format is YAML
---
# Perform a simple test scan of the insights-client config directory and process to verify installation and scanning
# are working correctly.  The results from this scan do not show up in the webUI.
# Once verified, disable this option to perform actual malware scans.
test_scan: true

# Scan the filesystem?
# When it is false, the filesystem isn't scanned and the filesystem_* options that follow are ignored
scan_filesystem: true

# filesystem_scan_only: a single or list of files/directories to be scanned and no others, for example:
# filesystem_scan_only:
# - /var/www
# - /home
# ... means only scan files in /var/www and /home.  May also be written as filesystem_scan_only: [/var/www, /home]
# No value means scan all files and directories
filesystem_scan_only:

# filesystem_scan_exclude: a single or list of files/directories to be excluded from filesystem scanning
# If an item appears in both filesystem_scan_only and filesystem_scan_exclude, filesystem_scan_exclude takes precedence
# and the item will be excluded
# filesystem_scan_exclude is pre-populated with a list of top level directories that are recommended to be excluded
filesystem_scan_exclude:
- /proc
- /sys
- /cgroup
- /selinux
- /net
- /mnt
- /media
- /dev

# filesystem_scan_since: scan files created or modified since X days ago or since the 'last' scan.
# Valid values are integers >= 1 or the string 'last'.  For example:
# filesystem_scan_since: 1
# ... means scan files created/modified since 1 day ago
# filesystem_scan_since: last
# ... means scan files created/modified since the last successful scan
# No value means scan all files regardless of created/modified date
filesystem_scan_since:

# Exclude mounted network/external filesystems mountpoints?
# Scanning files within mounted network filesystems may be slow and cause extra network traffic.
# They are excluded by default, meaning that files in network/externally mounted filesystems are not scanned.
# Their mountpoints will be added to the scan_exclude list of directories to be excluded from scanning
exclude_network_filesystem_mountpoints: true

# List of network/external filesystem types to search for mountpoints on the system.
# If any mountpoints are found for these filesystem types, the value of the exclude_network_filesystem_mountpoints
# option will determine if files within the mountpoints are scanned or not.
network_filesystem_types: [nfs, nfs4, cifs, smbfs, fuse.sshfs, ceph, glusterfs, gfs, gfs2]

# Scan the running processes?
# Disabled by default due to an existing issue in Yara that can impact system performance
# when scanning numerous or large processes.  Option will be enabled by default when the Yara fix is available.
# When it is false, no processes are scanned and the processes_scan_* options that follow are ignored
scan_processes: false

# processes_scan_only: processes to be scanned and no others, for example:
# processes_scan_only:
# - 123
# - 1..100
# - 10000..
# - docker
# - chrome
#... means only scan PID 123, PIDs from 1 to 100 inclusive, PIDs >= 10000 and process names containing the strings docker or chrome
# No value means scan all processes
processes_scan_only:

# processes_scan_exclude: processes to be excluded from scanning.  Uses the same syntax as processes_scan_only.
# If an item appears in both processes_scan_only and processes_scan_exclude, processes_scan_exclude takes precedence
# and the item will be excluded
# No value means don't exclude any processes
processes_scan_exclude:

# processes_scan_since: scan processes created since X days ago or since the 'last' scan.
# Valid values are integers >= 1 or the string 'last'.  For example:
# processes_scan_since: 1
# ... means scan processes created since 1 day ago
# processes_scan_since: last
# ... means scan processes created since the last successful scan
# No value means scan all processes regardless of created date
processes_scan_since:

# Add extra metadata about each scan match (if possible), eg file type & md5sum, matching line numbers, process name
# The extra metadata will display in the webUI along with the scan matches
add_metadata: true

# Abort a particular scan if it takes longer than scan_timeout seconds.  Default is 3600 seconds (1 hour)
scan_timeout: # 3600

# Run the yara process with this nice priority value.  Default is 19 (lowest priority)
nice_value: # 19

# The max number of CPUs threads used by yara when scanning.  Autodetected, but default is 2
cpu_thread_limit: # 2
""".lstrip()

# All the config options have corresponding environment variables
# Env vars are initially strings and need to be parsed to their appropriate type to match the yaml types
ENV_VAR_TYPES = {
    'boolean': ['SCAN_FILESYSTEM', 'SCAN_PROCESSES', 'TEST_SCAN', 'ADD_METADATA',
                'EXCLUDE_NETWORK_FILESYSTEM_MOUNTPOINTS'],
    'list': ['FILESYSTEM_SCAN_ONLY', 'FILESYSTEM_SCAN_EXCLUDE', 'PROCESSES_SCAN_ONLY', 'PROCESSES_SCAN_EXCLUDE',
             'NETWORK_FILESYSTEM_TYPES'],
    'integer': ['SCAN_TIMEOUT', 'NICE_VALUE', 'CPU_THREAD_LIMIT', 'STRING_MATCH_LIMIT'],
    'int_or_str': ['FILESYSTEM_SCAN_SINCE', 'PROCESSES_SCAN_SINCE']
}


class MalwareDetectionClient:
    def __init__(self, insights_config):
        # insights_config is used to access insights-client auth and network config when downloading rules
        self.insights_config = insights_config

        # Load the malware-detection config file
        self.config = self._load_config()

        # Early check if the yara binary exists.  No point continuing if it doesn't
        self.yara_binary = self._find_yara()

        # Get/set the values of assorted integer config values - mainly options used with the yara command
        for option, value in [('nice_value', 19),
                              ('scan_timeout', 3600),
                              ('cpu_thread_limit', 2),
                              ('string_match_limit', 10)]:
            try:
                setattr(self, option, int(self._get_config_option(option, value)))
            except Exception as e:
                logger.error("Problem setting configuration option %s: %s", option, str(e))
                exit(constants.sig_kill_bad)

        # If doing a test scan, then ignore the other scan_* options because test scan sets its own values for them
        if not self._parse_test_scan_option():
            self._parse_scan_options()

        # Obtain the rules to be used by yara
        self.rules_file = self._get_rules()

        # Build the yara command, with its various command line options, that will be run
        self.yara_cmd = self._build_yara_command()

        # host_scan is a dictionary into which all the scan matches are stored.  Its structure is like:
        # host_scan = {rule_name: [{source: ..., stringData: ..., stringIdentifier: ..., stringOffset: ...},
        #                          {source: ...}],
        #              rule_name: [{...}, {...}, {...}],
        #              ... }
        # host_scan_mutation is the host_scan dict converted to a GraphQL mutation query string
        self.host_scan = {}
        self.host_scan_mutation = ''

        # Check if we are adding extra metadata to each scan match
        self.add_metadata = self._get_config_option('add_metadata', False)

        self.matches = 0

    def run(self):
        # Start the scans and record the time they were started
        filesystem_scan_start = get_time()
        self.scan_filesystem()
        processes_scan_start = get_time()
        self.scan_processes()

        if self.do_filesystem_scan or self.do_process_scan:
            # If any scans were performed then get the results as a GraphQL mutation query
            # This mutation query is what is uploaded to the malware backend
            host_scan_mutation = self._create_host_scan_mutation()

            # Write a message to user informing them if there were matches or not and what to do next
            if self.matches == 0:
                logger.info("No rule matches found.\n")
            else:
                logger.info("Found %d rule match%s.", self.matches, 'es' if self.matches > 1 else '')
                if not self.test_scan:
                    logger.info("Please visit %s for more information\n", MALWARE_APP_URL)

            # Write the scan start times to disk if scans were performed
            # (used by the 'filesystem_scan_since: last' and 'processes_scan_since: last' options)
            # Only write the scan time after scans have completed without error or interruption, and its not a test scan
            if not self.test_scan:
                if self.do_filesystem_scan:
                    write_data_to_file(filesystem_scan_start, LAST_FILESYSTEM_SCAN_FILE)
                    os.chmod(LAST_FILESYSTEM_SCAN_FILE, 0o644)
                if self.do_process_scan:
                    write_data_to_file(processes_scan_start, LAST_PROCESSES_SCAN_FILE)
                    os.chmod(LAST_PROCESSES_SCAN_FILE, 0o644)
            else:
                logger.info("\nRed Hat Insights malware-detection app test scan complete.\n"
                            "Test scan results are not recorded in the Insights UI (%s)\n"
                            "To perform proper scans, please set test_scan: false in %s\n",
                            MALWARE_APP_URL, MALWARE_CONFIG_FILE)

            # This is what is uploaded to the malware backend
            return host_scan_mutation
        else:
            logger.error("No scans performed, no results to upload.")
            exit(constants.sig_kill_bad)

    @staticmethod
    def _load_config():
        # Load the malware-detection config file.  Write out a default one first if it doesn't already exist
        if not os.path.isfile(MALWARE_CONFIG_FILE):
            logger.info("Writing the malware-detection app default configuration to %s", MALWARE_CONFIG_FILE)
            write_data_to_file(DEFAULT_MALWARE_CONFIG, MALWARE_CONFIG_FILE)
            os.chmod(MALWARE_CONFIG_FILE, 0o644)

        try:
            with open(MALWARE_CONFIG_FILE) as m:
                return yaml.safe_load(m)
        except Exception as e:
            logger.error("Error encountered loading the malware-detection app config file %s:\n%s",
                         MALWARE_CONFIG_FILE, str(e))
            exit(constants.sig_kill_bad)

    def _find_yara(self):
        """
        Find the yara binary in particular locations on the local system.  Don't use 'which yara'
        and rely on the system path in case it finds a malicious yara.
        Also, don't let the user specify where yara is, again in case it is a malicious version of yara
        If found, check it's version >= MIN_YARA_VERSION
        """
        def yara_version_ok(yara):
            # Check the installed yara version >= MIN_YARA_VERSION
            installed_yara_version = call([[yara, '--version']]).strip()
            try:
                if float(installed_yara_version[:3]) < float(MIN_YARA_VERSION[:3]):
                    raise RuntimeError("Found %s with version %s, but malware-detection requires version >= %s\n"
                                       "Please install a later version of yara."
                                       % (yara, installed_yara_version, MIN_YARA_VERSION))
            except RuntimeError as e:
                logger.error(str(e))
                exit(constants.sig_kill_bad)
            except Exception as e:
                logger.error("Error getting the version of the specified yara binary %s: %s", yara, str(e))
                exit(constants.sig_kill_bad)
            # If we are here then the version of yara was ok
            return True

        # Try to find yara in only these usual locations.
        # /bin/yara and /usr/bin/yara will exist if yara is installed via rpm
        # /usr/local/bin/yara will (likely) exist if the user has compiled and installed yara manually
        for yara in ['/bin/yara', '/usr/bin/yara', '/usr/local/bin/yara']:
            if os.path.exists(yara) and yara_version_ok(yara):
                logger.debug("Using yara binary: %s", yara)
                return yara

        logger.error("Couldn't find yara.  Please ensure the yara package is installed")
        exit(constants.sig_kill_bad)

    def _parse_scan_options(self):
        """
        Initialize the various scan flags and lists and run methods that may change/populate them
        """
        self.do_filesystem_scan = self._get_config_option('scan_filesystem', True)
        self.do_process_scan = self._get_config_option('scan_processes', False)
        self.scan_fsobjects = []
        self.scan_pids = []

        if not (self.do_filesystem_scan or self.do_process_scan):
            logger.error("Both scan_filesystem and scan_processes are disabled.  Nothing to scan.")
            exit(constants.sig_kill_bad)

        # Check if old options are still in use and inform the user of their replacements
        for replaced_scan_option in ('scan_only', 'scan_exclude', 'scan_since'):
            if self._get_config_option(replaced_scan_option):
                logger.error("The '{0}' option has been replaced with the 'filesystem_{0}' and 'processes_{0}' options in {1}"
                             .format(replaced_scan_option, MALWARE_CONFIG_FILE))
                logger.error("Please remove the %s file and a new config file will be written with the new options", MALWARE_CONFIG_FILE)
                exit(constants.sig_kill_bad)

        # Try parsing the filesystem and processes scan_only options and exit under certain conditions
        parse_filesystem_scan_only = self._parse_filesystem_scan_only_option()
        parse_processes_scan_only = self._parse_processes_scan_only_option()
        if not (parse_filesystem_scan_only or parse_processes_scan_only):
            logger.error("Nothing to scan with the filesystem_scan_only and processes_scan_only options")
            exit(constants.sig_kill_bad)
        if not (parse_filesystem_scan_only or self.do_process_scan):
            logger.error("Nothing to scan with filesystem_scan_only option and scan_processes is disabled")
            exit(constants.sig_kill_bad)
        if not (self.do_filesystem_scan or parse_processes_scan_only):
            logger.error("Nothing to scan with processes_scan_only option and scan_filesystem is disabled")
            exit(constants.sig_kill_bad)

        # If we've made it here we are still doing scans, but disable scans if there were problems with scan_only
        if not parse_filesystem_scan_only:
            self.do_filesystem_scan = False
        if not parse_processes_scan_only:
            self.do_process_scan = False

        self._parse_filesystem_scan_exclude_option()
        self._parse_processes_scan_exclude_option()
        self._parse_filesystem_scan_since_option()
        self._parse_processes_scan_since_option()
        self._parse_exclude_network_filesystem_mountpoints_option()

    def _parse_test_scan_option(self):
        self.test_scan = self._get_config_option('test_scan', False)
        if not self.test_scan:
            return False

        self.filesystem_scan_exclude_list = []
        self.processes_scan_exclude_list = []
        self.filesystem_scan_since_dict = {'timestamp': None}
        self.processes_scan_since_dict = {'timestamp': None}
        self.network_filesystem_mountpoints = []

        # For matching the test rule, scan the insights config file and the currently running process
        # Make sure the config file exists first though!
        if os.path.isfile(MALWARE_CONFIG_FILE):
            self.do_filesystem_scan = True
            self.scan_fsobjects = [MALWARE_CONFIG_FILE]
        else:
            self.do_filesystem_scan = False
            self.scan_fsobjects = []

        self.do_process_scan = True
        self.scan_pids = [str(os.getpid())]
        logger.info("\nPerforming a test scan of %sthe current process (PID %s) "
                    "to verify the malware-detection app is installed and scanning correctly ...\n",
                    "%s and " % self.scan_fsobjects[0] if self.do_filesystem_scan else "", self.scan_pids[0])
        return True

    def _parse_filesystem_scan_only_option(self):
        """
        Parse the filesystem_scan_only option, if specified, to get a list of files/dirs to scan
        If parsing was successful, then self.scan_fsobjects is populated and true is returned
        If parsing was not successful, self.scan_fsobjects remains empty and false is returned
        """
        filesystem_scan_only = self._get_config_option('filesystem_scan_only')
        if filesystem_scan_only:
            if not self.do_filesystem_scan:
                logger.error("Skipping filesystem_scan_only option because scan_filesystem is false")
                return False
            # Process the filesystem_scan_only option as a list of files/dirs
            if not isinstance(filesystem_scan_only, list):
                filesystem_scan_only = [filesystem_scan_only]
            for item in filesystem_scan_only:
                # Remove extras slashes (/) in the file name and leading double slashes too (normpath doesn't)
                item = os.path.normpath(item).replace('//', '/')
                # Assume the item represents a filesystem item
                if os.path.exists(item):
                    self.scan_fsobjects.append(item)
                else:
                    logger.info("Skipping missing filesystem_scan_only item: '%s'", item)

            if self.scan_fsobjects:
                logger.info("Scan only the specified filesystem item%s: %s", "s" if len(self.scan_fsobjects) > 1 else "",
                            self.scan_fsobjects)
                return True
            else:
                logger.error("Unable to find the items specified for the filesystem_scan_only option.  Skipping ...")
                return False
        return True

    def _parse_filesystem_scan_exclude_option(self):
        """
        Simple parse of the filesystem_scan_exclude option (if specified) to get a list of valid items to exclude
        """
        if not self.do_filesystem_scan:
            return

        self.filesystem_scan_exclude_list = []
        filesystem_scan_exclude = self._get_config_option('filesystem_scan_exclude')
        if filesystem_scan_exclude:
            if not isinstance(filesystem_scan_exclude, list):
                # Convert filesystem_scan_exclude to a list if only a single non-list item was specified
                filesystem_scan_exclude = [filesystem_scan_exclude]
            for item in filesystem_scan_exclude:
                item = os.path.normpath(item).replace('//', '/')
                if os.path.exists(item):
                    self.filesystem_scan_exclude_list.append(item)
                else:
                    logger.debug("Skipping missing filesystem_scan_exclude item: '%s'", item)
            logger.info("Excluding specified filesystem item%s: %s", "s" if len(self.filesystem_scan_exclude_list) > 1 else "",
                        self.filesystem_scan_exclude_list)

    @staticmethod
    def _parse_processes_scan_option(option_items):
        """
        'option_items' is the list of items provided for either the processes_scan_only or processes_scan_exclude options
        It is parsed as a list of items that may contain:
        - a single PID, eg 1
        - a range of PIDs, eg 10..100 or 10000..  or  ..500
        - a process_name, eg chrome

        A list of PIDs is returned representing all the PIDs that were matched from parsing the items
        """
        pids = []
        ps_output = call([['ps', '-eo', 'pid=', '-o', 'comm=']]).splitlines()
        proc_names = list(map(lambda x: (int(x[0]), str(x[1])), map(lambda x: tuple(x.split()), ps_output)))
        proc_pids = list(map(lambda x: x[0], proc_names))
        if not isinstance(option_items, list):
            option_items = [str(option_items)]
        for item in option_items:
            if isinstance(item, float):
                # Handle floats so they don't cause exceptions
                item = str(item)
            if isinstance(item, int) or item.isdigit():
                # If it's digit, assume it represents a process ID
                if int(item) in proc_pids:
                    logger.debug("Found PID %s", item)
                    pids.append(str(item))
                else:
                    logger.info("Skipping missing PID: %s", item)
            elif '..' in item:
                # Assume the item represents a range of process IDs
                try:
                    start, end = item.split('..', 1)
                    start = 1 if not start else int(start.strip('.'))
                    end = int(open('/proc/sys/kernel/pid_max').read()) if not end else int(end.strip('.'))
                    pid_matches = [str(proc) for proc in proc_pids if start <= proc <= end]
                except Exception as err:
                    logger.error("Unable to parse '%s' in to a range of PIDs: %s", item, str(err))
                    continue
                logger.debug("Found PID(s) in range '%s': %s", item, pid_matches)
                if pid_matches:
                    pids.extend(pid_matches)
                else:
                    logger.info("No PIDs found in process range '%s'", item)
            else:
                # Assume the item is a string representing the name of one or multiple processes
                pid_matches = [str(proc[0]) for proc in proc_names if item in proc[1]]
                if pid_matches:
                    pids.extend(pid_matches)
                    logger.debug("Found PID(s) for string '%s': %s", item, pid_matches)
                else:
                    logger.info("No PID matches found for process name '%s'", item)

        return pids

    def _parse_processes_scan_only_option(self):
        """
        Parse the processes_scan_only option, if specified, to get a list of processes to scan
        """
        processes_scan_only = self._get_config_option('processes_scan_only')
        if processes_scan_only:
            if not self.do_process_scan:
                logger.error("Skipping processes_scan_only option because scan_processes is false")
                return False

            pids = self._parse_processes_scan_option(processes_scan_only)
            if pids:
                self.scan_pids = sorted(set(pids), key=lambda pid: int(pid))
                logger.info("Scan only the specified process ID%s: %s", "s" if len(self.scan_pids) > 1 else "",
                            self.scan_pids)
                return True
            else:
                logger.error("Unable to find the items specified for the processes_scan_only option.  Skipping ...")
                return False
        return True

    def _parse_processes_scan_exclude_option(self):
        """
        Simple parse of the processes_scan_exclude option (if specified) to get a list of processes to exclude
        """
        if not self.do_process_scan:
            return

        self.processes_scan_exclude_list = []
        processes_scan_exclude = self._get_config_option('processes_scan_exclude')
        if processes_scan_exclude:
            if not self.do_process_scan:
                logger.error("Skipping processes_scan_exclude option because scan_processes is false")
                return

            pids = self._parse_processes_scan_option(processes_scan_exclude)
            if pids:
                self.processes_scan_exclude_list = sorted(set(pids), key=lambda pid: int(pid))
                logger.info("Excluding specified process ID%s: %s", "s" if len(self.processes_scan_exclude_list) > 1 else "",
                            self.processes_scan_exclude_list)
            else:
                logger.error("Unable to find the items specified for the processes_scan_exclude option.  Skipping ...")

    def _parse_filesystem_scan_since_option(self):
        """
        filesystem_scan_since is specified as an integer representing the number of days ago to scan for modified files
        If the option was specified and valid, then get the corresponding unix timestamp for the specified
        number of days ago from now, which is used for comparing file modification times
        """
        if not self.do_filesystem_scan:
            return

        self.filesystem_scan_since_dict = {'timestamp': None, 'datetime': None}
        filesystem_scan_since = self._get_config_option('filesystem_scan_since')
        if filesystem_scan_since is not None:
            timestamp = get_scan_since_timestamp('filesystem_scan_since', filesystem_scan_since)
            if timestamp:
                self.filesystem_scan_since_dict['timestamp'] = timestamp
                self.filesystem_scan_since_dict['datetime'] = datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
                message = "Scan for files created/modified since %s%s"
                if isinstance(filesystem_scan_since, str):
                    submessage = 'last successful scan on '
                else:
                    submessage = '%s day%s ago on ' % (filesystem_scan_since, "s" if filesystem_scan_since > 1 else "")
                logger.info(message, submessage, self.filesystem_scan_since_dict['datetime'])

    def _parse_processes_scan_since_option(self):
        """
        processes_scan_since is specified as an integer representing the number of days ago to scan for new processes
        If the option was specified and valid, then get the corresponding unix timestamp for the specified
        number of days ago from now, which is used for comparing process start times
        """
        if not self.do_process_scan:
            return

        self.processes_scan_since_dict = {'timestamp': None, 'datetime': None}
        processes_scan_since = self._get_config_option('processes_scan_since')
        if processes_scan_since is not None:
            timestamp = get_scan_since_timestamp('processes_scan_since', processes_scan_since)
            if timestamp:
                self.processes_scan_since_dict['timestamp'] = timestamp
                self.processes_scan_since_dict['datetime'] = datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
                message = "Scan for processes started since %s%s"
                if isinstance(processes_scan_since, str):
                    submessage = 'last successful scan on '
                else:
                    submessage = '%s day%s ago on ' % (processes_scan_since, "s" if processes_scan_since > 1 else "")
                logger.info(message, submessage, self.processes_scan_since_dict['datetime'])

    def _parse_exclude_network_filesystem_mountpoints_option(self):
        """
        If exclude_network_filesystem_mountpoints is true, get a list of mountpoints of mounted network filesystems.
        The network_filesystem_types option has the list of network filesystems types to look for mountpoints for,
        eg NFS, CIFS, SMBFS, SSHFS, Ceph, GlusterFS, GFS.
        The list of network filesystem mountpoints will be added to the list of directories to exclude from scanning
        """
        if not self.do_filesystem_scan:
            return

        self.network_filesystem_mountpoints = []
        if not self._get_config_option('exclude_network_filesystem_mountpoints'):
            # We aren't excluding network filesystems, leave it as a blank list (ie nothing to exclude)
            return

        network_filesystem_types = self._get_config_option('network_filesystem_types')
        if not network_filesystem_types:
            logger.error("No value specified for 'network_filesystem_types' option")
            exit(constants.sig_kill_bad)

        if isinstance(network_filesystem_types, list):
            network_filesystem_types = ','.join(network_filesystem_types)
        cmd = ['findmnt', '-t', network_filesystem_types, '-n', '-o', 'TARGET']
        logger.debug("Command to find mounted network filesystems: %s", ' '.join(cmd))
        try:
            output = call([cmd])
        except CalledProcessError as err:
            logger.error("Unable to get network filesystem mountpoints: %s", err.output.strip())
            exit(constants.sig_kill_bad)

        self.network_filesystem_mountpoints = str(output).strip().split('\n') if output else []
        if self.network_filesystem_mountpoints:
            logger.info("Excluding network filesystem mountpoints: %s", self.network_filesystem_mountpoints)
        else:
            logger.debug("No mounted network filesystems found")

    def _get_rules(self):
        """
        Obtain the rules used by yara for scanning from the rules_location option.
        They can either be downloaded from the malware backend or obtained from a local file.
        """
        # The rules file that is downloaded from the backend should be automatically removed when the
        # malware-detection client exits.
        # However it can happen that the rules file isn't removed for some reason, so remove any existing
        # rules files before beginning a new scan, otherwise they may show up as matches in the scan results.
        old_rules_files = glob('/tmp/tmp_malware-detection-client_rules.*')
        for old_rules_file in old_rules_files:
            logger.debug("Removing old rules file %s", old_rules_file)
            os.remove(old_rules_file)

        self.rules_location = self._get_config_option(
            'rules_location', "https://console.redhat.com/api/malware-detection/v1/signatures.yar"
        )

        # If rules_location starts with a /, assume its a file rather than a URL
        if self.rules_location.startswith('/'):
            # Remove any extra slashes from the file name and from the start too (normpath doesn't remove those)
            rules_file = os.path.normpath(self.rules_location).replace('//', '/')
            if not os.path.isfile(rules_file):
                logger.error("Couldn't find specified rules file: %s", rules_file)
                exit(constants.sig_kill_bad)
            logger.debug("Using specified rules file: %s", rules_file)
            return rules_file

        # If we are here, then we are downloading the rules from the malware backend
        # Check if insights-config is defined first because we need to access its auth and network config
        if not self.insights_config:
            logger.error("Couldn't access the insights-client configuration")
            exit(constants.sig_kill_bad)

        # Make sure the rules_location starts with https://
        if not re.match('^https?://', self.rules_location):
            self.rules_location = 'https://' + self.rules_location

        # if insights-client is using cert auth or basic auth without a username and password, append 'cert.' to the url
        authmethod = self.insights_config.authmethod if hasattr(self.insights_config, 'authmethod') else 'CERT'
        username = self.insights_config.username if hasattr(self.insights_config, 'username') else ''
        password = self.insights_config.password if hasattr(self.insights_config, 'password') else ''
        if authmethod == 'CERT' or (authmethod == 'BASIC' and not (username or password)):
            self.insights_config.authmethod = 'CERT'
            parsed_url = urlparse(self.rules_location)
            if not parsed_url.netloc.startswith('cert.'):
                self.rules_location = urlunparse(parsed_url._replace(netloc='cert.' + parsed_url.netloc))

        # If doing a test scan, replace signatures.yar (or any other file suffix) with test-rule.yar
        log_rule_contents = False
        if self.test_scan:
            self.rules_location = self._get_test_rule_location(self.rules_location)
            log_rule_contents = True

        logger.debug("Downloading rules from: %s", self.rules_location)
        try:
            self.insights_config.cert_verify = True
            conn = InsightsConnection(self.insights_config)
            response = conn.get(self.rules_location, log_response_text=log_rule_contents)
            if response.status_code != 200:
                logger.error("%s %s: %s", response.status_code, response.reason, response.text)
                exit(constants.sig_kill_bad)
        except Exception as e:
            logger.error("Unable to download rules from %s: %s", self.rules_location, str(e))
            exit(constants.sig_kill_bad)

        self.temp_rules_file = NamedTemporaryFile(prefix='tmp_malware-detection-client_rules.', mode='wb', delete=True)
        self.temp_rules_file.write(response.content)
        self.temp_rules_file.flush()
        return self.temp_rules_file.name

    def _build_yara_command(self):
        """
        Get all the switches for the yara command to be run, for example:
        - whether the rules file is compiled or not (-C)
        - the number of CPU threads to use (-p)
        - the nice command and its value to use (nice -n 'value')
        - scan timeouts (-a)
        """
        # Detect if the rules file is a text or binary (compiled) file (or otherwise)
        output = call([['file', '-b', self.rules_file]])
        rule_type = output.strip().lower()
        if os.path.getsize(self.rules_file) == 0 or rule_type == 'empty':
            logger.error("Rules file %s is empty", self.rules_file)
            exit(constants.sig_kill_bad)

        compiled_rules_flag = '-C' if rule_type.startswith('yara') or rule_type == 'data' else ''
        logger.debug("Rules file type: '%s', Compiled rules: %s", rule_type, compiled_rules_flag == '-C')

        # Quickly test the rules file to make sure it contains usable rules!
        # Note, if the compiled_rules_flag is '' it must be removed from the list or it causes problems
        cmd = list(filter(None, [self.yara_binary, '--fail-on-warnings', '-p', '1', '-f', compiled_rules_flag,
                                 self.rules_file, '/dev/null']))
        try:
            call([cmd])
        except CalledProcessError as err:
            logger.error("Unable to use rules file %s: %s", self.rules_file, err.output.strip())
            exit(constants.sig_kill_bad)

        # Limit the number of threads used by yara to limit the CPU load of the scans
        # If system has 2 or fewer CPUs, then use just one thread
        nproc = call('nproc').strip()
        if not nproc or int(nproc) <= 2:
            self.cpu_thread_limit = 1
        logger.debug("Using %s CPU thread(s) for scanning", self.cpu_thread_limit)

        # Construct the (partial) yara command that will be used later for scanning files and processes
        # The argument for the files and processes that will be scanned will be added later
        yara_cmd = list(filter(None, ['nice', '-n', str(self.nice_value), self.yara_binary, '-s', '-N',
                                      '-a', str(self.scan_timeout), '-p', str(self.cpu_thread_limit), '-r', '-f',
                                      compiled_rules_flag, self.rules_file]))
        logger.debug("Yara command: %s", yara_cmd)
        return yara_cmd

    def scan_filesystem(self):
        """
        Process the filesystem items to scan
        If self.scan_fsobjects is set, then just scan its items, less any items in the exclude list
        scan_dict will contain all the toplevel directories to scan, and any particular files/subdirectories to scan
        """
        if not self.do_filesystem_scan:
            return False

        # Exclude the rules file and insights-client log files, unless they are things we specifically want to scan
        if self.rules_file not in self.scan_fsobjects:
            self.filesystem_scan_exclude_list.append(self.rules_file)
        insights_log_files = glob(constants.default_log_file + '*')
        self.filesystem_scan_exclude_list.extend(list(set(insights_log_files) - set(self.scan_fsobjects)))

        scan_dict = process_include_exclude_items(include_items=self.scan_fsobjects,
                                                  exclude_items=self.filesystem_scan_exclude_list,
                                                  exclude_mountpoints=self.network_filesystem_mountpoints)
        if not scan_dict:
            self.do_filesystem_scan = False
            return False

        logger.debug("Filesystem objects to be scanned in: %s", sorted(scan_dict.keys()))

        logger.info("Starting filesystem scan ...")
        fs_scan_start = time.time()

        for toplevel_dir in sorted(scan_dict):
            # Make a copy of the self.yara_cmd list and add to it the thing to scan
            cmd = self.yara_cmd[:]
            dir_scan_start = time.time()

            specified_log_txt = "specified " if 'include' in scan_dict[toplevel_dir] else ""
            if self.filesystem_scan_since_dict['timestamp']:
                logger.info("Scanning %sfiles in %s modified since %s ...", specified_log_txt, toplevel_dir,
                            self.filesystem_scan_since_dict['datetime'])
                # Find the recently modified files in the given top level directory
                scan_list_file = NamedTemporaryFile(prefix='%s_scan_list.' % os.path.basename(toplevel_dir),
                                                    mode='w', delete=True)
                if 'include' in scan_dict[toplevel_dir]:
                    find_modified_include_items(scan_dict[toplevel_dir]['include'], self.filesystem_scan_since_dict['timestamp'], scan_list_file)
                else:
                    find_modified_in_directory(toplevel_dir, self.filesystem_scan_since_dict['timestamp'], scan_list_file)

                scan_list_file.flush()
                cmd.extend(['--scan-list', scan_list_file.name])
            else:
                logger.info("Scanning %sfiles in %s ...", specified_log_txt, toplevel_dir)
                if 'include' in scan_dict[toplevel_dir]:
                    scan_list_file = NamedTemporaryFile(prefix='%s_scan_list.' % os.path.basename(toplevel_dir),
                                                        mode='w', delete=True)
                    scan_list_file.write('\n'.join(scan_dict[toplevel_dir]['include']))
                    scan_list_file.flush()
                    cmd.extend(['--scan-list', scan_list_file.name])
                else:
                    cmd.append(toplevel_dir)

            logger.debug("Yara command: %s", cmd)
            try:
                output = call([cmd]).strip()
            except CalledProcessError as cpe:
                logger.debug("Unable to scan %s: %s", toplevel_dir, cpe.output.strip())
                continue

            self.parse_scan_output(output.strip())

            dir_scan_end = time.time()
            logger.info("Scan time for %s: %d seconds", toplevel_dir, (dir_scan_end - dir_scan_start))

        fs_scan_end = time.time()
        logger.info("Filesystem scan time: %s", time.strftime("%H:%M:%S", time.gmtime(fs_scan_end - fs_scan_start)))
        return True

    def scan_processes(self):
        if not self.do_process_scan:
            return False

        # Get a list of all PIDs to scan if none were specified with scan only option
        if not self.scan_pids:
            self.scan_pids = [entry for entry in os.listdir('/proc') if entry.isdigit()]

        # Add this currently running process' PID to the list of processes to exclude (unless its a test_scan)
        # Then remove the excluded processes from the list of PIDs to scan
        if not self.test_scan:
            self.processes_scan_exclude_list.append(str(os.getpid()))  # make sure to exclude our script's pid
        self.scan_pids = sorted(list(set(self.scan_pids) - set(self.processes_scan_exclude_list)), key=lambda pid: int(pid))

        if not self.scan_pids:
            logger.error("No processes to scan because the specified exclude items cancel them out")
            self.do_process_scan = False
            return False

        # If process_scan_since is specified, get a list of processes started since the specified date
        if hasattr(self, 'processes_scan_since_dict') and self.processes_scan_since_dict['timestamp']:
            # First get a list of all running processes and their start times
            ps_output = call([['ps', '-eo', 'pid=', '-o', 'lstart=']]).splitlines()
            all_proc_starts = list(map(lambda x: (str(x[0]), str(x[1])), map(lambda x: tuple(x.strip().split(' ', 1)), ps_output)))
            scan_since_pids = []
            # Loop through all processes and if the process start time <= the specified scan_since time then
            # make note of the process
            for proc in all_proc_starts:
                proc_start_secs = float(datetime.strptime(proc[1], '%a %b %d %H:%M:%S %Y').strftime('%s'))
                if proc_start_secs >= self.processes_scan_since_dict['timestamp']:
                    scan_since_pids.append(proc[0])
            # Finally, do a set intersection of the current list of scan_pids and the list of processes started
            # since processes_scan_since.  The resulting list is the list of processes to scan.
            self.scan_pids = sorted(list(set(self.scan_pids) & set(scan_since_pids)), key=lambda pid: int(pid))

            if not self.scan_pids:
                logger.error("No processes to scan because none were started since %s", self.processes_scan_since_dict['datetime'])
                self.do_process_scan = False
                return False

        logger.info("Starting processes scan ...")
        pids_scan_start = time.time()

        for scan_pid in self.scan_pids:
            pid_scan_start = time.time()
            logger.info("Scanning process %s ...", scan_pid)
            cmd = self.yara_cmd + [str(scan_pid)]
            logger.debug("Yara command: %s", cmd)
            try:
                output = call([cmd]).strip()
            except CalledProcessError as cpe:
                logger.debug("Unable to scan process %s: %s", scan_pid, cpe.output.strip())
                continue

            self.parse_scan_output(output)

            pid_scan_end = time.time()
            logger.info("Scan time for process %s: %d seconds", scan_pid, (pid_scan_end - pid_scan_start))

        pids_scan_end = time.time()
        logger.info("Processes scan time: %s", time.strftime("%H:%M:%S", time.gmtime(pids_scan_end - pids_scan_start)))
        return True

    def parse_scan_output(self, output, exclude_items=[]):
        if not output:
            return

        # Each 'set' of output lines consists of 1 line containing the rule and file/pid (aka source) it matches
        # Followed by one or more related lines of matching string data from that source, eg
        # ...
        # rule_name source                            + Set of 3 related lines
        # 0x_offset:string_identifier:string_data     |
        # 0x_offset:string_identifier:string_data     +
        # rule_name source                            + Set of 2 related lines
        # 0x_offset:string_identifier:string_data     +
        # ...

        def skip_string_data_lines(string_data_lines):
            # Skip the 0x... lines containing string match data
            while string_data_lines and string_data_lines[0].startswith('0x'):
                logger.debug("Skipping string data line '%s'", string_data_lines[0])
                string_data_lines.pop(0)

        output_lines = output.split("\n")
        while output_lines:
            if 'error scanning ' in output_lines[0]:
                if output_lines[0].endswith('error: 4'):
                    # Yara 'could not map file' error - only display this error if debugging (spammy)
                    logger.debug(output_lines[0])
                else:
                    logger.info(output_lines[0])
                output_lines.pop(0)  # Skip the error scanning line
                # Skip any string match lines after the error scanning line
                skip_string_data_lines(output_lines)
                continue
            # Get the rule_name and source from the first line in the set
            try:
                rule_name, source = output_lines[0].rstrip().split(" ", 1)
            except ValueError as err:
                # Hopefully shouldn't happen but log it and continue processing
                logger.debug("Error parsing rule match '%s': %s", output_lines[0], err)
                output_lines.pop(0)  # Skip the erroneous line
                # Skip any string match lines afterwards until we get to the next rule match line
                skip_string_data_lines(output_lines)
                continue

            # All good so far, skip over the line containing the rule name and matching source file/pid
            output_lines.pop(0)

            # If the rule or the source file/pid is to be excluded, then skip over its scan matches
            # and move onto the next match
            # if any([exclude_rule.lower() in rule_name.lower() for exclude_rule in self.exclude_rules]) \
            #         or any([ei in source for ei in exclude_items]):
            #     skip_string_data_lines(output_lines)
            #     continue

            # Check if the rule name contains a ':' or doesn't start with a char/string
            # It shouldn't and its likely to be due to a malformed string_offset line
            # Skip any further scan matches until the next rule match
            if ':' in rule_name or not re.match('^[a-zA-Z]+', rule_name):
                skip_string_data_lines(output_lines)
                continue

            rule_match = {'rule_name': rule_name, 'matches': []}
            source_type = "process" if source.isdigit() else "file"

            # Parse the string match data for the remaining lines in the set
            string_matches = 0
            while output_lines and output_lines[0].startswith('0x'):
                if string_matches < self.string_match_limit:
                    try:
                        string_offset, string_identifier, string_data = output_lines[0].split(':', 2)
                        string_offset = int(string_offset, 0)
                    except ValueError as err:
                        logger.debug("Error parsing string match '%s': %s", output_lines[0], err)
                        output_lines.pop(0)
                        continue
                    rule_match_dict = {'source': source,
                                       'string_data': string_data.strip(),
                                       'string_identifier': string_identifier,
                                       'string_offset': string_offset,
                                       'metadata': {'source_type': source_type}}
                    rule_match['matches'].extend([rule_match_dict])
                output_lines.pop(0)
                string_matches += 1

            # If string_match_limit is 0 or there was no string data, there will be no rule_matches,
            # but still record the file/pid source that was matched
            if not rule_match['matches']:
                rule_match_dict = {'source': source,
                                   'string_data': '',
                                   'string_identifier': '',
                                   'string_offset': -1,
                                   'metadata': {'source_type': source_type}}
                rule_match['matches'] = [rule_match_dict]

            if self.add_metadata:
                # Add extra data to each rule match, beyond what yara provides
                # Eg, for files: line numbers & context, checksums; for processes: process name
                # TODO: find more pythonic ways of doing this stuff instead of using system commands
                metadata_func = self._add_file_metadata if source_type == 'file' else self._add_process_metadata
                metadata_func(rule_match['matches'])

            self.matches += 1
            logger.info("Matched rule %s in %s %s", rule_name, source_type, source)
            logger.debug(rule_match)
            if self.host_scan.get(rule_match['rule_name']):
                self.host_scan[rule_match['rule_name']].extend(rule_match['matches'])
            else:
                self.host_scan[rule_match['rule_name']] = rule_match['matches']

    def _add_process_metadata(self, rule_matches):
        """
        Add extra data to the process scan matches beyond what is provided by yara, eg process name
        """
        # All passed in rule_matches will have the same source PID
        # Check the process still exists before obtaining the metadata about it
        source = rule_matches[0]['source']
        if not os.path.exists('/proc/%s' % source):
            return

        # Get name of process from ps command
        # -h: no output header, -q: only the specified process, -o args: just the process name and args
        try:
            process_name = call([['ps', '-hq', source, '-o', 'args']]).strip()
        except CalledProcessError:
            process_name = 'unknown'

        for rule_match in rule_matches:
            rule_match['metadata'].update({'process_name': process_name})

    def _add_file_metadata(self, rule_matches):
        """
        Add extra data to the file scan matches beyond what is provided by yara
        - eg matching line numbers, line context, file checksum
        - Use grep to get the line numbers & sed to get the line
        """
        def get_line_from_file(file_name, line_number):
            # Extract the line at line_number from file_name
            line_length_limit = 120
            try:
                line = call([['sed', '%dq;d' % line_number, file_name]]).strip()
            except CalledProcessError:
                line = ""
            # Limit line length if necessary and urlencode it to minimize problems with GraphQL when uploading
            return urlencode(line if len(line) < line_length_limit else line[:line_length_limit] + "...")

        # All passed in rule_matches will have the same source file
        # Check the file still exists before obtaining the metadata about it
        source = rule_matches[0]['source']
        if not os.path.exists(source):
            return

        # Get the file type, mime type and md5sum hash of the source file
        try:
            file_type = call([['file', '-b', source]]).strip()
        except Exception:
            file_type = ""
        try:
            mime_type = call([['file', '-bi', source]]).strip()
        except Exception:
            mime_type = ""
        try:
            md5sum = call([['md5sum', source]]).strip().split()[0]
        except Exception:
            md5sum = ""

        grep_string_data_match_list = []
        if mime_type and 'charset=binary' not in mime_type:
            # Get the line numbers for each of yara's string_data matches in the source file, but not for binary files
            # Build a grep command that searches for each of the string_data patterns in the source file
            # For each string_data pattern that grep finds, the grep output will have the form...
            # line_number:offset_from_0:string_data_pattern

            # Get the set of patterns to grep for, eg ['pattern1', 'pattern2', etc], ie remove duplicate patterns
            grep_string_data_pattern_set = set([match['string_data'] for match in rule_matches])
            if grep_string_data_pattern_set:
                # Build an option list for grep, eg ['-e', 'pattern1', '-e', 'pattern2', ... etc]
                # zip creates a list of tuples, eg [('-e', 'pattern'), ('-e', 'pattern2'), ...], then flatten the list
                grep_string_data_patterns = [item for tup in list(zip(['-e'] * len(grep_string_data_pattern_set),
                                                                      grep_string_data_pattern_set))
                                             for item in tup]
                # Create the grep command to execute.  -F means don't interpret regex special chars in the patterns
                grep_command = ['/bin/grep', '-Fbon'] + grep_string_data_patterns + [source]
                logger.debug("grep command: %s", grep_command)
                try:
                    grep_output = call([grep_command])
                except CalledProcessError:
                    grep_output = ""

                # Now turn the grep output into a list of tuples for easier searching a little later, ie
                # [(line_number, offset_from_0, string_data_pattern), (...), ]
                if grep_output and not grep_output.lower().startswith('binary'):
                    grep_string_data_match_list = list(map(lambda grep_output_line: tuple(grep_output_line.split(':', 3)),
                                                           grep_output.splitlines()))

        for rule_match in rule_matches:
            metadata = rule_match['metadata']
            metadata.update({'file_type': file_type,
                             'mime_type': mime_type,
                             'md5sum': md5sum})
            if grep_string_data_match_list:
                # Now, for each offset_from_0 in the grep output, we want to match it with the corresponding
                # string_offset value from the yara output so we can get the line number for that string_data match
                # And while we are here, get the line from the source file at that line number
                line_number = None
                for grep_list_item in grep_string_data_match_list:
                    if int(grep_list_item[1]) == rule_match['string_offset']:
                        line_number = int(grep_list_item[0])
                        break
                if line_number:
                    metadata.update({'line_number': line_number,
                                     'line': get_line_from_file(source, line_number)})

    def _create_host_scan_mutation(self):
        # Build the mutation text
        mutation_header = """
        mutation HostScan {
          recordHostScan(
            input: {
              scannedhost: {
                insightsId: "%s"
                rulesScanned: [""" % generate_machine_id()

        mutation_footer = """
                ]
              }
            }
          ) {
            success
          }
        }"""

        mutation = mutation_header
        for rule_name in self.host_scan.keys():
            rule_scan = """{
                ruleName: "%s"
                stringsMatched: [""" % rule_name
            for match in self.host_scan[rule_name]:
                rule_scan += """{
                    source: "%s"
                    stringData: %s
                    stringIdentifier: %s
                    stringOffset: "%s"
                    metadata: "%s"
                }, """ % (match['source'],
                          json.dumps(match['string_data']),
                          json.dumps(match['string_identifier']),
                          match['string_offset'],
                          json.dumps(match['metadata']).replace('"', '\\"'))
            rule_scan += "]}, "
            mutation += rule_scan

        mutation += mutation_footer
        return mutation

    def _get_config_option(self, option, default_value=None):
        """
        Get the value of a config option or if it doesn't exist or is None, the default_value
        """
        value = os.getenv(option.upper())
        if value is not None:
            return self._parse_env_var(option.upper(), value)
        value = self.config.get(option)
        return value if value is not None else default_value

    @staticmethod
    def _parse_env_var(env_var, value):
        """
        Parse specific environment variables to make sure they have appropriate values
        """
        logger.debug("Found environment variable: %s, value: %s", env_var, value)
        # Parse these env vars as booleans
        if env_var in ENV_VAR_TYPES['boolean']:
            return value.lower() in ('true', 'yes', 't', 'y')

        # Parse these as lists by splitting at the commas
        if env_var in ENV_VAR_TYPES['list']:
            if value:
                return value.split(',') if ',' in value else [value]
            else:
                return []

        # Parse *_scan_since, can be either an int or a string (ie 'last')
        if env_var in ENV_VAR_TYPES['int_or_str']:
            return int(value) if value.isdigit() else value

        # Parse these as ints
        if env_var in ENV_VAR_TYPES['integer']:
            try:
                return int(value)
            except ValueError as e:
                logger.error("Problem parsing environment variable %s: %s", env_var, str(e))
                exit(constants.sig_kill_bad)

        # env_var value doesn't require parsing, just return it as is (ie. as a string)
        return value

    @staticmethod
    def _get_test_rule_location(rules_location):
        test_rule = 'test-rule.yar'
        # Nothing to do if the location already ends with test_rule
        if rules_location.endswith(test_rule):
            return rules_location
        # Replace the last entry from the path with the test-rule
        # A bit crude but it seems to work ok with both URL and file paths
        return os.path.join(os.path.dirname(rules_location), test_rule)


#
# Utility functions
# Mainly for including / excluding certain directories for scanning
# And also for finding files that have been modified recently
#
def get_toplevel_dirs():
    """
    Returns a list of the top level directories directly under root (/),
    """
    toplevel_dirs = sorted(filter(lambda x: not os.path.islink(x), map(lambda x: "/" + x, os.listdir('/'))))
    return toplevel_dirs


def get_parent_dirs(item, parent_dir_list, base_case='/'):
    """
    Get a list of parent directories of a particular filesystem item, stopping at base_case (root by default)
    Eg for get_parent_dirs('/path/to/some/item', parent_dir_list) ->
        parent_dir_list = ['/path', '/path/to', '/path/to/some', '/path/to/some/item']
    """
    if os.path.samefile(item, base_case) or os.path.samefile(item, '/'):
        return
    get_parent_dirs(os.path.dirname(item), parent_dir_list, base_case)
    parent_dir_list.append(item)


def process_include_items(include_items=[]):
    """
    Process the include items to a get list of directories to be scanned
    If there are no entries then get the list of top level directories under root (/),
    :return: a list of directories to be scanned.  It never returns an empty list.
    """
    default_values = get_toplevel_dirs()

    logger.debug("Parsing include items ...")
    parsed_list = []
    for item in include_items:
        item = item.strip()
        if not item or item.startswith('#'):
            continue
        include_item = os.path.normpath(item).replace('//', '/')
        if os.path.exists(include_item):
            # ignore the include_item if its not a full directory path
            if not include_item.startswith('/'):
                logger.debug("Skipping partial directory path '%s' ...", include_item)
                continue
            elif os.path.islink(include_item):
                logger.debug("Skipping link '%s' ...", include_item)
                continue
            elif os.path.samefile(include_item, '/'):
                # Found / in include item list.  No need to get the other items because / trumps all
                logger.debug("Found root directory in list of items to scan.  Ignoring the other items ...")
                parsed_list = default_values
                break
            else:
                parsed_list.append(include_item)
        else:
            logger.debug("Skipping missing item '%s' ...", include_item)

    if not parsed_list:
        logger.debug("No items specified to be scanned.  Using default values %s ...", default_values)
        parsed_list = default_values
    else:
        # Remove any duplicates and any children of parent directories before returning
        parsed_list = remove_child_items(sorted(list(set(parsed_list))))

    logger.debug("Include items: %s", parsed_list)
    return parsed_list


def process_exclude_items(exclude_items=[]):
    """
    Process the exclude items to get list of directories to NOT be scanned
    :return: a list of directories to not be scanned if any, otherwise an empty list
    """
    logger.debug("Parsing exclude items ...")
    parsed_list = []
    for item in exclude_items:
        item = item.strip()
        if not item or item.startswith('#'):
            continue
        exclude_item = os.path.normpath(item).replace('//', '/')
        if os.path.exists(exclude_item):
            # ignore the exclude_item if its not a full directory path
            if os.path.samefile(exclude_item, '/'):
                # Found / in exclude list.  No need to get the other items because / trumps all
                logger.debug("Found root directory in the exclude list.  Expanding it to all toplevel directories ...")
                parsed_list = get_toplevel_dirs()
                break
            elif not exclude_item.startswith('/'):
                logger.debug("Skipping partial directory path '%s' ...", exclude_item)
                continue
            else:
                parsed_list.append(exclude_item)
        else:
            logger.debug("Skipping missing item '%s' ...", exclude_item)

    if not parsed_list:
        logger.debug("No items specified to be excluded")
    else:
        # Remove any duplicates and any children of parent directories before returning
        parsed_list = remove_child_items(sorted(list(set(parsed_list))))

    logger.debug("Exclude items: %s", parsed_list)
    return parsed_list


def remove_child_items(item_list):
    """
    For a list of filesystem items, remove those items that are duplicates or children of other items
    Eg, for remove_child_items['/path/to/some/item/child', '/path/to/another/item', '/path/to/some/item']
        returns ['/path/to/another/item', '/path/to/some/item']
    If one if the items is root, then it wins
    Also, all items should be the full path starting at root (/).  Any that aren't are removed
    """
    if '/' in item_list:
        return ['/']

    # Remove duplicates and any non-full path items
    item_list = sorted(list(set(filter(lambda x: x.startswith('/'), item_list))))
    remove_items = set([])
    for i, item1 in enumerate(item_list[:-1]):
        for item2 in item_list[i + 1:]:
            if item1 != item2 and item2.startswith(item1 + '/'):
                remove_items.add(item2)
    for remove_item in remove_items:
        item_list.remove(remove_item)
    return sorted(list(set(item_list)))


def remove_included_excluded_items(included_items, excluded_items):
    """
    Go through the list of included items and remove any that are in the exclude list,
        or are children of excluded items (no need to scan an included item if its parent is to be excluded)
    """
    # Clean up the lists, just in case this hasn't been done already
    included_items = remove_child_items(included_items)
    excluded_items = remove_child_items(excluded_items)

    remove_items = set([])
    for included_item in included_items:
        for excluded_item in excluded_items:
            if excluded_item == included_item or included_item.startswith(excluded_item + '/'):
                remove_items.add(included_item)
    for remove_item in remove_items:
        included_items.remove(remove_item)
    return included_items


def process_include_exclude_items(include_items=[], exclude_items=[], exclude_mountpoints=[]):
    """
    Process the include and exclude items, where the exclude items are effectively subtracted from the include_items.
    It builds a scan_dict dictionary of items to scan keyed by the filesystem top level directories.
    Only the toplevel directories from items in the include_items list will be present in scan_dict.
    scan_dict = {'/boot': {'include': ['/boot/include/me', ...], 'exclude: ['/boot/exclude/me', ...]},
                 '/etc': {'include': ['/etc/include/me', ...], 'exclude: ['/etc/exclude/me', ...]},
                 ...
    :return: scan_dict
    """
    # Get a list of excluded items from the exclude file and network filesystem mountpoints
    initial_exclude_list = process_exclude_items(exclude_items)
    final_exclude_list = remove_child_items(list(set(exclude_mountpoints) | set(initial_exclude_list)))
    logger.debug("Final exclude items: %s", final_exclude_list)

    # Get a list of included items from the include file, minus the excluded items
    initial_include_list = process_include_items(include_items)
    if not initial_include_list:
        logger.error("No filesystem items to scan because the include items doesn't contain any valid items")
        return {}
    final_include_list = remove_included_excluded_items(initial_include_list, final_exclude_list)
    logger.debug("Final include items after removing exclude items: %s", final_include_list)
    if not final_include_list:
        logger.error("No filesystem items to scan because the specified exclude items cancel them out")
        return {}

    # This is the dictionary that will hold all the items to scan (after processing the include and exclude items)
    # It will be keyed by each of the toplevel directories containing items to scan
    # yara will scan each of the toplevel dir's 'include' keys (if present), or just the toplevel dir itself
    scan_dict = {}

    # Populate the scan_dict by creating keys for each toplevel directory of the items to include/scan
    # Create an 'include' key for each toplevel directory containing items to include in that toplevel directory
    logger.debug("Populating scan_dict's include items ...")
    for include_item in final_include_list:
        item_subpaths = []
        get_parent_dirs(include_item, item_subpaths)
        include_item_toplevel_dir = item_subpaths[0]
        if include_item_toplevel_dir not in scan_dict:
            # Create an 'include' key if the item to scan isn't just the toplevel directory itself
            scan_dict[include_item_toplevel_dir] = {'include': set([include_item])}\
                if include_item != include_item_toplevel_dir else {}
        else:
            scan_dict[include_item_toplevel_dir]['include'].add(include_item)

    logger.debug("Scan dict after adding include items: %s", scan_dict)

    # Populate an 'exclude' key for the toplevel dirs in the scan_dict that also have items to exclude
    # Or remove the toplevel dirs from the scan dict where the toplevel dir itself is to be excluded
    logger.debug("Populating scan_dict's exclude items ...")
    for exclude_item in final_exclude_list:
        item_subpaths = []
        get_parent_dirs(exclude_item, item_subpaths)
        exclude_item_toplevel_dir = item_subpaths[0]
        if exclude_item_toplevel_dir not in scan_dict:
            # This exclude_item's toplevel dir isn't in the scan dict, so skip it (since its not being included)
            continue
        if 'exclude' not in scan_dict[exclude_item_toplevel_dir]:
            # Create the 'exclude' key if it doesn't already exist
            scan_dict[exclude_item_toplevel_dir]['exclude'] = {'items': [], 'subpaths': set([])}

        scan_dict[exclude_item_toplevel_dir]['exclude']['items'].append(exclude_item)

        # Add the list of subpaths leading to this exclude item.
        # The subpaths are needed later for listing the contents each subpath
        scan_dict[exclude_item_toplevel_dir]['exclude']['subpaths'].update(item_subpaths)

    logger.debug("Scan dict after adding exclude items: %s", scan_dict)

    # For each toplevel dir with items to exclude, re-populate the include key with directory content listings
    # of the subpaths, minus the items to exclude and only including items to include.  Yep, its complicated.
    # These directory listings will be used with yara's --scan-list option
    logger.debug("Re-populating scan_dict's include items with directory content listings to pass to yara ...")
    for toplevel_dir in scan_dict:
        if 'exclude' not in scan_dict[toplevel_dir]:
            continue

        # Get directory listings of each of the subpaths
        if 'include' in scan_dict[toplevel_dir]:
            scan_items = set(scan_dict[toplevel_dir]['include'])
        else:
            scan_items = set([])
        toplevel_dir_exclude = scan_dict[toplevel_dir]['exclude']
        for exclude_item in toplevel_dir_exclude['items']:
            subpaths = []
            get_parent_dirs(exclude_item, subpaths)
            for i, subpath in enumerate(subpaths[:-1]):
                dir_list = os.listdir(subpath)
                dir_list = sorted(map(lambda x: subpath + '/' + x, dir_list))
                dir_list.remove(subpaths[i + 1])
                scan_items.update(dir_list)

        # Go through the list of scan items and remove any exclude items or exclude item subpaths
        for scan_item in list(scan_items):
            for exclude_item in toplevel_dir_exclude['items']:
                if scan_item == exclude_item or scan_item.startswith(exclude_item + '/'):
                    scan_items.remove(scan_item)
                    break
            else:
                for exclude_subpath in toplevel_dir_exclude['subpaths']:
                    if scan_item == exclude_subpath:
                        scan_items.remove(scan_item)

        # If there is an include list, make sure the scan_items only include items in the include list
        if 'include' in scan_dict[toplevel_dir]:
            for maybe_include in list(scan_items):
                if os.path.islink(maybe_include) or (not os.path.isfile(maybe_include) and not os.path.isdir(maybe_include)):
                    scan_items.remove(maybe_include)
                    continue
                if any([maybe_include == definitely_include or maybe_include.startswith(definitely_include + '/')
                        for definitely_include in scan_dict[toplevel_dir]['include']]):
                    continue
                else:
                    scan_items.remove(maybe_include)

        # Overwrite the existing include key list with the new list of scan_items
        scan_dict[toplevel_dir]['include'] = sorted(list(scan_items))

    logger.debug("Final scan_dict: %s", scan_dict)
    return scan_dict


def get_scan_since_timestamp(scan_since_option, since):
    """
    Return a unix timestamp corresponding to how long ago to scan for files or processes (depending on scan_since_option)
    Valid values of 'since' are integers > 0 meaning the number of days back in time from now,
                             or 'last' meaning get the timestamp of the last scan
    If 'since' is not one of these valid values, then terminate
    """
    now = time.time()
    timestamp_file = LAST_FILESYSTEM_SCAN_FILE if scan_since_option == 'filesystem_scan_since' else LAST_PROCESSES_SCAN_FILE

    def get_lastscan_timestamp(scan_since_option, lastscan):
        try:
            # Convert the datetime string into a unix timestamp
            lastscan_seconds = float(datetime.strptime(lastscan, '%Y-%m-%dT%H:%M:%S.%f').strftime('%s'))
            if lastscan_seconds > now:
                raise RuntimeError("Last scan time is in the future.")
        except Exception as err:
            logger.error("Error getting time of last malware scan: %s.  Ignoring '%s: last' option ...", str(err), scan_since_option)
            return None
        return lastscan_seconds

    if isinstance(since, str) and since.lower().startswith('l'):
        # Get the timestamp of the last scan
        if os.path.isfile(timestamp_file):
            with open(timestamp_file) as f:
                lastscan = f.readline().strip()
            return get_lastscan_timestamp(scan_since_option, lastscan)
        else:
            logger.info("File %s doesn't exist for '%s: last' option.  Continuing ...", timestamp_file, scan_since_option)
            return None
    elif isinstance(since, str):
        logger.error("Unknown value '%s' for %s option.  Valid values are integers >= 1 and 'last'", since, scan_since_option)
        exit(constants.sig_kill_bad)

    try:
        since_int = int(since)
        if since_int >= 1:
            return now - (since_int * 86400)  # 86400 seconds in a day
        else:
            raise ValueError("Invalid %s value %s.  Valid values are integers >= 1 and 'last'" % (scan_since_option, since))
    except ValueError as e:
        logger.error(str(e))
        exit(constants.sig_kill_bad)


def is_recent_mtime(item, timestamp):
    """
    Return True if the given 'item' has a modification time that is newer than 'timestamp'
    Return False otherwise, or if the the 'item' is a link or another non-file type (eg pipes)
    """
    if os.path.exists(item) and not os.path.islink(item) and os.path.isfile(item):
        return os.path.getmtime(item) > timestamp
    return False


def find_modified_in_directory(directory, timestamp, output_file):
    """
    Find files in 'directory' that have been created/modified since 'timestamp'
    and write their names to 'output_file'
    """
    for root, dirs, files in os.walk(directory):
        for afile in files:
            path = os.path.join(root, afile)
            if is_recent_mtime(path, timestamp):
                output_file.write(path + "\n")


def find_modified_include_items(item_list, timestamp, output_file):
    """
    Find files in the given list of items (files/directories) that have been created/modified since 'timestamp'
    and write their names to 'output_file'
    """
    for item in item_list:
        if os.path.isdir(item):
            find_modified_in_directory(item, timestamp, output_file)
        else:
            if is_recent_mtime(item, timestamp):
                output_file.write(item + '\n')
