"""Provides the filter task object."""

import os
from . import task
from ..logs import log_templates_pb2


class FilterTask(task.Task):
    """This class provides the mechanism for performing filter tasks.

    The filter task filters downloads in the sense that it looks for duplicate
    feed downloads and removes them. If following a strategy of downloading
    feeds at a fixed interval of time, as the download task does, it is likely
    that there will be many duplicates, especially if downloading frequently
    for redundancy.

    The filter task iterates through each file in the downloaded directory.
    For each file it tries to interpret it (given the file name) as a feed
    download, if it can't the file is deleted. For valid downloads, the task
    attempts to read the file using the user provided timestamp function.
    The timestamp function takes the path of the file and returns the time
    the feed was generated by the transit authority. If the function returns
    -1, or if an exception is raised, that download is deemed corrupt and
    deleted. Otherwise the file is copied to the filtered directory, with a
    new file name based on the transit authority timestamp. Duplicate files
    are easily discovered as the transit authority timestamp, and hence the
    new file name, are the same.

    To use the download task, initialize in the common way for all tasks:

        task = FilterTask(root_dir=, feeds=, quiet=, log_file_path=)

    see the task class for details on the arguments here. Additional
    initialization is likely desired by setting the limit attribute:

        task.limit = 1000

    The task is then run using the run() method:

        task.run()

    Attributes:
        limit (int): an integer imposing an upper bound on the number of
                     downloaded files to process. If set to -1, no limit
                     is imposed. Default is -1.
        n_total (int): number of downloaded files processed.
        n_corrupt (int): number of downloaded files that were deemed corrupt.
        n_copied (int): number of downloaded files copied to the filtered
                        directory.
        n_skipped (int): number of downloaded files that were not copied
                         because they were duplicated of files already copied.
    """

    def __init__(self, **kwargs):

        super().__init__(**kwargs)

        # Initialize the log directory
        self._init_log('filter', log_templates_pb2.FilterTaskLog)

        # Initialize task configuration
        self.limit = -1
        self.file_access_lag = 120

    def _run(self):
        """Run a filter task."""

        # Place the task configuration in the log
        self._log_run_configuration()
        self._log.limit = self.limit
        self._log.file_access_lag = self.file_access_lag

        # These variables will be populated during the run and the logged
        self.num_corrupt_by_feed_id = {feed[0]: 0 for feed in self.feeds}
        self.num_copied_by_feed_id = {feed[0]: 0 for feed in self.feeds}
        self.num_duplicate_by_feed_id = {feed[0]: 0 for feed in self.feeds}
        self.num_total = 0
        self.source_dirs = set()
        self.target_dirs = set()
        limit_reached = False

        # Walk over the directory containing the raw downloaded files,
        # inspecting every file. We attempt to process files that are
        # feed downloaded, and remove other ('zombie') files.
        iterable = self._files_schema.list_downloaded_feeds(self.time())
        for (feed, timestamp, source_file_path) in iterable:
            (feed_id, _, feed_ext, feed_func) = feed

            # This file will be processed, so increment the
            # n_total counter
            self.num_total += 1
            self.source_dirs.add(os.path.dirname(source_file_path))

            # The next step is to try to read the feed timestamp: the
            # time the transit authority distributed the feed.
            # In general this will be different to the download time
            # because we may have downloaded a few seconds late.
            # The timestamp is read using the user-provided function.
            # If there is an exception raised, or if the timestamp is
            # negative, mark the file as corrupt and delete it.
            try:
                feed_timestamp = feed_func(source_file_path)
                if feed_timestamp < 0:
                    raise Exception('File timestamp is negative.')
            except Exception as e:
                self._print('Corrupt file: {}'.format(source_file_path))
                self._print('  (reason given: {})'.format(str(e)))
                self.num_corrupt_by_feed_id[feed_id] += 1
                self._file_system.remove(source_file_path)
                continue

            target_file_path = self._files_schema.filtered_file_path(
                feed_timestamp,
                feed_id,
                feed_ext
                )

            # If the target file does not exist, copy it
            # Otherwise this is a duplicate feed and can be skipped
            if not self._file_system.isfile(target_file_path):
                self._file_system.copyfile(source_file_path, target_file_path)
                self.num_copied_by_feed_id[feed_id] += 1
                self.target_dirs.add(os.path.dirname(target_file_path))
            else:
                self.num_duplicate_by_feed_id[feed_id] += 1

            # Remove the original downloaded file
            self._file_system.remove(source_file_path)

            # Check if the limit of number of files to be processed
            # has been reached
            if self.limit >= 0 and self.num_total >= self.limit:
                limit_reached = True
                self._log.limit_reached = True
                self._print('Copy threshold reached; closing.')
                self._print('Run again to filter more files.')
                break

    def _close(self):

        # At this stage the filtering process has ended, and we need to do
        # some cleaning up.
        # First, by moving downloaded files over we may have a lot of empty
        # subdirectories in the downloaded store, delete these.
        self._file_system.prune_dir_tree(
            self._files_schema.downloaded_root_dir
            )

        # Write the concluding statistics to the log file
        for (feed_id, _, _, _) in self.feeds:
            self._log.num_copied.append(
                self.num_copied_by_feed_id[feed_id]
                )
            self._log.num_duplicate.append(
                self.num_duplicate_by_feed_id[feed_id]
                )
            self._log.num_corrupt.append(
                self.num_corrupt_by_feed_id[feed_id]
                )
        for source_dir in self.source_dirs:
            self._log.source_directories.append(source_dir)
        for target_dir in self.target_dirs:
            self._log.target_directories.append(target_dir)

        # Write some user readable statistics to the terminal
        total_corrupt = sum(self.num_corrupt_by_feed_id.values())
        total_copied = sum(self.num_copied_by_feed_id.values())
        total_duplicate = sum(self.num_duplicate_by_feed_id.values())
        if self.num_total > 0:
            percent_valid = (
                100
                * (total_duplicate + total_copied)
                / (self.num_total)
                )
        else:
            percent_valid = 100
        if total_duplicate + total_copied > 0:
            percent_duplicate = (
                100
                * total_duplicate
                / (total_duplicate + total_copied)
                )
        else:
            percent_duplicate = 100

        self._print('\n'.join([
            'Filter task ended. Statistics:',
            '  * {} feed downloads processed.'.format(self.num_total),
            '  * {} corrupt downloads.'.format(total_corrupt),
            '  * {} valid downloads.'.format(total_copied+total_duplicate),
            '  * {:.2f}% of downloads deemed valid.'.format(percent_valid),
            '  * {:.2f}% of downloads were duplicates.'.format(
                percent_duplicate
                )
            ]))
