import asyncio
import os
import re
import shutil
from abc import abstractmethod
from collections import namedtuple
from queue import Empty, Queue
from threading import Thread

from secedgar.client import NetworkClient
from secedgar.exceptions import EDGARQueryError
from secedgar.filings._base import AbstractFiling
from secedgar.utils import make_path


class IndexFilings(AbstractFiling):
    """Abstract Base Class for index filings.

    Args:
        client (secedgar.client._base, optional): Client to use. Defaults to
            ``secedgar.client.NetworkClient``.

        entry_filter (function, optional): A boolean function to determine
            if the FilingEntry should be kept. E.g. `lambda l: l.form_type == "4"`.
            Defaults to `None`.
        kwargs: Any keyword arguments to pass to ``NetworkClient`` if no client is specified.
    """

    def __init__(self, client=None, entry_filter=None, **kwargs):
        super().__init__()
        self._client = client if client is not None else NetworkClient(**kwargs)
        self._listings_directory = None
        self._master_idx_file = None
        self._filings_dict = None
        self._paths = []
        self._urls = {}
        self._entry_filter = entry_filter

    @property
    def entry_filter(self):
        """A boolean function to be tested on each listing entry.

        This is tested regardless of download method.
        """
        return self._entry_filter

    @property
    def client(self):
        """``secedgar.client._base``: Client to use to make requests."""
        return self._client

    @property
    def params(self):
        """Params should be empty."""
        return {}

    @property
    @abstractmethod
    def year(self):
        """Passed to children classes."""
        pass  # pragma: no cover

    @property
    @abstractmethod
    def quarter(self):
        """Passed to children classes."""
        pass  # pragma: no cover

    @property
    @abstractmethod
    def idx_filename(self):
        """Passed to children classes."""
        pass  # pragma: no cover

    @abstractmethod
    def _get_tar(self):
        """Passed to child classes."""
        pass  # pragma: no cover

    @property
    def tar_path(self):
        """str: Tar.gz path added to the client base."""
        return "Archives/edgar/Feed/{year}/QTR{num}/".format(year=self.year, num=self.quarter)

    def _get_listings_directory(self, update_cache=False, **kwargs):
        """Get page with list of all idx files for given date or quarter.

        Args:
            update_cache (bool, optional): Whether quarterly directory should update cache. Defaults
                to False.
            kwargs: Any keyword arguments to pass to the client's `get_response` method.

        Returns:
            response (requests.Response): Response object from page with all idx files for
                given quarter and year.
        """
        if self._listings_directory is None or update_cache:
            self._listings_directory = self.client.get_response(self.path, self.params, **kwargs)
        return self._listings_directory

    def _get_master_idx_file(self, update_cache=False, **kwargs):
        """Get master file with all filings from given date.

        Args:
            update_cache (bool, optional): Whether master index should be updated
                method call. Defaults to False.
            kwargs: Keyword arguments to pass to
                ``secedgar.client._base.AbstractClient.get_response``.

        Returns:
            text (str): Idx file text.

        Raises:
            EDGARQueryError: If no file of the form master.<DATE>.idx
                is found.
        """
        if self._master_idx_file is None or update_cache:
            if self.idx_filename in self._get_listings_directory().text:
                master_idx_url = "{path}{filename}".format(
                    path=self.path, filename=self.idx_filename)
                self._master_idx_file = self.client.get_response(
                    master_idx_url, self.params, **kwargs).text
            else:
                raise EDGARQueryError("""File {filename} not found.
                                     There may be no filings for the given day/quarter.""".format(
                    filename=self.idx_filename))
        return self._master_idx_file

    def get_filings_dict(self, update_cache=False, **kwargs):
        """Get all filings inside an idx file.

        Args:
            update_cache (bool, optional): Whether filings dict should be
                updated on each method call. Defaults to False.

            kwargs: Any kwargs to pass to _get_master_idx_file. See
                ``secedgar.filings.daily.DailyFilings._get_master_idx_file``.
        """
        if self._filings_dict is None or update_cache:
            idx_file = self._get_master_idx_file(**kwargs)
            # Will have CIK as keys and list of FilingEntry namedtuples as values
            self._filings_dict = {}
            FilingEntry = namedtuple(
                "FilingEntry", ["cik", "company_name", "form_type", "date_filed", "file_name",
                                "path"])
            # idx file will have lines of the form CIK|Company Name|Form Type|Date Filed|File Name
            entries = re.findall(r'^[0-9]+[|].+[|].+[|][0-9\-]+[|].+$', idx_file, re.MULTILINE)
            for entry in entries:
                fields = entry.split("|")
                path = "Archives/{file_name}".format(file_name=fields[-1])
                entry = FilingEntry(*fields, path=path)
                if self.entry_filter is not None and not self.entry_filter(entry):
                    continue
                # Add new filing entry to CIK's list
                if entry.cik in self._filings_dict:
                    self._filings_dict[entry.cik].append(entry)
                else:
                    self._filings_dict[entry.cik] = [entry]
        return self._filings_dict

    def get_urls(self):
        """Get all URLs for day.

        Expects client _BASE to have trailing "/" for final URLs.

        Returns:
            urls (list of str): List of all URLs to get.
        """
        if not self._urls:
            filings_dict = self.get_filings_dict()
            self._urls = {company: [self.client._prepare_query(entry.path) for entry in entries]
                          for company, entries in filings_dict.items()}
        return self._urls

    @staticmethod
    def _do_create_and_copy(q):
        """Create path and copy file to end of path.

        Args:
            q (Queue.queue): Queue to get filename, new directory,
                and old path information from.
        """
        while True:
            try:
                filename, new_dir, old_path = q.get(timeout=1)
            except Empty:
                return
            make_path(new_dir)
            path = os.path.join(new_dir, filename)
            shutil.copyfile(old_path, path)
            q.task_done()

    @staticmethod
    def _do_unpack_archive(q, extract_directory):
        """Unpack archive file in given extract directory.

        Args:
            q (Queue.queue): Queue to get filname from.
            extract_directory (): Where to extract archive file.
        """
        while True:
            try:
                filename = q.get(timeout=1)
            except Empty:
                return
            shutil.unpack_archive(filename, extract_directory)
            os.remove(filename)
            q.task_done()

    def _unzip(self, extract_directory):
        """Unzips files from tar files into extract directory.

        Args:
            extract_directory (str): Temporary path to extract files to.
                Note that this directory will be completely removed after
                files are unzipped.
        """
        tar_files = self._get_tar()
        unpack_queue = Queue(maxsize=len(tar_files))
        unpack_threads = len(tar_files)

        for _ in range(unpack_threads):
            worker = Thread(target=self._do_unpack_archive,
                            args=(unpack_queue, extract_directory))
            worker.start()
        for f in tar_files:
            full_path = os.path.join(extract_directory, f)
            unpack_queue.put_nowait(full_path)

        unpack_queue.join()

    def _move_to_dest(self, urls, extract_directory, directory, file_pattern, dir_pattern):
        """Moves all files from extract_directory into proper final format in directory.

        Args:
            urls (dict): Dictionary of URLs that were retrieved. See
            ``secedgar.filings._index.IndexFilings.get_urls()`` for more.
            extract_directory (str): Location of extract directory as used in
                ``secedgar.filings.daily.DailyFilings._unzip``
            directory (str): Final parent directory to move files to.
            dir_pattern (str): Format string for subdirectories. Default is `{cik}`.
                Valid options are `cik`. See ``save`` method for more.
            file_pattern (str): Format string for files. Default is `{accession_number}`.
                Valid options are `accession_number`. See ``save`` method for more.
        """
        # Allocate threads to move files according to pattern
        link_list = [item for links in urls.values() for item in links]

        move_queue = Queue(maxsize=len(link_list))
        move_threads = 64
        for _ in range(move_threads):
            worker = Thread(target=self._do_create_and_copy, args=(move_queue,))
            worker.start()

        (_, _, extracted_files) = next(os.walk(extract_directory))

        for link in link_list:
            link_cik = link.split('/')[-2]
            link_accession = self.get_accession_number(link)
            filepath = link_accession.split('.')[0]
            possible_endings = ('nc', 'corr04', 'corr03', 'corr02', 'corr01')
            for ending in possible_endings:
                full_filepath = filepath + '.' + ending
                # If the filepath is found, move it to the correct path
                if full_filepath in extracted_files:

                    formatted_dir = dir_pattern.format(cik=link_cik)
                    formatted_file = file_pattern.format(
                        accession_number=link_accession)
                    old_path = os.path.join(extract_directory, full_filepath)
                    full_dir = os.path.join(directory, formatted_dir)
                    move_queue.put_nowait((formatted_file, full_dir, old_path))
                    break
        move_queue.join()

    def save_filings(self,
                     directory,
                     dir_pattern="{cik}",
                     file_pattern="{accession_number}",
                     download_all=False):
        """Save all filings.

        Will store all filings under the parent directory of ``directory``, further
        separating filings using ``dir_pattern`` and ``file_pattern``.

        Args:
            directory (str): Directory where filings should be stored.
            dir_pattern (str): Format string for subdirectories. Default is `{cik}`.
                Valid options are `{cik}`.
            file_pattern (str): Format string for files. Default is `{accession_number}`.
                Valid options are `{accession_number}`.
            download_all (bool): Type of downloading system, if true downloads all tar files,
                if false downloads each file in index. Default is `False`.
        """
        urls = self._check_urls_exist()

        if download_all:
            # Download tar files into huge temp directory
            extract_directory = os.path.join(directory, 'temp')
            i = 0
            while os.path.exists(extract_directory):
                # Ensure that there is no name clashing
                extract_directory = os.path.join(directory, 'temp{i}'.format(i=i))
                i += 1

            make_path(extract_directory)
            self._unzip(extract_directory=extract_directory)
            self._move_to_dest(urls=urls,
                               extract_directory=extract_directory,
                               directory=directory,
                               file_pattern=file_pattern,
                               dir_pattern=dir_pattern)

            # Remove the initial extracted data
            shutil.rmtree(extract_directory)
        else:
            inputs = []
            for company, links in urls.items():
                formatted_dir = dir_pattern.format(cik=company)
                for link in links:
                    formatted_file = file_pattern.format(
                        accession_number=self.get_accession_number(link))
                    path = os.path.join(directory, formatted_dir, formatted_file)
                    inputs.append((link, path))
            loop = asyncio.get_event_loop()
            loop.run_until_complete(self.client.wait_for_download_async(inputs))
