"""
.. include:: ../docs/file_helper.md
"""
from simple_progress import progress
from pyaddons import file_helper, gzopen
from collections import OrderedDict
import sys
import csv
import os
import gzip
import re
import warnings
import hashlib
import tempfile
import glob
import string
import shutil

csv.field_size_limit(sys.maxsize)

# Additional extensions that we look for after spitext
EXTENSIONS = ('.txt', '.tsv', '.tab', 'csv')


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def get_output_directory(indir, outdir, levels=0):
    """
    Build an output directory based on the options, if outdir is `NoneType`
    then indir is the outdir. If outdir is supplied then it is used as the
    output directory if `levels==0`. However, if `levels > 0`, then `levels`
    number of directories are taken from the indir and added to the outdir.
    For example, is indir is `/this/is/my/dir` and output direcory is
    `/my/output/dir`, if `levels == 1` then output dir would be
    `/my/output/dir/dir`. if `levels == 2` then output dir would be
    `/my/output/dir/my/dir`. If levels is set > than the number of levels
    that exist, then it will just go down to the bottom level but will not
    error out.

    Parameters
    ----------
    indir : str
        The input directory
    outdir : str or NoneType
        A specific output directory. If `NoneType`, then the output
        directory will be gathered from the location of the `indir`
    levels : int, optional
        add #levels down from the input file to the outdir (default=0)

    Returns
    -------
    outdir : str
        The output directory
    """
    if outdir is None:
        return indir

    if levels > 0:
        outpath = ""
        for level in range(levels):
            subdir = os.path.basename(indir)
            indir = os.path.dirname(indir)
            outpath = os.path.join(subdir, outpath)
        outdir = os.path.join(outdir, outpath)

    return outdir


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def outfile_from_regexp(infile, regexp, delimiter="_"):
    """
    Generate an output file name from a regexp and group capture of the input
    file name.

    Parameters
    ----------
    infile : str
        The input file name, this will be converted to a full path and the
        regexp will be applied to it
    regexp : str
        The regexp should be a precompiled regexp that will be applied using
        re.search. All groups will be captured, although the groups required
        in the output file name should be labeled "A", "B", "C"" using
        the named capure groups `(?P<A>)` etc... . Remember to also use
        non-capturing groups `(?:)` where necessary.
    delimiter : str, optional
        The character to use to join all the regexp extracted groups
        together with (default="_")

    Returns
    -------
    outfile : str
        The output file as defined by the regexp

    Raises
    ------
    ValueError
        If no groups were captured from the regular expression
    """
    # Ensure that the outout file is a full path
    infile = os.path.realpath(infile)

    outreg = regexp.search(infile)

    outfile_components = []

    for i in string.ascii_uppercase:
        try:
            outfile_components.append(outreg.group(i))
        except (IndexError, AttributeError):
            # Can't match groups either because the regexp did not match
            # at all or we have no more groups to match
            break

    if len(outfile_components) == 0:
        raise ValueError("could not match regexp to filename: '{0}'".format(
            infile))

    return delimiter.join(outfile_components), len(outfile_components)


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def gather_input_files(infiles=[], indir=None, inglob=None):
    """
    Using the arguments get all the input files requested by the user

    Parameters
    ----------
    infiles : list of str, optional
        A list of input files, this will be merged with indir contents
    indir : str or NoneType, optional
        An input directory
    inglob : str or NoneType, optional
        Used in conjunction with `indir`. If `NoneType`, then all files in
        the root of `indir` are gathered, otherwise a glob of `indir` and
        `inglob` is used to gather files

    Returns
    -------
    files : list of str
        A list of full file names

    Raises
    ------
    ValueError
        If the arguments produce an empty list of files
    """
    # Make sure all the input files have the absolute path
    all_files = [os.path.abspath(os.path.expanduser(i)) for i in infiles]

    # if an input directory has been supplied
    if indir is not None:
        # Make sure that the input directory is set to the absolute path
        indir = os.path.abspath(os.path.expanduser(indir))

        # Now if we are not globing anything, then list all the files in the
        # input directory
        if inglob is None:
            for f in os.listdir(indir):
                f = os.path.join(indir, f)
                if os.path.isfile(f):
                    all_files.append(f)
        else:
            # Get any globed file names
            for f in glob.glob(os.path.join(indir, inglob)):
                if os.path.isfile(f):
                    all_files.append(f)

    if len(all_files) == 0:
        raise ValueError("no input files found")

    return all_files


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def get_file_components(infile):
    """
    This takes a file name, makes it a realpath and then splits it into
    dirname, filename, extension . The extension goes a bit beyond splitext as
    it will look for things like .txt.gz instead of just gz

    Parameters
    ----------
    infile :str
        The file to process

    Returns
    -------
    directory :str
        The directory location of the file
    file_name :str
        The base name for the file with the extension removed
    extension :str
        The extension of the file
    """
    # First make sure the input file is represented as a full path
    infile = os.path.realpath(infile)

    # Now get the dirname
    directory = os.path.dirname(infile)
    file_name = os.path.basename(infile)

    file_name, extension = os.path.splitext(file_name)

    # there are a lot of .txt.gz files out there so just check and if so
    # do it again
    for i in EXTENSIONS:
        if file_name.endswith(i):
            file_name, other_extension = os.path.splitext(file_name)
            extension = other_extension + extension
            break

    return directory, file_name, extension


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def md5_file(file_name, chunksize=4096, verbose=False):
    """
    Get the MD5 of a file, this reads the file in chunks and accumilates the
    MD5sum to prevent loading the whole lot into memory. Taken from
    [here](https://stackoverflow.com/questions/3431825)

    Parameters
    ----------
    file_name : str
        A file name to check the MD5 sum
    chunk : int, optional
        The size of the chunks to read from the file (default=4096 bytes)
    verbose : bool, optional
        If the file is huge then this could take a while. Setting verbose to
        try will output a remaining progress monitor if needed (default=False)

    Returns
    -------
    md5sum : str
        The md5 hash of the file (hex)
    """
    statinfo = os.stat(file_name)

    prog = progress.RemainingProgress(statinfo.st_size, verbose=verbose)
    hash_md5 = hashlib.md5()
    with open(file_name, "rb") as f:
        prog.log_start()
        while True:
            chunk = f.read(chunksize)
            prog.next(lines=chunksize)
            prog.animate()
            if not chunk:
                break
            hash_md5.update(chunk)
        prog.log_finish()
    return hash_md5.hexdigest()


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def temp_dir_file(use_gzip=False, **kwargs):
    """
    This generates a temp directory with a temp file within it

    Parameters
    ----------
    use_gzip : bool, optional
        Do you want the temp file to be openned as a gzip file?
        (default=`False`)
    **kwargs
        The arguments accepted by tempfile.mkstemp and tempfile.mkdtemp. Note
        the parent argument will only be applied to `tempfile.mkdtemp`

    Returns
    -------
    temp_file_obj : :obj:`File`
        A file object, will be a gzip file object if gzip is `True`
    temp_file_name : str
        the name of the temp file
    temp_dir : str
        The name of the temp directory
    """
    temp_dir_args = {}
    for kw in ['suffix', 'prefix', 'dir']:
        try:
            temp_dir_args[kw] = kwargs[kw]
        except KeyError:
            pass

    temp_dir = tempfile.mkdtemp(**temp_dir_args)

    kwargs['dir'] = temp_dir

    temp_file_obj, temp_file_name = temp_file(use_gzip=use_gzip, **kwargs)
    return temp_file_obj, temp_file_name, temp_dir


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def temp_file(use_gzip=False, **kwargs):
    """
    Create a temp file but return a regular file object instead of an `os.open`
    level one

    Parameters
    ----------
    use_gzip : bool, optional
        Do you want the temp file to be openned as a gzip file?
        (default=`False`)
    **kwargs
        The arguments accepted by `tempfile.mkstemp`

    Returns
    -------
    temp_file_obj: :obj:`File`
        A file object
    temp_file_name :str
        the name of the temp file, will be a gzip file object if gzip is `True`
    """
    mode = kwargs.pop('mode', 'wt')
    # gzip_file = kwargs.pop('gzip', False)

    if 'w' not in mode:
        mode = 'w{0}'.format(mode)

    temp_file_obj, temp_file_name = tempfile.mkstemp(**kwargs)

    if use_gzip is False:
    # if gzip_file is False:
        return os.fdopen(temp_file_obj, mode), temp_file_name
    else:
        # TODO: Is it possible to convert the file description to a file and
        # TODO: that to gzipped? without closing and re-openning?
        os.close(temp_file_obj)
        # TODO: What about mode??
        # return gzip.open(os.fdopen(temp_file_obj, mode), mode), temp_file_name
        return gzip.open(temp_file_name, mode), temp_file_name


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def get_dialect(infile, sniff_bytes=8192, encoding='utf-8', **kwargs):
    """
    Determine the dialect of the file based on the arguments to the
    object and/or sniffing out the real dialect

    Parameters
    ----------
    infile : str
        The input file to check
    sniff_bytes : int, optional
        If no delimiter is specified, then this is the number of bytes that
        is read in from the file when attemping to detect the delimiter
        (default=8192)

    Returns
    -------
    dialect : :obj:`csv.Dialect`
        A csv dialect object that will contain the delimiter value.
        note that if a delimiter is supplied to the object then it is
        used directly in the dialect
    """
    # Now we want to determine the file delimiter
    with gzopen.gzip_agnostic(infile, **kwargs) as csvfile:
        dialect = csv.Sniffer().sniff(csvfile.read(sniff_bytes))
    return dialect


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# def detect_header(infile, delimiter='\t', test_rows=50):
def detect_header(infile, dialect=None, sniff_bytes=8192):
    """
    Attempt to detect if the file has a header. This reads `sniff_bytes`
    of data from the file and attempts to detect the header using
    `csv.Sniffer().has_header()`. I am not 100% sure how good this approach
    is. It also, returns the first row (which may or may not be the header)
    If the dialect is supplied then that is used to read the first row. If
    not then a dialect is sniffed out.

    Parameters
    ----------
    infile : str
        The input file to check
    dialect : :obj:`csv.Dialect`, optional
        A dialect to use as parameters to extract the first row
    sniff_bytes : int, optional
        If no delimiter is specified, then this is the number of bytes that
        is read in from the file when attemping to detect the delimiter
        (default=8192)

    Returns
    -------
    first_row : list of str
        The first row from the file, this may or may not be the header
    has_header : bool
        `True` if the sniffer thinks we have a header `False` if not
    """
    # Open the file being agnistic for gzip
    with gzopen.gzip_fh(infile) as csvfile:
        if dialect is None:
            # have a sniff for the dialect so I can use this later to return
            # the first row (that may well be the header)
            dialect = csv.Sniffer().sniff(csvfile.read(sniff_bytes))

        # Move back to the begining and attempt to sniff out the header
        csvfile.seek(0)
        has_header = csv.Sniffer().has_header(
            csvfile.read(sniff_bytes))

        # Now move back to the begining and attempt to get the first row
        csvfile.seek(0)
        reader = csv.reader(csvfile, dialect=dialect)
        first_row = next(reader)

    # return the first row (which may or maynot be a header) and a
    # bool indicating if the sniffer thinks we have a header
    return first_row, has_header


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def check_header(header, columns=None, previous_header=None):
    """
    Make sure the header is valid. if columns are defined then these
    are checked for their presence in the `header`. If they are not
    defined and previous_header is not `NoneType`, then the headers are
    checked to see if they are an exact match. So if both `columns` and
    `previous_header` are `NoneType` then this has the effect of just
    adding index numbers to the header names.

    Parameters
    ----------
    header : list of str
        The header to check
    columns : NoneType or list of str or tuple, optional
        If `NoneType`, then we check to see if the `header` is an exact
        match to `previous_header` (if that is defined). Otherwise it
        should be a subset of columns that are required to be present
        in the header. In this case the header can be in any order. If the
        list element is a `str`, then it is assumed that the column name
        must appear in all files. If it is a `tuple` then it is a group
        of possible column names where exactly one of them should be in the
        header. It is an error if > than one appears (default=`NoneType`)
    previous_header : list of str or NoneType, optional
        A previous header to check against. If not supplied then no check
        is made (default=`NoneType`)

    Returns
    -------
    found_columns : list of tuple
        A list of the columns that have been found in the header,
        each tuple contains the column name that has been found `[0]`
        and the column index in the file at `[1]`

    Raises
    ------
    ValueError
        If the column is not found in the `header` or too many columns
        are found in the `header` or if the `header` does not equal the
        `previous_header`
    RuntimeError
        If the format of `columns` can't be processed
    """
    # if there are columns then attempt to identify them
    found_columns = []

    if columns is not None:
        for c in columns:
            # If the column element is a string (i.e. a single column)
            # then attempt to find the index of it, this will raise
            # ValueError if .index() fails
            if isinstance(c, str):
                found_columns.append((c, header.index(c)))
            elif isinstance(c, (tuple, list)):
                # If we have a group of columns, see how many of them
                # occur in the header
                noccurs = sum([i in header for i in c[1]])
                if noccurs == 0:
                    raise ValueError("{0} not in header".format(
                        ','.join(c[1])))
                elif noccurs > 1:
                    raise ValueError("{0} too many columns in header "
                                     "'{1}'".format(','.join(c[1]),
                                                    noccurs))

                # If we get here then we will have exactly 1 occurence in
                # the header
                found_columns.append((c, header.index(c)))
            else:
                raise RuntimeError("bad column format")

        return found_columns

    # If no specific column checks are being carried out then we will
    # look at the header as a whole
    if previous_header is not None and header != previous_header:
        raise ValueError("current header does not match previous "
                         "header")

    # Otherwise we return the column name and the
    # column index for each item in the header
    return [(h, c) for c, h in enumerate(header)]


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def check_file_contents(infile, error=True, **kwargs):
    """
    This will go through the file and store the numimal number of columns
    detected and the maximal number of columns detected. However, if
    `error=True` then it will error out if `min_cols` != `max_cols`

    Parameters
    ----------
    infile : str
        The input file to check
    error : bool, optional
        Error out if min_col != max_col, if `False`, then it will return
        the `nrows`, `min_col` and `max_col` values
    **kwargs
        Keyword arguments to `csv.reader`

    Returns
    -------
    nrows : int
        The number of rows in the file
    min_cols : int
        The minimum number of columns detected in the file
    max_cols : int
        The maximum number of columns detected in the file

    Raises
    ------
    ValueError
        If error is `True` and `min_cols` != `max_cols`
    StopIteration
        error is `True` and the file is empty
    """
    min_cols = 0
    max_cols = 0
    with gzopen.gzip_fh(infile) as csvfile:
        reader = csv.reader(csvfile, **kwargs)

        try:
            header = next(reader)
        except StopIteration:
            if error is True:
                raise
            else:
                return 0, 0, 0

        min_cols = len(header)
        max_cols = len(header)

        for row_no, row in enumerate(reader):
            ncols = len(row)
            min_cols = min(min_cols, ncols)
            max_cols = max(max_cols, ncols)

            if error is True and min_cols != max_cols:
                raise IndexError(
                    "min_cols '{0}' != max_cols '{1}' at "
                    "line '{2}': {3}".format(min_cols,
                                             max_cols,
                                             row_no+2,
                                             infile))
    return row_no+2, min_cols, max_cols


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def check_file(infile, ncols=False, md5hash=False, sniff_bytes=8192,
               columns=None, dialect=None, regexp_id=None):
    """
    Perform a standard set of checks on a file and return a summary of
    the checks. The following checks are performed:

    * Does the file exist?
    * The MD5 hash of the file (if `md5hash` is `True`)
    * Is the file gzipped? I am not 100% sure if this works with
      bgzipped files yet
    * The `csv.Dialect`
    * The delimiter (extracted from the dialect)
    * Does the file have a header?
    * The header length (in columns) - this will be the first line
      length if the file does not have a header
    * The number of rows (if ncols is `True`)
    * The min/max number of columns in the file (if ncols is `True`)
    * A regexp identifier (if regexp_id is supplied)

    Parameters
    ----------
    infile : str
        The file name of the file to check
    ncols : bool, optional
        If `True` then the min number of columns and the max number of
        columns will be determined for each file. If `False` then these
        will be set to `NoneType` in the result (default=`False`)
    md5hash : bool, optional
        Generate an MD5 hash for each file (default=`False`)
    sniff_bytes :int, optional
        If no delimiter is specified, then this is the number of bytes that
        is read in from the file when attemping to detect the delimiter
        (default=8192)
    columns : NoneType or list of str or tuple, optional
        A subset of columns that are required to be present, rather than
        all of the columns. If None, then all columns are checked to see if
        the headers of each file are the same. If columns are defined, then
        the header can appear in any order. If the list element is a str,
        then it is assumed that the column name must appear in all files.
        If it is a tuple then is is a group of possible column names that
        will appear in each file. It is assumed that only one of the
        possible column names should appear in each file and it is an error
        if > than one appears (default=`NoneType`)
    dialect : :obj:`csv.Dialect` or NoneType, optional
        A csv dialect object that will contain the delimiter value. Note that
        if a dialect is supplied then it is used and we do not attempt to sniff
        one out (default=`NoneType`)
    regexp_id : :obj:`re.Pattern` or NoneType, optional
        This generates an ID column based on a REGEXP applied to the input
        file name, the regexps must create unique IDs within all the files
        this is done prior to merge. (default=`NoneType`)

    Returns
    -------
    result : :obj:`OrderedDict`
        A result of the checks that has the following keys
        (in this order): `filename`, `file_exists`, `is_gzip`,
        `delimiter`, `has_header`, `header_len`, `columns_ok`, `nrows`,
        `min_cols`, `max_cols`, `md5hash`, `regexp_id`, `has_errors`,
        `dialect`, `columns`, `header_row`. Where dialect is a
        `csv.Dialect` object

    Raises
    ------
    ValueError
        If the format of columns is incorrect
    """

    # Get the dict to store all of the results from all the tests
    result = OrderedDict.fromkeys(FileInfo.CHECK_RESULT_KEYS)

    # At present we set the check results to have no errors
    result['has_errors'] = False

    # Make sure the input file is represented as a full path
    infile = os.path.realpath(os.path.expanduser(infile))
    result['filename'] = infile

    try:
        # First attempt to open the file and close it, this will
        # generate file based errors that we will either log or raise
        open(infile).close()
        result['file_exists'] = True
    except (FileNotFoundError, PermissionError):
        result['file_exists'] = False
        result['has_errors'] = True

        # if we are not erroring out there is not much more we can
        # do as we can't access the file so all the other checks
        # will be None and we will move onto the next file
        return result

    # If the md5hash option is enabled then take it
    result['md5hash'] = None
    if md5hash is True:
        result['md5hash'] = file_helper.md5_file(infile)

    # Determine if the file is gzipped
    result['is_gzip'] = gzopen.is_gzip(infile)

    if dialect is None:
        result['dialect'] = get_dialect(infile, sniff_bytes=sniff_bytes)
        result['delimiter'] = result['dialect'].delimiter

    result['header_row'], result['has_header'] = detect_header(
        infile,
        dialect=result['dialect'],
        sniff_bytes=sniff_bytes)

    # Store the length of the header
    result['header_len'] = len(result['header_row'])

    # Now process the header, is it the same as the previous
    # headers or if column groups have been supplied does a
    # single column from the group match a column in the header
    try:
        result['columns'] = check_header(result['header_row'],
                                         columns=columns)
        result['columns_ok'] = True
    except ValueError:
        # There are issues with the header
        result['columns_ok'] = False
        result['has_errors'] = True
    except RuntimeError as e:
        # Reraise errors due to the formatting of the search columns
        # as a ValueError and then add the file name
        raise ValueError("{0}: {1}".format(str(e), infile)) from e

    if ncols is True:
        # Now check the file contents to get rows/columns. depending on
        # error= this may raise ValueError
        result['nrows'], result['min_cols'], result['max_cols'] = \
            check_file_contents(infile, error=False,
                                dialect=result['dialect'])

        if result['nrows'] == 0 or result['min_cols'] != result['max_cols']:
            result['has_errors'] = True

    # If we are generating REGEXP identifiers
    if regexp_id is not None:
        # generate
        result['regexp_id'], ncomponents = \
            outfile_from_regexp(infile, regexp_id)

        # If we do not match any of the regular expression then that
        # classifies as an error
        if ncomponents == 0:
            # If we are not erroring out then we just log that we
            # have had an error
            result['has_errors'] = True

    return result


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class FlexiWriter(object):
    """
    An attempt at a flexible file writer that will either write to STDOUT or a
    file depending on parameters. If writing to a file this is all done into
    a temp file that is moved to the final file upon successful execution.
    """
    ALLOWED_MODES = ['b', 't']

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __init__(self, filename, **kwargs):
        """
        Parameters
        ----------

        filename : str or NoneType
            The file name to write to, if `NoneType`, we setup to write to
            STDOUT
        mode : str, optional
            The writing mode must be `t` (the default) or `b`
        force_gzip : bool, optional
            Usually a GZIP file is created if the output file has the
            extension .gz or .bgz . However, if True this will force the
            output file to be gzipped (default=`False`)
        dir : str or NoneType, optional
            All writing to a file happens via a temp file that is moved upon
            successful execution to the final location. This is the parental
            location of the temp file as in `tempfile.mkstemp`
        prefix : str or NoneType
            A prefix to add onto the temp file and temp directory if being used

        Raises
        ------
        ValueError
            if the mode is not one of the allowed modes
        """
        # Store the file name and an alias
        self.filename = filename
        self.name = filename

        # The write object will store a csv writer
        self.write_obj = None

        # Are we writting to STDOUT
        self._stdout_write = False

        # If the filename is None then we are writing to STDOUT so we adjust
        # out alias and indicate in the boolean flag that we are writing to
        # STDOUT
        if self.filename is None:
            self.name = 'STDOUT'
            self._stdout_write = True
        else:
            # Otherwise we make sure that we have the full path to the file
            self.filename = os.path.abspath(os.path.expanduser(self.filename))
            self.name = self.filename

        # Now error check the writting mode
        self.mode = kwargs.pop('mode', 't')
        if self.mode not in self.__class__.ALLOWED_MODES:
            raise ValueError("mode should be one of '{0}'".format(
                ','.join(self.__class__.ALLOWED_MODES)))

        # Define if we want to write a GZIP file
        self.gzip = kwargs.pop('force_gzip', False)
        if re.search(r'\.b?gz$', self.name):
            self.gzip = True

        self.dir = kwargs.pop('dir', None)
        self.prefix = kwargs.pop('prefix', None)

        # Any other arguments to csv
        self.kwargs = kwargs

        # Indicator that we are open
        self._opened = False

        # Indicates that the file has been relocated to the final location
        self._moved = False

        # This a flag that will be set if we get into an error state
        self._error = False

        # Are we using via the context manager?
        self._using_context_manager = False

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __enter__(self):
        """
        The entry point for the context manager
        """
        # Opent the file and indicate that we are using the context manager
        self.open()
        self._using_context_manager = True
        return self

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __exit__(self, exc_type, exc_value, exc_traceback):
        """
        The exit point for the context manager

        Parameters
        ----------
        exc_type
        exc_value
        exc_traceback
        """
        if exc_type is None:
            self.close()
            self.move()
        else:
            self._error = True
            self.close()
            self.delete()

        # We are not using the context manager anymore
        self._using_context_manager = False

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def open(self):
        """
        Set up the flexiwriter for writing

        Raises
        ------
        IOError
            If the FlexiWriter is already open
        """
        # If we have already openned the file then error out
        if self._opened is True:
            raise IOError("FlexiWriter already open")

        # If we are not writing to STDOUT then we aill want to write to a temp
        # file
        if self._stdout_write is False:
            # If we want our temp file to be located in a temp directory
            # then set it up
            # Otherwise we are writing to a temp file that is not located
            # in a specific temp directory
            self.write_obj, self.temp_file = temp_file(
                prefix=self.prefix,
                use_gzip=self.gzip,
                dir=self.dir,
                mode='w{0}'.format(self.mode))
        else:
            # We want to write to STDOUT
            self.write_obj = sys.stdout
            self.temp_file = None

        self.csv_writer = csv.writer(self.write_obj, **self.kwargs)

        # Indicate that we are now open
        self._opened = True

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def close(self):
        """
        Close the FlexiWriter

        Raises
        ------
        IOError
            If the FlexiWriter is not open
        """
        if self._opened is False:
            raise IOError("FlexiWriter not open")

        if self._stdout_write is False:
            # close and sort out the temp files
            self.write_obj.close()

        # Indicate that we are not open
        self._opened = False

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def move(self):
        """
        Move the file to it's final location

        Raises
        ------
        IOError
            If the FlexiWriter is open or the file has already been moved
        """
        if self._opened is True:
            raise IOError("FlexiWriter is still open, please close first")

        if self._moved is True:
            raise IOError("file already moved")

        # If we are writing to STDOUT then there is nothing to move
        if self._stdout_write is False:
            shutil.move(self.temp_file, self.filename)

        # Indicate tat the file is moved
        self._moved = True

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def delete(self):
        """
        Delete the temp file. If the file has already been moved then it is an
        error

        Raises
        ------
        IOError
            If the FlexiWriter is open or the file has already been
            moved/deleted
        """
        if self._opened is True:
            raise IOError("FlexiWriter is still open, please close first")

        if self._moved is True:
            raise IOError("file already moved")

        # If we are writing to STDOUT then there is nothing to move
        if self._stdout_write is False:
            os.unlink(self.temp_file)
            # Even though we are deleting we will set moved to True once we
            # are done
            self._moved = True

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def writerow(self, *args):
        """
        Write a row with csv

        Parameters
        ----------
        *args
            The arguments that would normally be applied to
            `csv.writer.writerow`
        """
        self.csv_writer.writerow(*args)


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class FileInfo(object):
    """
    Gather information on a bunch of files and yield the result lines. The
    files should be tabular with a header. If no delimiter is specified then
    it is determined with `csv.Sniffer`. If all the files do have the same
    delimiter then a fixed delimiter can be supplied. Under default conditions
    it is assumed that the headers should be identical, however, there is an
    option to check if certain a column from a group of columns exists in each
    file, for example, there may be different column names that contain the
    same type of data (although the data is not checked).
    """

    # The keys for the file check result OrderedDict
    CHECK_RESULT_KEYS = ['filename', 'file_exists', 'is_gzip', 'delimiter',
                         'has_header', 'header_len', 'columns_ok', 'nrows',
                         'min_cols', 'max_cols', 'md5hash', 'regexp_id',
                         'has_errors', 'dialect', 'columns', 'header_row']

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __init__(self, infiles, columns=None, dialect=None, ncols=True,
                 md5hash=True, errors=True, regexp_id=None, sniff_bytes=8192):
        """
        Initialise with:

        Parameters
        ----------
        infiles :list of str
            Input files that need to be checked
        columns : NoneType or list of str or tuple, optional
            A subset of columns that are required to be present, rather than
            all of the columns. If None, then all columns are checked to see if
            the headers of each file are the same. If columns are defined, then
            the header can appear in any order. If the list element is a str,
            then it is assumed that the column name must appear in all files.
            If it is a tuple then is is a group of possible column names that
            will appear in each file. It is assumed that only one of the
            possible column names should appear in each file and it is an error
            if > than one appears (default=`NoneType`)
        dialect : :obj:`csv.Dialect` or NoneType, optional
            A csv dialect object that will contain the delimiter value.
            note that if a dialect is supplied to the object then it is
            used and we do not attempt to sniff one out (default=`NoneType`)
        ncols : bool, optional
            If `True` then the min number of columns and the max number of
            columns will be determined for each file. If `False` then these
            will be set to `NoneType` in the result (default=`True`)
        md5hash : bool, optional
            Generate an MD5 hash for each file (default=`True`)
        errors : bool, optional
            If any errors are detected raise them. If `False`, the errors
            are not raised but are evident from the results (default=`True`)
        regexp_id : :obj:`re.Pattern` or NoneType, optional
            This generates an ID column based on a REGEXP applied to the input
            file name, the regexps must create unique IDs within all the files
            this is done prior to merge. (default=`NoneType`)
        sniff_bytes :int, optional
            If no delimiter is specified, then this is the number of bytes that
            is read in from the file when attemping to detect the delimiter
            (default=8192)

        Raises
        ------
        ValueError
            If there are no input files supplied
        """
        self._infiles = infiles
        self._columns = columns
        self._dialect = dialect
        self._ncols = ncols
        self._md5hash = md5hash
        self._errors = errors
        self._regexp_id = regexp_id
        self._sniff_bytes = sniff_bytes

        # A flag to indicate if all the files are good
        self.all_files_ok = False
        self.passed_files = 0
        self._cur_idx = -1

        # This will store the regexp IDs so we can make sure they are
        # unique after the file names have been processed
        self._regexp_ids = []
        self._previous_header = None

        # Make sure that we actually have some input files
        if len(self._infiles) == 0:
            raise ValueError("there are no input files")

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __next__(self):
        """
        Called on each iteration

        Returns
        -------
        info : :obj:`OrderedDict`
            A result of the checks that has the following keys
            (in this order): `filename`, `file_exists`, `is_gzip`,
            `delimiter`, `has_header`, `header_len`, `columns_ok`, `nrows`,
            `min_cols`, `max_cols`, `md5hash`, `regexp_id`, `has_errors`,
            `dialect`, `columns`, `header_row`. Where dialect is a
            `csv.Dialect` object
        """
        try:
            self._cur_idx += 1
            return self._check_file(self._infiles[self._cur_idx])
        except IndexError as e:
            failed_files = len(self._infiles) - self.passed_files

            if failed_files > 0:
                warnings.warn("'{0}' files have errors".format(failed_files))

            raise StopIteration("all files processed") from e

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __iter__(self):
        """
        Will return a result for each call

        Returns
        -------
        info : :obj:`OrderedDict`
            A result of the checks that has the following keys
            (in this order): `filename`, `file_exists`, `is_gzip`,
            `delimiter`, `has_header`, `header_len`, `columns_ok`, `nrows`,
            `min_cols`, `max_cols`, `md5hash`, `regexp_id`, `has_errors`,
            `dialect`, `columns`, `header_row`. Where dialect is a
            `csv.Dialect` object
        """
        return self

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _check_file(self, infile):
        """
        Perform the file check

        Parameters
        ----------
        infile : str
            The file to check
        """
        # Use the external checking function to do the checks
        # TODO: Do I need columns here is is below OK - I think below is fine
        result = check_file(infile, ncols=self._ncols, md5hash=self._md5hash,
                            sniff_bytes=self._sniff_bytes,
                            dialect=self._dialect,
                            regexp_id=self._regexp_id)

        try:
            # Do another header check but we can set the previous header
            result['columns'] = check_header(
                result['header_row'],
                previous_header=self._previous_header,
                columns=self._columns)
            result['columns_ok'] = True
        except ValueError:
            # There are issues with the header
            result['columns_ok'] = False
            result['has_errors'] = True

        # Make sure the regexp ID is unique, if it is not then we error
        # out if the error option is on otherwise we register that we
        # have an error
        if self._regexp_id is not None and result['regexp_id'] in \
           self._regexp_ids:
            result['has_errors'] = True
            if self._errors is True:
                raise ValueError("regexp_id is not unique for file: "
                                 "{0}".format(infile))

        # If we want to error out as a result of failed checks, then we need
        # to evaluate the check results
        if self._errors is True and result['has_errors'] is True:
            raise IOError("file has errors")

        # We add the regexp_id to the group
        self._regexp_ids.append(result['regexp_id'])

        # Keep count of the number of files that have actually passed
        self.passed_files += not result['has_errors']

        self._previous_header = result['header_row']
        return result
