#!/usr/bin/env python
"""
.. include:: ../docs/file_info.md
"""
from pyaddons import __version__, __name__ as PKG_NAME
from pyaddons import file_helper
from simple_progress import progress
from terminaltables import AsciiTable
import argparse
import os
import sys
import re
import csv

# These are file info result keys that we do not want output when outputting
# to a file or the command line
EXCLUDE_KEYS = ['dialect', 'columns', 'header_row']

# The header when outputting to a file or the command line
HEADER = [i for i in file_helper.FileInfo.CHECK_RESULT_KEYS
          if i not in EXCLUDE_KEYS]


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def get_output_row(row, basename=False):
    """
    Return the components of the result row that we want to output. This
    takes a result `OrderedDict` and the basename requrement and produces a
    list that can be output to a file. Python, `NoneType`, `True` and
    `False` are mapped to `NA`, `Y` and `N` respectively. A physical tab
    '\t' is also mapped to the string `TAB` so it is visable on the
    command line.

    Parameters
    ----------
    row : :obj:`OrderedDict`
        A file info result row. The expected keys are: `filename`,
        `file_exists`, `is_gzip`, `delimiter`, `has_header`, `header_len`,
        `columns_ok`, `nrows`, `min_cols`, `max_cols`, `md5sum`,
        `regexp_id`, `has_errors`
    basename : bool, optional
        Convert the full path to a file basename (default=False)

    Returns
    -------
    out_row : list or str
        The data from the `OrderedDict` keys above with the python types
        converted to strings. If any `KeyErrors` occur than an `NA` is inserted
    """
    if basename is True:
        row['filename'] = os.path.basename(row['filename'])

        out_row = []
    if row['delimiter'] == "\t":
        row['delimiter'] = "TAB"

    for h in HEADER:
        try:
            if row[h] is None:
                row[h] = "NA"
            elif row[h] is True:
                row[h] = "Y"
            elif row[h] is False:
                row[h] = "N"
        except KeyError:
            # I am not expecting any of these but just in case
            row[h] = "NA"
        out_row.append(row[h])
    return out_row


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def init_cmd_line_args(description=None):
    """
    Use argparse to initialise the command line arguments. Note that this does
    not actually parse the arguments but returns the parser object so that
    other arguments can be added if needed.

    Parameters
    ----------
    description : str or NoneType, optional
        The description for the `--help` of the argument parse. If this is
        `NoneType` then the deault description is used.

    Returns
    -------
    args : :obj:`argparse.ArgumentParser`
        The parser with the default commandline arguments set. This can be
        added to in other functions if neded
    """

    if description is None:
        description = ('A helper script to merge a bunch of files to form a'
                       'single file. This acts like `cat` but is slower and '
                       'also performs more checks. So the files should be '
                       'tabular with all the same delimiter and header. '
                       'However, there is an option to supply groups of '
                       'column names that will act as a proxy for a single'
                       ' column type. It will also produce a merge log file '
                       'and there are options to leave the files being merged'
                       ' intact or to have them removed during merging, either'
                       ' as the merge is being done (--unsafe) or after the'
                       ' merge has been completed')

    parser = argparse.ArgumentParser(
        description=description)

    parser.add_argument('infiles', type=str, nargs='*',
                        help="The input files to extract from, It is expected"
                        "that all input files have a header. The files can be"
                        "compressed (gzip) or uncompressed (default=None)")
    parser.add_argument('-i', '--indir', type=str,
                        help="The input directory, this should be set if there"
                        "are no input files, it can also be used in "
                        "conjunction with input files (default=None)")
    parser.add_argument('-g', '--inglob', type=str,
                        help="An input directory glob that is used in "
                        "conjunction with the --indir, the default is no "
                        "glob pattern")
    parser.add_argument('-o', '--outfile', type=str,
                        help="The output file, if not provided, output is to "
                        "STDOUT, default is to STDOUT")
    parser.add_argument('-c', '--columns', type=str, nargs='+',
                        help="Specific column names to use from the files (if"
                        " they all have the same header) or groups of column"
                        " names that contain the same data, groups should be"
                        " given in the format:"
                        " 'input_col1,input_col2,input_colX"
                        "...' with no sapces and may need quoting. The idea"
                        " is that the presence of any columns or column groups"
                        " will be checked in each file. If single column names"
                        " are given, then they must be in each fil. If groups"
                        " of columns are set then a single column from the "
                        "group must be in each file. If no column names are"
                        " passed (the default) then the files are tested to "
                        "make sure they have exactly the same header. If "
                        "there are any violations of these then the "
                        "'has_errors' column is yes to 'Y' for the file.")
    parser.add_argument('-r', '--regexp', type=str,
                        help="A regular expression used to generate the "
                        "ID column from the full input file path. The"
                        " regexp will be applied using re.search. All groups"
                        " will be captured, although the groups required in "
                        "the ID should be labeled 'A', 'B', 'C' using the "
                        "named capure groups `(?P<C>)` etc... . "
                        "Remember to also use non-capturing groups `(?:)` "
                        "where necessary. (default=no regexp)")
    parser.add_argument('-d', '--delimiter', type=str,
                        help="The delimiter of the input files. Only use if "
                        "all input files have the same delimiter. If not, "
                        "the delimiter is sniffed using csv.Sniffer "
                        "(the default)")
    parser.add_argument('-t', '--out-delimiter', type=str, default="\t",
                        help="The delimiter of the output file (or STDOUT)."
                        "if it is set to - (hyphen) then a fixed width "
                        "terminal table will be output instead of a "
                        "delimited file (default='\t')")
    parser.add_argument('-m', '--md5', action='store_true',
                        help="Take an MD5 of each input file"
                        " (default=False)")
    parser.add_argument('-n', '--ncols', action='store_true',
                        help="calculate the min/max number of columns for "
                        "each file and also the number of rows (including the "
                        "header) (default=False)")
    parser.add_argument('-v', '--verbose', action='store_true',
                        help='Shall I print progress to STDERR? '
                        '(default=False)')
    parser.add_argument('-b', '--basename', action='store_true',
                        help="Shall I output the file file basename instead"
                        " of the full filename. Can be useful if the file "
                        "names are very long and you are outputting to STDOUT"
                        "(default=Full file path)")
    return parser


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def parse_cmd_line_args(parser):
    """
    Parse and process the command line arguments that have been defined
    using argparse. This will fail if there is no `delimiter`, `columns`,
    `delete`, `unsafe` or `regexp` argument, as they are all processed within
    it.

    Parameters
    ----------
    args : :obj:`argparse.ArgumentParser`
        The parser with at least `delimiter`, `columns`, `delete`, `unsafe` or
        `regexp` arguments set

    Returns
    -------
    args : :obj:`Namespace`
        The arguments from parsing the cmd line args
    """
    args = parser.parse_args()

    # Make a literal tab into a python tab, just in case the user passes '\t'
    # on the command line and not $'\t'
    if args.delimiter == r'\t':
        args.delimiter = "\t"

    # Make a literal tab into a python tab, just in case the user passes '\t'
    # on the command line and not $'\t'
    if args.out_delimiter == r'\t':
        args.delimiter = "\t"

    # Make sure that all the columns are parsed correctly
    args.columns = parse_column_arg(args.columns)

    # Compile the regular expression
    args.regexp = set_regexp(args.regexp)

    return args


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def set_regexp(regexp):
    """
    Compile the regexp that has been passed on the command line.

    Parameters
    ----------
    regexp :str or NoneType, optional
        A regular expression to compile or `Nonetype`. If it is `NoneType`,
        then it is simply returned, if not then we attempt to compile it.

    Returns
    -------
    compiled_regexp : :obj:`re.Pattern`
        A compiled regexp. Note that no flags are added to the compilation.
    """
    if regexp is not None:
        regexp = r'{0}'.format(regexp)
        return re.compile(regexp)
    return regexp


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def parse_column_arg(col_arg):
    """
    Parse the column args into the correct format of a `list` of `str` or
    `tuple`.

    Parameters
    ----------
    col_arg : list of str or NoneType
        This is the columns that have been parsed in from the command line.
        Each element will either be a single string or a string that
        represents a group of columns with the format:
        `input_col1,input_col2,input_colX...`. It could also be `NoneType`,
        in which case it is passed through.

    Return
    ------
    columns : NoneType or list of str or tuple
        A subset of columns that are required to be checked, rather than
        all of the columns. If `NoneType`, then all columns are checked and
        the headers of each file must be the same. If defined, then the header
        can appear in any order. If the list element is a `str`, then it is
        assumed that the column name must appear in all files. If it is a
        `tuple`, then it is a group of possible column names and only one of
        that group will appear in each file. It is an error if > than one
        appears or none appear (default=`NoneType`) (the whole header must
        be the same in each file)

    Raises
    ------
    ValueError
        If the column specification format does not look good.
    """
    columns = []
    try:
        for i in col_arg:
            # We force the element to be a string, just in case
            i = str(i)

            # We have some column proxies, so we go on to split them on
            # commas and remove any leading/lagging whitespace that could
            # be present
            col_args = [c.strip() for c in i.split(',')]

            # If there is a single argument it means that we are selecting a
            # single column that we expect to be in all output files
            if len(col_args) == 1:
                columns.append(col_args[0])
            elif len(col_args) > 1:
                columns.append(col_args)
            else:
                # I doubt this will ever get raised
                raise ValueError("unknown column format '{0}'".format(i))
    except TypeError:
        # col_arg is probably NoneType
        return None


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def main():
    """
    The main entry point for the script
    """
    # Initialise and parse the command line arguments
    parser = init_cmd_line_args()
    args = parse_cmd_line_args(parser)

    # Setup output messaging respecting verbosity and redirecting to STDOUT
    msg = progress.Msg(verbose=args.verbose, file=sys.stderr)
    msg.msg('=== finfo ({0} v{1}) ==='.format(PKG_NAME, __version__))
    msg.prefix = '[info]'

    # Now get the input files to merge
    args.infiles = file_helper.gather_input_files(infiles=args.infiles,
                                                  indir=args.indir,
                                                  inglob=args.inglob)

    # Output the program arguments (according to verbose)
    msg.msg_atts(args)

    # If a delimiter has been supplied then we convert it into a dialect
    if args.delimiter is not None:
        args.delimiter = csv.Dialect(delimiter=args.delimiter)

    # Now create the file info object. It is an iterator so at this stage
    # nothing happens
    file_info = file_helper.FileInfo(args.infiles, columns=args.columns,
                                     dialect=args.delimiter,
                                     ncols=args.ncols,  md5hash=args.md5,
                                     errors=False, regexp_id=args.regexp)

    # Setup a progress monitor based on the number of files, this is only
    # shown if verbose is True. It maynot be 100% accurate for remaining times
    # as the files could be very different sizes
    prog = progress.RemainingProgress(len(args.infiles),
                                      verbose=args.verbose,
                                      file=sys.stderr)

    # If we want to output a terminal table
    if args.out_delimiter == '-':
        # if the outfile is not supplied then we will be outputting the
        # terminal table to STDOUT
        if args.outfile is None:
            args.outfile = sys.stdout

        # When outputting the terminal table. all the result rows
        # (including the header) are buffered into a list and passed to
        # terminal tables in one go
        all_results = [HEADER]

        # Generate the checks on each file
        for result in prog.progress(file_info):
            all_results.append(get_output_row(result, basename=args.basename))

        # print with terminal tables to what ever location we have decided
        table = AsciiTable(all_results)
        print(table.table, file=args.outfile)
    else:
        # We want to output into some sort of delimited table
        try:
            # Determine where we want the temp directory. This is because we
            # are using the FlexiWriter class and that writes everything to a
            # temp location (if outfile is defined) and then when the writing
            # is finished it will copy to the final location. This is designed
            # so that you do not get half written output files should the
            # program crash or get aborted. If outfile is not defined then
            # FlexiWriter will output to STDOUT so temp does not come into it
            temp_loc = os.path.dirname(os.path.expanduser(args.outfile))
        except TypeError:
            temp_loc = None

        # The flexiwriter will control all the tempoutput or STDOUT output
        with file_helper.FlexiWriter(args.outfile,
                                     delimiter=args.out_delimiter,
                                     lineterminator=os.linesep,
                                     use_temp_dir=True,
                                     dir=temp_loc,
                                     prefix='.') as writer:
            writer.writerow(HEADER)
            for result in prog.progress(file_info):
                row = get_output_row(result, basename=args.basename)
                writer.writerow(row)


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
if __name__ == '__main__':
    main()
