import pandas as pd
import csv
import inspect
import os, time
import datetime
import fnmatch
import sys
import click
import json
import yaml
import glob
import copy
import subprocess
import cProfile, pstats
import carrot
import carrot.tools as tools

@click.group(help="Commands for mapping data to the OMOP CommonDataModel (CDM).")
def run():
    pass


@click.command(help="Generate a python class from the OMOP mapping json")
@click.option("--name",
              help="give the name of the dataset, this will be the name of the .py class file created")
@click.option("--register",
              is_flag=True,
              help="also register the python config with the tool")
@click.argument("rules")
@click.pass_context

def make_class(ctx,name,rules,register):
    data = tools.load_json(rules)
    if name == None:
        name = data['metadata']['dataset']

    fname = tools.extract.make_class(data,name)
    if register:
        ctx.invoke(register_class,pyconfig=fname)
    return fname

@click.command(help="Register a python class with the tool")
@click.argument("pyconfig")
def register_class(pyconfig):
    pyconfig = os.path.abspath(pyconfig)
    _,conf_extension = os.path.splitext(pyconfig)
    if conf_extension != '.py':
        raise NotImplementedError(f"You're trying to register the file {pyconfig} which isnt a .py file")
    tools.extract.register_class(pyconfig)



@click.command(help="List all the python classes there are available to run")
def list_classes():
    print (json.dumps(tools.get_classes(),indent=6))

@click.command(help="remove a registered class")
@click.pass_context
@click.argument("name")
def remove_class(ctx,name):
    classes = tools.get_classes()
    if name not in classes:
        print (f"{name} is not a registered class")
        ctx.invoke(list_classes)
    else:
        _class = classes[name]
        os.unlink(_class['sympath'])

@click.command(help="Execute the running of the test dataset")
@click.pass_context
def test(ctx):
    _dir = os.path.dirname(os.path.abspath(carrot.__file__))
    _dir = f"{_dir}{os.path.sep}data{os.path.sep}test"
    inputs = glob.glob(f"{_dir}{os.path.sep}inputs{os.path.sep}*.csv")
    rules = f"{_dir}{os.path.sep}rules{os.path.sep}rules_14June2021.json"
    output_folder = 'tests'
    ctx.invoke(run,inputs=inputs,rules=rules,output_folder=output_folder)
    for fname in glob.glob(f"{output_folder}{os.path.sep}*.tsv"):
        tools.diff_csv(fname,fname)


@click.command(help="apply operations (transforms) to a dataset")
@click.option("--config","-c",required=True,type=str)
@click.option("--number-of-rows-per-chunk","--nc",default=1e5,type=int)
@click.option("--output-folder","-o",required=True,type=str)
@click.argument("inputs",nargs=-1,required=True)
def transform(inputs,config,number_of_rows_per_chunk,output_folder):

    logger = tools.logger.Logger("transform")

    if not os.path.exists(output_folder):
        os.makedirs(output_folder)

    if os.path.isfile(config):
        config = json.load(open(config))
    else:
        config = json.loads(config)

    logger.info(json.dumps(config,indent=6))

    inputs = {
        os.path.basename(x):x
        for x in inputs
    }
    input_data = tools.load_csv(inputs,chunksize=number_of_rows_per_chunk)

    operation_tools = carrot.cdm.OperationTools()

    header=True
    mode='w'

    i = 0
    while True:
        if i > 0:
            mode = 'a'
            header=False
        i+=1
        for fname,rules in config.items():
            logger.info(f"Working on {fname}")
            df = input_data[fname]
            for colname,operations in rules.items():
                series = df[colname]
                for operation in operations:
                    logger.info(f".. transforming '{colname}' with operation '{operation}'")
                    fn_operation = operation_tools[operation]
                    series = fn_operation(series)
                df[colname] = series
            fout = f"{output_folder}/{fname}"
            df.to_csv(fout,header=header,mode=mode,index=False)

        try:
            input_data.next()
        except StopIteration:
            break

@click.command()
@click.option("--number-of-rows-per-chunk","--nc",default=1e5,type=int)
@click.option("--output-folder","-o",required=True,type=str)
@click.argument("inputs",nargs=-1,required=True)
def format(inputs,number_of_rows_per_chunk,output_folder):
    """
    Format a CDM model by passing all CDM objects and applying formatting.
    """
    types = list(set([
        os.path.splitext(fname)[1]
        for fname in inputs
    ]))
    if len(types) > 1:
        raise Exception(f"Running with mixed input files '{types}'. Only input tsv or csv files.")
    types = types[0]

    inputs = {
        os.path.basename(x):x
        for x in inputs
    }
    if types == '.csv':
        inputs = tools.load_csv(inputs,chunksize=number_of_rows_per_chunk)
    else:
        inputs = tools.load_tsv(inputs,chunksize=number_of_rows_per_chunk)

    cdm = carrot.cdm.CommonDataModel.from_existing(inputs=inputs,
                                                      output_folder=output_folder,
                                                      format_level=1)
    #cdm.save_files = False
    cdm.process()
    cdm.end()

@click.command()
@click.option("--output-folder","-o",required=True,type=str,help='specify an output folder of where to put the merged tables')
@click.argument("inputs",nargs=-1,required=True)
def merge(inputs,output_folder):
    """
    Build a CDM model from subfiles
    """
    types = list(set([
        os.path.splitext(fname)[1]
        for fname in inputs
    ]))
    if len(types) > 1:
        raise Exception(f"Running with mixed input files '{types}'. Only input tsv or csv files.")
    types = types[0]

    inputs = {
        os.path.basename(x):x
        for x in inputs
    }
    if types == '.csv':
        inputs = tools.load_csv(inputs)#,chunksize=number_of_rows_per_chunk)
    else:
        inputs = tools.load_tsv(inputs)#,chunksize=number_of_rows_per_chunk)

    outputs = carrot.tools.create_csv_store(output_folder=output_folder,sep='\t',write_mode='w',write_separate=False)

    cdm = carrot.cdm.CommonDataModel.from_existing(inputs=inputs,
                                                      do_mask_person_id=False,
                                                      drop_duplicates=True,
                                                      format_level=0,
                                                      outputs=outputs)
    cdm.process()


@click.command(help="loading")
@click.option("--config","-c",required=True,type=str)
@click.argument("inputs",nargs=-1,required=True)
def load(inputs,config):
    inputs = {os.path.basename(x).split('.')[0]:x for x in inputs}
    config = json.loads(config)
    if 'bclink' in config:
        outputs = carrot.io.BCLinkDataCollection(config['bclink'])
    else:
        raise NotImplementedError(f"Not setup to implement configuration for load {config}")

    for destination_table,fname in inputs.items():
        outputs.load(fname,destination_table)

    outputs.finalise()

@click.command()
@click.option("--input",
              required=True,
              help='input csv file')
@click.option("--column",
              required=True,
              help="which column of the input data to modify")
@click.option("--operation",
              required=True,
              help="which operation to apply to the column")
def format_input_data(column,operation,input):
    """
    Useful formatting command for applying an operation on some data before being passed into the ETL-Tool

    """
    optools = carrot.cdm.OperationTools()
    allowed_operations = optools.keys()
    if operation not in allowed_operations:
        raise Exception(f"Operation '{operation}' is not a known operation. Choose from {allowed_operations}")
    df_input = pd.read_csv(input)
    df_input[column] = optools[operation](df_input[column])
    n = 5 if len(df_input) > 5 else len(df_input)
    print (df_input[column].sample(n))
    df_input.to_csv(input)


def _condor(name,commands,jobscript):
    #hold_jid {name} - wait for previous job
    def wrapper(args):
        script = jobscript + '\n' + ' '.join(commands)
        fname = f"{name}.sh"
        with open(fname,"w") as f:
            f.write(script)
        output = subprocess.check_output(['qsub','-N',name,fname]).decode()
        return output
    return wrapper

from importlib import import_module

@click.command()
@click.option("analysis_name","--analysis",default=None,type=str)
@click.argument("analysis_file")
@click.pass_context
def analysis(ctx,analysis_file,analysis_name):
    """
    Use this command to run analyses on input data (in the CDM format) given a configuration yaml file
    """
    _dir = os.path.dirname(analysis_file)
    fname = os.path.splitext(os.path.basename(analysis_file))[0]
    sys.path.append(_dir)
    module = import_module(fname)
    clsmembers = inspect.getmembers(module, inspect.isclass)
    name,cls = clsmembers[0]
    obj = cls()
    if analysis_name:
        ana = obj.get_analysis(analysis_name)
        res = obj.run_analysis(ana)
        print (res)
    else:
        #print (obj.get_analyses())
        #exit(0)
        obj.run_analyses()


@click.command()
@click.option("--max-workers",default=None,type=int)
@click.option("--batch",default=None,type=click.Choice(['condor']))
@click.option("analysis_names","--analysis-name",default=None,multiple=True,type=str)
@click.argument("config")
@click.pass_context
def ___analysis(ctx,config,analysis_names,max_workers,batch):
    """
    Use this command to run analyses on input data (in the CDM format) given a configuration yaml file
    """
    from importlib import import_module
    fname = config
    stream = open(config)
    config = yaml.safe_load(stream)

    inputs = config['cdm']
    inputs = carrot.tools.load_tsv(config['cdm'],
                                      dtype=None)

    cdm = carrot.cdm.CommonDataModel.load(inputs=inputs)

    analyses = config['analyses']

    jobscript = config['condor']['jobscript']


    if analysis_names:
        temp = copy.copy(analyses)
        for name in temp:
            if name not in analysis_names:
                analyses.pop(name)

    for name,_def in analyses.items():
        analysis = _def['analysis']
        cohort = _def['filter']
        if batch:
            if batch == 'condor':
                commands = copy.copy(sys.argv)
                commands.remove('--batch')
                commands.remove('condor')
                commands.extend(['--analysis-name',name])

                f = _condor(name=name,commands=commands,jobscript=jobscript)
            else:
                raise NotImplementedError(f"{batch} mode for --batch not a thing")
        else:
            func = import_module(analysis)
            f = func.create_analysis(cohort)
        cdm.add_analysis(f)

    results = cdm.run_analyses(max_workers=max_workers)



@click.command()
@click.option("--rules",
              required=True,
              help="input json file containing all the mapping rules to be applied")
@click.option("--indexing-conf",
              default=None,
              help="configuration file to specify how to start the indexing")
@click.option("--csv-separator",
              default='\t',
              type=click.Choice([';',':','\t',',',' ',]),
              help="choose a separator to use when dumping output csv files")
@click.option("--use-profiler",
              is_flag=True,
              help="turn on saving statistics for profiling CPU and memory usage")
@click.option("format_level","--format-level",
              default='1',
              type=click.Choice(['0','1','2']),
              help="Choose the level of formatting to apply on the output data. 0 - no formatting. 1 - automatic formatting. 2 (default) - check formatting (will crash if input data is not already formatted).")
@click.option("--output-folder",
              default=None,
              help="define the output folder where to dump csv files to")
@click.option("--write-mode",
              default='w',
              type=click.Choice(['w','a']),
              help="force the write-mode on existing files")
@click.option("--split-outputs",
              is_flag=True,
              help="force the output files to be split into separate files")
@click.option("--allow-missing-data",
              is_flag=True,
              help="don't crash if there is data tables in rules file that hasnt been loaded")
@click.option("output_database","--database",
              default=None,
              help="define the output database where to insert data into")
@click.option("-nc","--number-of-rows-per-chunk",
              default=-1,
              help="Choose the number of rows (INTEGER) of input data to load (chunksize). The option 'auto' will work out the ideal chunksize. Inputing a value <=0 will turn off data chunking (default behaviour).")
@click.option("-np","--number-of-rows-to-process",
              default=None,
              type=int,
              help="the total number of rows to process")
@click.option("--person-id-map",
              default=None,
              help="pass the location of a file containing existing masked person_ids")
@click.option("--db",
              default=None,
              help="instead, pass a connection string to a db")
@click.option("--merge-output",
              is_flag=True,
              help="merge the output into one file")
@click.option("no_mask_person_id","--parse-original-person-id",
              is_flag=True,
              help="turn off automatic conversion (creation) of person_id to (as) Integer")
@click.option("dont_automatically_fill_missing_columns","--no-fill-missing-columns",
              is_flag=True,
              help="Turn off automatically filling missing CDM columns")
@click.option("log_file","--log-file",
              default = 'none',
              help="specify a path for a log file")
@click.option("--max-rules",
              default = None,
              type=int,
              help="maximum number of rules to process")
@click.option("objects","--object",
              default = None,
              multiple=True,
              type=str,
              help="give a list of objects by name to process")
@click.option("tables","--table",
              default = None,
              multiple=True,
              type=str,
              help="give a list of tables by name to process")
@click.argument("inputs",
                required=False,
                nargs=-1)
@click.pass_context
def map(ctx,rules,inputs,format_level,
        output_folder,output_database,
        csv_separator,use_profiler,log_file,
        no_mask_person_id,indexing_conf,
        person_id_map,max_rules,merge_output,
        objects,tables,db,write_mode,split_outputs,
        dont_automatically_fill_missing_columns,
        number_of_rows_per_chunk,allow_missing_data,
        number_of_rows_to_process):
    """
    Perform OMOP Mapping given an json file and a series of input files

    INPUTS should be a space separated list of individual input files or directories (which contain .csv files)
    """

    if output_folder is None:
        output_folder = f'{os.getcwd()}{os.path.sep}output_data{os.path.sep}'

    #if log_file == 'auto' and carrot.params['log_file'] is None:
    if log_file == 'auto':
        log_file = f"{output_folder}{os.path.sep}logs{os.path.sep}carrot.log"
        carrot.params['log_file'] = log_file
    elif log_file == 'none':
        pass
    else:
        carrot.params['log_file'] = log_file

    #load the json loads
    if type(rules) == dict:
        config = rules
    else:
        config = tools.load_json(rules)

    if tables:
        tables = list(set(tables))
        config = carrot.tools.filter_rules_by_destination_tables(config,tables)

    if objects:
        objects = list(set(objects))
        config = carrot.tools.filter_rules_by_object_names(config,objects)

    if max_rules:
        i = 0
        n = max_rules
        new = {}
        for destination_table,rule_set in config['cdm'].items():
            if destination_table == 'person':
                new[destination_table] = rule_set
            else:
                for name,_rules in rule_set.items():
                    if i>=n:
                        break
                    if destination_table not in new:
                        new[destination_table] = {}
                    new[destination_table][name] = _rules
                    i+=1

        config['cdm'] = new

    name = config['metadata']['dataset']

    if indexing_conf is not None:
        if isinstance(indexing_conf,dict):
            pass
        elif indexing_conf.endswith(".json") and os.path.exists(indexing_conf):
            indexing_conf = tools.load_json(indexing_conf)
        elif indexing_conf.endswith(".csv") and os.path.exists(indexing_conf):
            try:
                indexing_conf = pd.read_csv(indexing_conf,header=None,index_col=0)[1].to_dict()
            except pd.errors.EmptyDataError:
                indexing_conf = None
                pass


    #automatically calculate the ideal chunksize
    if number_of_rows_per_chunk == 'auto':
        #get the fields that are going to be used/loaded
        used_fields = tools.get_mapped_fields_from_rules(config)
        #calculate the number of fields that are to be used per dataset
        n_used_fields = [ len(sublist) for sublist in used_fields.values() ]
        #find what's the largest number of fields loaded by any dataset
        max_n_used_fields = max(n_used_fields)
        #get the number of files used
        n_files = len(n_used_fields)

        # If there is one dataset and one column being used, the max loaded to memory
        #   is 2million rows (this is fairly arbitrary)
        #   it is an approximation assuming the data in the values is relatively small
        #   this should keep the memory usage down
        # When there is more fields and more files loaded, reduce the of rows per chunk
        max_n_rows = 2e6
        number_of_rows_per_chunk = int(max_n_rows/(max_n_used_fields*n_files))
    else:
        try:
            number_of_rows_per_chunk = int(number_of_rows_per_chunk)
        except ValueError:
            raise ValueError(f"number_of_rows_per_chunk must be an Integer or 'auto', you inputted '{number_of_rows_per_chunk}'")

        #turn off chunking if 0 or negative chunksizes are given
        if number_of_rows_per_chunk <= 0 :
            number_of_rows_per_chunk = None

    #check if exists
    if any('*' in x for x in inputs):
        data_dir = os.path.dirname(carrot.__file__)
        data_dir = f'{data_dir}{os.path.sep}data{os.path.sep}'

        new_inputs = []
        for i,x in enumerate(inputs):
            if not os.path.exists(x):
                new_inputs.extend(glob.glob(f"{data_dir}{os.path.sep}{x}"))
            else:
                new_inputs.append(x)
        inputs = new_inputs

    inputs = list(inputs)

    for x in inputs:
        if os.path.isdir(x):
            inputs.remove(x)
            inputs.extend(glob.glob(f'{x}{os.path.sep}*.csv'))

    #convert the list into a map between the filename and the full path
    inputs = {
        os.path.basename(x):x
        for x in inputs
    }

    if db:
        inputs = tools.load_sql(connection_string=db,chunksize=number_of_rows_per_chunk,nrows=number_of_rows_to_process)
    else:
        if allow_missing_data:
            config = carrot.tools.remove_missing_sources_from_rules(config,inputs)

        inputs = tools.load_csv(inputs,
                                rules=config,
                                chunksize=number_of_rows_per_chunk,
                                nrows=number_of_rows_to_process)

    #do something with
    #person_id_map

    if isinstance(output_database,dict):
        if 'bclink' in output_database:
            outputs = carrot.tools.create_bclink_store(bclink_settings=output_database['bclink'],
                                                          output_folder=output_database['cache'],
                                                          sep=csv_separator,
                                                          write_separate=split_outputs,
                                                          write_mode=write_mode)
        else:
            raise NotImplementedError(f"dont know how to configure outputs... {output_database}")
    elif output_database == None:
        outputs = carrot.tools.create_csv_store(output_folder=output_folder,
                                                   sep=csv_separator,
                                                   write_separate=split_outputs,
                                                   write_mode=write_mode)
    else:
        outputs = carrot.tools.create_sql_store()

    #build an object to store the cdm
    cdm = carrot.cdm.CommonDataModel(name=name,
                                        inputs=inputs,
                                        format_level=format_level,
                                        do_mask_person_id=not no_mask_person_id,
                                        outputs = outputs,
                                        #output_folder=output_folder,
                                        #output_database=output_database,
                                        automatically_fill_missing_columns=not dont_automatically_fill_missing_columns,
                                        use_profiler=use_profiler)
    #allow the csv separator to be changed
    #the default is tab (\t) separation
    #if not csv_separator is None:
    #    cdm.set_csv_separator(csv_separator)
    cdm.create_and_add_objects(config)

    cdm.process(conserve_memory=True)
    cdm.close()

    if merge_output:
        ctx.invoke(merge,
                   inputs=glob.glob(f"{output_folder}{os.path.sep}*"),
                   output_folder=output_folder)
@click.command()
@click.option("--rules-file",
              required=True,
              help="json file containing mapping rules")
@click.option("--output-dir",
              default=None,
              help="define the output directory for OMOP-format tsv files")
@click.option("--write-mode",
              default='w',
              type=click.Choice(['w','a']),
              help="force write-mode on output files")
@click.option("--person-file",
              required=True,
              help="File containing person_ids in the first column")
@click.option("--omop-ddl-file",
              required=True,
              help="File containing OHDSI ddl statements for OMOP tables")
@click.option("--omop-config-file",
              required=True,
              help="File containing additional / override json config for omop outputs")
@click.option("--saved-person-id-file",
              default=None,
              required=False,
              help="Full path to person id file used to save person_id state and share person_ids between data sets")
@click.option("--use-input-person-ids",
              required=False,
              default='N',
              help="Use person ids as input without generating new integers")
@click.option("--last-used-ids-file",
              default=None,
              required=False,
              help="Full path to last used ids file for OMOP tables - format: tablename\tlast_used_id, \nwhere last_used_id must be an integer")
@click.option("--log-file-threshold",
              required=False,
              default=0,
              help="Lower outcount limit for logfile output")
@click.argument("input-dir",
                required=False,
                nargs=-1)
def mapstream(rules_file, output_dir, write_mode, person_file, omop_ddl_file, omop_config_file, saved_person_id_file, use_input_person_ids, last_used_ids_file, log_file_threshold, input_dir):
    """
    Map to output using input streams
    """
    print(input_dir)
    if os.path.isdir(input_dir[0]) == False:
        print("Not a directory, input dir {0}".format(input_dir[0]))
        sys.exit(1)

    if os.path.isdir(output_dir) == False:
        print("Not a directory, output dir {0}".format(output_dir))
        sys.exit(1)

    if saved_person_id_file == None:
        saved_person_id_file = output_dir + "/" + "person_ids.tsv"
        if os.path.exists(saved_person_id_file):
            os.remove(saved_person_id_file)
   
    starttime = time.time()
    omopcdm = tools.omopcdm.OmopCDM(omop_ddl_file, omop_config_file)
    #print(omopcdm.dump_ddl())
    mappingrules = tools.mappingrules.MappingRules(rules_file, omopcdm)
    metrics = tools.metrics.Metrics(mappingrules.get_dataset_name(), log_file_threshold)
    nowtime = time.time()

    print("--------------------------------------------------------------------------------")
    print("Loaded mapping rules from: {0} after {1:.5f} secs".format(rules_file, (nowtime - starttime)))
    output_files = mappingrules.get_all_outfile_names()
    record_numbers = {}
    for output_file in output_files:
        record_numbers[output_file] = 1

    fhd = {}
    tgtcolmaps = {}

    try:
        # Add in a saved-person-file existence test and reload from it is necessary returning the last used integer
        if os.path.isfile(saved_person_id_file):
            person_lookup, last_used_integer = load_saved_person_ids(saved_person_id_file)
        else:
            person_lookup = {}
            last_used_integer = 1
        if last_used_ids_file != None:
            if os.path.isfile(last_used_ids_file):
                record_numbers = load_last_used_ids(last_used_ids_file, record_numbers)
 
        #fhp = open(person_file, mode="r", encoding="utf-8-sig")
        #csvrp = csv.reader(fhp)
        person_lookup, rejected_person_count = load_person_ids(person_file, person_lookup, mappingrules, use_input_person_ids, last_used_integer)
        fhpout = open(saved_person_id_file, mode="w")
        fhpout.write("SOURCE_SUBJECT\tTARGET_SUBJECT\n")
        for person_id, person_assigned_id in person_lookup.items():
            fhpout.write("{0}\t{1}\n".format(str(person_id), str(person_assigned_id)))
        fhpout.close()
        for tgtfile in output_files:
            fhd[tgtfile] = open(output_dir + "/" + tgtfile + ".tsv", mode=write_mode)
            if write_mode == 'w':
                outhdr = omopcdm.get_omop_column_list(tgtfile)
                fhd[tgtfile].write("\t".join(outhdr) + "\n")
            tgtcolmaps[tgtfile] = omopcdm.get_omop_column_map(tgtfile)

    except IOError as e:
        print("I/O - error({0}): {1} -> {2}".format(e.errno, e.strerror, str(e)))
        exit()

    print("person_id stats: total loaded {0}, reject count {1}".format(len(person_lookup), rejected_person_count))

    # TODO get this list of input files from the  parsed rules
    existing_input_files = fnmatch.filter(os.listdir(input_dir[0]), '*.csv')
    rules_input_files = mappingrules.get_all_infile_names()
    for infile in existing_input_files:
        if infile not in rules_input_files:
            msg = "ERROR: no mapping rules found for existing input file - {0}".format(infile)
            print(msg)
            metrics.add_log_data(msg)
    for infile in rules_input_files:
        if infile not in existing_input_files:
            msg = "ERROR: no data for mapped input file - {0}".format(infile)
            print(msg)
            metrics.add_log_data(msg)
    rejidcounts = {}
    rejdatecounts = {}
    #src_tgt_counts = {}
    print(rules_input_files)

    for srcfilename in rules_input_files:
        rejidcounts[srcfilename] = 0
        rejdatecounts[srcfilename] = 0

    for srcfilename in rules_input_files:
        outcounts = {}
        rejcounts = {}
        rcount = 0

        try:
            fh = open(input_dir[0] + "/" + srcfilename, mode="r", encoding="utf-8-sig")
            csvr = csv.reader(fh)
        except IOError as e:
            print("Unable to open: {0}".format(input_dir[0] + "/" + srcfilename))
            print("I/O error({0}): {1}".format(e.errno, e.strerror))
            continue

        tgtfiles, src_to_tgt = mappingrules.parse_rules_src_to_tgt(srcfilename)
        infile_datetime_source, infile_person_id_source = mappingrules.get_infile_date_person_id(srcfilename)
        for tgtfile in tgtfiles:
            outcounts[tgtfile] = 0
            rejcounts[tgtfile] = 0
        datacolsall = []
        hdrdata = next(csvr)
        dflist = mappingrules.get_infile_data_fields(srcfilename)
        for colname in hdrdata:
            datacolsall.append(colname)
        inputcolmap = omopcdm.get_column_map(hdrdata)
        pers_id_col = inputcolmap[infile_person_id_source]
        datetime_col = inputcolmap[infile_datetime_source]
        print("--------------------------------------------------------------------------------")
        print("Processing input: {0}".format(srcfilename))
#        print("Processing input: {0}, All input cols = {1}, Data cols = {2}".format(srcfilename, str(datacolsall), str(dflist)))

        for indata in csvr:
            #indata = inputline.strip().split(",")
            key = srcfilename + "~all~all~all~"
            metrics.increment_key_count(key, "input_count")
            rcount += 1
            strdate = indata[datetime_col].split(" ")[0]
            fulldate = parse_date(strdate)
            if fulldate != None:
                #fulldate = "{0}-{1:02}-{2:02}".format(dt.year, dt.month, dt.day)
                indata[datetime_col] = fulldate
            else:
                metrics.increment_key_count(key, "invalid_date_fields")
                continue

            for tgtfile in tgtfiles:
                tgtcolmap = tgtcolmaps[tgtfile]
                auto_num_col = omopcdm.get_omop_auto_number_field(tgtfile)
                pers_id_col = omopcdm.get_omop_person_id_field(tgtfile)

                datacols = datacolsall
                if tgtfile in dflist:
                    datacols = dflist[tgtfile]

                for datacol in datacols:
                    built_records, outrecords, metrics = get_target_records(tgtfile, tgtcolmap, src_to_tgt, datacol, indata, inputcolmap, srcfilename, omopcdm, metrics)
                    if built_records == True:
                        for outrecord in outrecords:
                            if auto_num_col != None:
                                outrecord[tgtcolmap[auto_num_col]] = str(record_numbers[tgtfile])
                                record_numbers[tgtfile] += 1
                            if (outrecord[tgtcolmap[pers_id_col]]) in person_lookup:
                                outrecord[tgtcolmap[pers_id_col]] = person_lookup[outrecord[tgtcolmap[pers_id_col]]]
                                outcounts[tgtfile] += 1
                                key = srcfilename + "~all~all~all~"
                                metrics.increment_key_count(key, "output_count")
                                key = "all~all~" + tgtfile + "~all~"
                                metrics.increment_key_count(key, "output_count")
                                key = srcfilename + "~all~" + tgtfile + "~all~"
                                metrics.increment_key_count(key, "output_count")
                                if tgtfile == "person":
                                    key = srcfilename + "~all~" + tgtfile + "~" + outrecord[1] +"~"
                                    metrics.increment_key_count(key, "output_count")
                                    key = srcfilename + "~" + datacol +"~" + tgtfile + "~" + outrecord[1] + "~" + outrecord[2]
                                    metrics.increment_key_count(key, "output_count")
                                else:
                                    key = srcfilename + "~" + datacol +"~" + tgtfile + "~" + outrecord[2] + "~"
                                    metrics.increment_key_count(key, "output_count")
                                    key = srcfilename + "~all~" + tgtfile + "~" + outrecord[2] + "~"
                                    metrics.increment_key_count(key, "output_count")
                                    key = "all~all~" + tgtfile + "~" + outrecord[2] + "~"
                                    metrics.increment_key_count(key, "output_count")
                                    key = "all~all~all~" + outrecord[2] + "~"
                                    metrics.increment_key_count(key, "output_count")
                                fhd[tgtfile].write("\t".join(outrecord) + "\n")
                            else:
                                key = srcfilename + "~all~" + tgtfile + "~all~"
                                metrics.increment_key_count(key, "invalid_person_ids")
                                rejidcounts[srcfilename] += 1

        fh.close()

        nowtime= time.time()
        print("INPUT file data : {0}: input count {1}, time since start {2:.5} secs".format(srcfilename, str(rcount), (nowtime - starttime)))
        for outtablename, count in outcounts.items():
            print("TARGET: {0}: output count {1}".format(outtablename, str(count)))

    print("--------------------------------------------------------------------------------")
    data_summary = metrics.get_mapstream_summary()
    log_report = metrics.get_log_data()
    try:
        dsfh = open(output_dir + "/summary_mapstream.tsv", mode="w")
        dsfh.write(data_summary)
        dsfh.close()
        logfh = open(output_dir + "/error_report.txt", mode="w")
        logfh.write(log_report)
        logfh.close()
    except IOError as e:
        print("I/O error({0}): {1}".format(e.errno, e.strerror))
        print("Unable to write file")

    nowtime = time.time()
    print("Elapsed time = {0:.5f} secs".format(nowtime - starttime))
    #profiler.disable()
    #stats = pstats.Stats(profiler).sort_stats('ncalls')
    #stats.print_stats()

def get_target_records(tgtfilename, tgtcolmap, rulesmap, srcfield, srcdata, srccolmap, srcfilename, omopcdm, metrics):
    build_records = False
    tgtrecords = []
    date_col_data = omopcdm.get_omop_datetime_linked_fields(tgtfilename)
    date_component_data = omopcdm.get_omop_date_field_components(tgtfilename)
    notnull_numeric_fields = omopcdm.get_omop_notnull_numeric_fields(tgtfilename)

    srckey = srcfilename + "~" + srcfield + "~" + tgtfilename
    summarykey = srcfilename + "~" + srcfield + "~" + tgtfilename + "~all~"
    if valid_value(str(srcdata[srccolmap[srcfield]])):
        srcfullkey = srcfilename + "~" + srcfield + "~" + str(srcdata[srccolmap[srcfield]]) + "~" + tgtfilename
        dictkeys = []
        if srcfullkey in rulesmap:
            build_records = True
            dictkeys.append(srcfullkey)
        if srckey in rulesmap:
            build_records = True
            dictkeys.append(srckey)
        if build_records == True:
            for dictkey in dictkeys:
                for out_data_elem in rulesmap[dictkey]:
                    valid_data_elem = True
                    tgtarray = ['']*len(tgtcolmap)
                    for req_integer in notnull_numeric_fields:
                        tgtarray[tgtcolmap[req_integer]] = "0"
                    for infield, outfield_list in out_data_elem.items():
                        for output_col_data in outfield_list:
                            if "~" in output_col_data:
                                outcol, term = output_col_data.split("~")
                                tgtarray[tgtcolmap[outcol]] = term
                            else:
                                tgtarray[tgtcolmap[output_col_data]] = srcdata[srccolmap[infield]]
                            if output_col_data in date_component_data:
                                strdate = srcdata[srccolmap[infield]].split(" ")[0]
                                dt = get_datetime_value(strdate)
                                if dt != None:
                                    year_field = date_component_data[output_col_data]["year"]
                                    month_field = date_component_data[output_col_data]["month"]
                                    day_field = date_component_data[output_col_data]["day"]
                                    tgtarray[tgtcolmap[year_field]] = str(dt.year)
                                    tgtarray[tgtcolmap[month_field]] = str(dt.month)
                                    tgtarray[tgtcolmap[day_field]] = str(dt.day)
                                    fulldate = "{0}-{1:02}-{2:02}".format(dt.year, dt.month, dt.day)
                                    tgtarray[tgtcolmap[output_col_data]] = fulldate
                                else:
                                    metrics.increment_key_count(summarykey, "invalid_date_fields")
                                    valid_data_elem = False
                            elif output_col_data in date_col_data:
                                fulldate = srcdata[srccolmap[infield]]
                                tgtarray[tgtcolmap[output_col_data]] = fulldate
                                tgtarray[tgtcolmap[date_col_data[output_col_data]]] = fulldate
                    if valid_data_elem == True:
                        tgtrecords.append(tgtarray)
    else:
        metrics.increment_key_count(summarykey, "invalid_source_fields")


    return build_records, tgtrecords, metrics

def valid_value(item):
    """
    Check if an item is non blank (null)
    """
    if item.strip() == "":
        return(False)
    return(True)

def valid_date_value(item):
    """
    Check if a date item is non null and parses as ISO (YYYY-MM-DD), reverse-ISO
    or dd/mm/yyyy or mm/dd/yyyy
    """
    if item.strip() == "":
        return(False)
    if not valid_iso_date(item) and not valid_reverse_iso_date(item) and not valid_uk_date(item):
        #print("Bad date : {0}".format(item))
        return(False)
    return(True)

def get_datetime_value(item):
    """
    Check if a date item is non null and parses as ISO (YYYY-MM-DD), reverse-ISO
    or dd/mm/yyyy or mm/dd/yyyy
    """
    dt = None
    # Does the date parse as an ISO date?
    try:
        dt = datetime.datetime.strptime(item, "%Y-%m-%d")
    except ValueError:
        pass
    if dt != None:
      return(dt)

    # Does the date parse as a reverse ISO date?
    try:
        dt = datetime.datetime.strptime(item, "%d-%m-%Y")
    except ValueError:
        pass

    if dt != None:
      return(dt)

    # Does the date parse as a UK old-style date?
    try:
        dt = datetime.datetime.strptime(item, "%d/%m/%Y")
    except ValueError:
        pass

    if dt != None:
      return(dt)

    return None

def parse_date(item):
    """
    Crude hand-coded check on date format
    """
    datedata = item.split("-")
    if len(datedata) != 3:
        datedata = item.split("/")
    if len(datedata) != 3:
        return None
    if len(datedata[2]) == 4:
        return("{0}-{1}-{2}".format(datedata[2], datedata[1], datedata[0]))
    return("{0}-{1}-{2}".format(datedata[0], datedata[1], datedata[2]))


def valid_iso_date(item):
    """
    Check if a date item is non null and parses as ISO (YYYY-MM-DD)
    """
    try:
        datetime.datetime.strptime(item, "%Y-%m-%d")
    except ValueError:
        return(False)

    return(True)

def valid_reverse_iso_date(item):
    """
    Check if a date item is non null and parses as reverse ISO (DD-MM-YYYY)
    """
    try:
        datetime.datetime.strptime(item, "%d-%m-%Y")
    except ValueError:
        return(False)

    return(True)

def valid_uk_date(item):
    """
    Check if a date item is non null and parses as UK format (DD/MM/YYYY)
    """
    try:
        datetime.datetime.strptime(item, "%d/%m/%Y")
    except ValueError:
        return(False)

    return(True)

def load_last_used_ids(last_used_ids_file, last_used_ids):
    fh = open(last_used_ids_file, mode="r", encoding="utf-8-sig")
    csvr = csv.reader(fh, delimiter="\t")

    for last_ids_data in csvr:
        last_used_ids[last_ids_data[0]] = int(last_ids_data[1]) + 1

    fh.close()
    return last_used_ids

def load_saved_person_ids(person_file):
    fh = open(person_file, mode="r", encoding="utf-8-sig")
    csvr = csv.reader(fh, delimiter="\t")
    last_int = 1
    person_ids = {}

    next(csvr)
    for persondata in csvr:
        person_ids[persondata[0]] = persondata[1]
        last_int += 1

    fh.close()
    return person_ids, last_int

def load_person_ids(person_file, person_ids, mappingrules, use_input_person_ids, person_number=1, delim=","):
    fh = open(person_file, mode="r", encoding="utf-8-sig")
    csvr = csv.reader(fh, delimiter=delim)
    person_columns = {}
    person_col_in_hdr_number = 0
    reject_count = 0

    personhdr = next(csvr)
    print(personhdr)

    # Make a dictionary of column names vs their positions
    for col in personhdr:
        person_columns[col] = person_col_in_hdr_number
        person_col_in_hdr_number += 1

    birth_datetime_source, person_id_source = mappingrules.get_person_source_field_info("person")
    print("Load Person Data {0}, {1}".format(birth_datetime_source, person_id_source))
    person_col = person_columns[person_id_source]

    for persondata in csvr:
        if not valid_value(persondata[person_columns[person_id_source]]):
            reject_count += 1
            continue
        if not valid_date_value(persondata[person_columns[birth_datetime_source]]):
            reject_count += 1
            continue
        if persondata[person_col] not in person_ids:
            if use_input_person_ids == "N":
                person_ids[persondata[person_col]] = str(person_number)
                person_number += 1
            else:
                person_ids[persondata[person_col]] = str(persondata[person_col])
    fh.close()

    return person_ids, reject_count

@click.command(help="Perform OMOP Mapping given a python configuration file.")
@click.option("--rules",
              help="input json file containing all the mapping rules to be applied")
@click.option("--pyconf","--conf",
              required=True,
              help="Run with a python configuration file")
@click.option("--object","objects",
              multiple=True,
              help="Only run specific objects.")
@click.option("--type",
              default='csv',
              type=click.Choice(['csv']),
              help="specify the type of inputs, the default is .csv inputs")
@click.option("--use-profiler",
              is_flag=True,
              help="turn on saving statistics for profiling CPU and memory usage")
@click.option("--output-folder",
              default=None,
              help="define the output folder where to dump csv files to")
@click.option("-nc","--number-of-rows-per-chunk",
              default=None,
              type=int,
              help="choose to chunk running the data into nrows")
@click.option("-np","--number-of-rows-to-process",
              default=None,
              type=int,
              help="the total number of rows to process")
@click.argument("inputs",
                #help="give a list of input files to process, and/or an input directory",
                nargs=-1)
@click.pass_context
def run_pyconfig(ctx,rules,pyconf,inputs,objects,
        output_folder,type,use_profiler,
        number_of_rows_per_chunk,
        number_of_rows_to_process):

    object_list = list(objects)
    if len(object_list) == 0:
        object_list = None

    if type != 'csv':
        raise NotImplementedError("Can only handle inputs that are .csv so far")

    #check if exists
    if any('*' in x for x in inputs):
        data_dir = os.path.dirname(carrot.__file__)
        data_dir = f'{data_dir}{os.path.sep}data{os.path.sep}'

        new_inputs = []
        for i,x in enumerate(inputs):
            if not os.path.exists(x):
                new_inputs.extend(glob.glob(f"{data_dir}{os.path.sep}{x}"))
            else:
                new_inputs.append(x)
        inputs = new_inputs

    inputs = list(inputs)
    for x in inputs:
        if os.path.isdir(x):
            inputs.remove(x)
            inputs.extend(glob.glob(f'{x}{os.path.sep}*.csv'))

    #convert the list into a map between the filename and the full path
    inputs = {
        os.path.basename(x):x
        for x in inputs
    }

    if output_folder is None:
        output_folder = f'{os.getcwd()}{os.path.sep}output_data{os.path.sep}'

    outputs = carrot.tools.create_csv_store(output_folder=output_folder)


    if not inputs:
        raise Exception('no inputs defined!')

    inputs = tools.load_csv(inputs,
                            rules=rules,
                            chunksize=number_of_rows_per_chunk,
                            nrows=number_of_rows_to_process)

    available_classes = tools.get_classes()
    if pyconf not in available_classes:
        ctx.invoke(list_classes)
        raise KeyError(f"cannot find config {pyconf}. Run 'carrot map py list' to see available classes.")

    module = __import__(available_classes[pyconf]['module'],fromlist=[pyconf])
    defined_classes = [
        m[0]
        for m in inspect.getmembers(module, inspect.isclass)
        if m[1].__module__ == module.__name__
    ]
    #should only be running one class anyway
    defined_class = defined_classes[0]
    cls = getattr(module,defined_class)

    #build a class object
    cdm = cls(inputs=inputs,
              outputs=outputs,
              use_profiler=use_profiler)
    #run it
    cdm.process(object_list)

@click.command(help="run as a Graphical User Interface (GUI)")
@click.pass_context
def gui(ctx):
    import PySimpleGUI as sg

    carrot_theme = {'BACKGROUND': 'white',
                       'TEXT': '#000000',
                       'INPUT': '#c4c6e2',
                       'TEXT_INPUT': '#000000',
                       'SCROLL': '#c4c6e2',
                       'BUTTON': ('white', '#475da7','#3DB28C'),
                       'PROGRESS': ('#01826B', '#D0D0D0'),
                       'BORDER': 1,
                       'SLIDER_DEPTH': 1,
                       'PROGRESS_DEPTH': 0}

    # Add your dictionary to the PySimpleGUI themes
    sg.theme_add_new('carrot', carrot_theme)
    sg.theme('carrot')

    _dir = os.path.dirname(os.path.abspath(carrot.__file__))
    data_dir = f"{_dir}{os.path.sep}data{os.path.sep}"

    layout = [
        [sg.Image(f'{data_dir}logo.png'),sg.T("CO-CONNECT: Dataset2CDM",font = ("Roboto", 25))],
        [sg.T('Select the rules json:')],
        [sg.Input(key='_RULES_'), sg.FilesBrowse(initial_folder=os.getcwd())],
        [sg.T('Select the input CSVs:')],
        [sg.Input(key='_INPUTS_'), sg.FilesBrowse(initial_folder=os.getcwd())],
        [sg.T('Select an output folder:')],
        [sg.Input(key='_OUTPUT_',default_text='.'), sg.FolderBrowse(initial_folder=os.getcwd())],
        #[sg.Checkbox("Mask the person_id",key="_MASK_PERSON_ID_",default=False)],
        #[[sg.T('Change the default data chunksize:'),
        #  sg.Slider(range=(0,1000000),
        #            default_value=100000,
        #            resolution=10000,
        #            orientation='horizontal')]],
        [sg.OK('Run'), sg.Cancel(button_color=('white','#3DB28C'))]
    ]

    font = ("Roboto", 15)

    window = sg.Window('carrot', layout, font=font)
    while True:
        event, values = window.Read()

        if event == 'Cancel' or event == None:
            break

        output_folder = values['_OUTPUT_']
        if output_folder == '':
            output_folder = None

        rules = values['_RULES_']
        if rules == '':
            sg.Popup(f'Error: please select a rules file')
            continue
        elif len(rules.split(';'))>1:
            sg.Popup(f'Error: only select one file for the rules!')
            continue

        inputs = values['_INPUTS_']
        if inputs == '':
            sg.Popup(f'Error: please select at least one file or directory for the inputs')
            continue
        inputs = inputs.split(';')

        #mask_person_id = values['_MASK_PERSON_ID_']

        try:
            #ctx.invoke(run,rules=rules,inputs=inputs,output_folder=output_folder,mask_person_id=mask_person_id)
            ctx.invoke(run,rules=rules,inputs=inputs,output_folder=output_folder)
            sg.Popup("Done!")
        except Exception as err:
            sg.popup_error("An exception occurred!",err)

        break

    window.close()

@click.group(help="Commands for using python configurations to run the ETL transformation.")
def py():
    pass

py.add_command(make_class,"make")
py.add_command(register_class,"register")
py.add_command(list_classes,"list")
py.add_command(remove_class,"remove")
py.add_command(run_pyconfig,"map")
run.add_command(py,"py")
run.add_command(map,"map")
run.add_command(mapstream,"mapstream")
run.add_command(list_classes, "list_classes")
run.add_command(load,"load")
run.add_command(analysis,"analysis")
run.add_command(format,"format")
run.add_command(merge,"merge")
run.add_command(transform,"transform")
run.add_command(gui,"gui")
run.add_command(test,"test")
