"""
.. Support for querying GCAM's XML database and processing results.

.. Copyright (c) 2016 Richard Plevin
   See the https://opensource.org/licenses/MIT for license details.
"""
import os
import re
import subprocess

from lxml import etree as ET

from .Xvfb import Xvfb
from .config import getParam, getParamAsBoolean, getParamAsFloat
from .constants import NUM_AEZS, GCAM_32_REGIONS
from .error import PygcamException, ConfigFileError, FileFormatError, CommandlineError, FileMissingError
from .log import getLogger
from .queryFile import QueryFile, RewriteSetParser
from .utils import (mkdirs, deleteFile, ensureExtension, ensureCSV, parse_version_info,
                    pathjoin, saveToFile, getExeDir, writeXmldbDriverProperties)
from .temp_file import TempFile, getTempFile

_logger = getLogger(__name__)

def limitYears(df, years):
    """
    Modify df to drop all years outside the range given by `years`.

    :param years: a sequence of two years (str or int); only values in this
        range (inclusive) are kept. Data for other years is dropped.
    :return: (DataFrame) df, modified in place.
    """
    first, last = map(int, years)
    yearCols  = map(int, filter(str.isdigit, df.columns))
    dropYears = map(str, filter(lambda y: y < first or y > last, yearCols))
    df.drop(dropYears, axis=1, inplace=True)
    return df

# TBD: simplify using this approach:

# def interp(df):
#     import numpy as np
#
#     years = sorted(map(int, filter(str.isdigit, df.columns)))
#     all_years = range(years[0], years[-1] + 1)
#     interp_yrs = sorted(set(all_years) - set(years))
#
#     # create the cols to interpolate and set to NaN for interpolate()
#     for y in interp_yrs:
#         df[str(y)] = np.nan
#
#     df = df.sort_index(axis=1).interpolate(axis=1)
#     return df


def interpolateYears(df, startYear=0, inplace=False):
    """
    Interpolate linearly between each pair of years in the GCAM output. The
    time-step is calculated from the numerical (string) column headings given
    in the `DataFrame`_ `df`, which are assumed to represent years in the time-series.
    The years to interpolate between are read from `df`, so there's no dependency
    on any particular time-step, or even on the time-step being constant.

    :param df: (DataFrame) Data of the format returned by batch queries
        on the GCAM XML database
    :param startYear: (int) If non-zero, begin interpolation at this year.
    :param inplace: (bool) If True, modify `df` in place; otherwise modify a copy.
    :return: if `inplace` is True, `df` is returned; otherwise a copy
      of `df` with interpolated values is returned.
    """
    yearCols = filter(str.isdigit, df.columns)
    years = map(int, yearCols)

    for i in range(0, len(years)-1):
        start = years[i]
        end   = years[i+1]
        timestep = end - start

        if timestep == 1:       # don't interpolate annual results if already annual
            continue

        startCol = df[str(start)]
        endCol   = df[str(end)]

        # compute vector of annual deltas for each row
        delta = (endCol - startCol)/timestep

        # interpolate the whole column -- but don't interpolate before the start year
        for j in range(1, timestep):
            nextYear = start + j
            df[str(nextYear)] = df[str(nextYear-1)] + (0 if nextYear < startYear else delta)

    yearCols = filter(str.isdigit, df.columns)  # get annualized year columns
    years = map(int, yearCols)       # sort as integers
    years.sort()
    yearCols = map(str, years)       # convert back to strings, now sorted

    nonYearCols = list(set(df.columns) - set(yearCols))
    result = df.reindex_axis(nonYearCols + yearCols, axis=1, copy=(not inplace))
    return result

_csvCache = {}

def readCsv(filename, skiprows=1, years=None, interpolate=False, startYear=0, cache=False):
    """
    Read a CSV file of the form generated by GCAM batch queries, i.e., skip one
    row and then read column headings and data. Optionally drop all years outside
    the `years` given. Optionally, linearly interpolate annual values between
    time-steps.

    :param filename: (str) the path to a CSV file
    :param skiprows: (int) the number of rows to skip before reading the data matrix
    :param years: (iterable of two values coercible to int) the year columns to
        keep; others are dropped
    :param interpolate: (bool) If True, interpolate annual values between time-steps
    :param startYear: (int) If interpolating, the year to begin interpolation
    :param cache: (bool) If True, file will be sought in, and saved to, a CSV cache.
       The "raw" file data is cached, so if called with different processing args,
       the same initial DataFrame is used, but it will be processed correctly.
    :return: (DataFrame) the data read in, processed as per arguments
    """
    import pandas as pd

    found = False

    if cache and filename in _csvCache:
        _logger.debug("Found %s in CSV cache", filename)
        df = _csvCache[filename]
        found = True

    else:
        try:
            _logger.debug("Reading %s", filename)
            df = pd.read_table(filename, sep=',', skiprows=skiprows, index_col=None)

        except IOError as e:
            raise FileMissingError(os.path.abspath(filename), e)

        except Exception as e:
            raise PygcamException('Error reading %s: %s' % (filename, e))

    # Cache a copy, unless it was already in the cache
    if cache and not found:
        _csvCache[filename] = df.copy()

    if years:
        limitYears(df, years)

    if interpolate:
        df = interpolateYears(df, startYear=startYear)

    return df

def writeCsv(df, filename, header='', float_format="%.4f"):
    """
    Save a DataFrame a file in "standard" GCAM csv format', which
    means without a numerical index, and with column headers on
    the second line.

    :param df: (pandas.DataFrame) a DataFrame holding the data to write
    :param filename: (str) the name of the file to create
    :param header: (str) a string to write as the first line of the file,
      (a single line preceding column headers is standard GCAM query result
      format.)
    :param float_format: (str) a format string indicating how to represent
      numeric values. Default shows 4 decimal places. To limit results to,
      for example, 2 decimal places, use float_format="%.2f".
    :return: none
    """
    _logger.info("Writing %s", filename)

    txt = df.to_csv(None, float_format=float_format)
    with open(filename, 'w') as f:
        f.write("%s\n" % header)  # add a header line to match batch-query output format
        f.write(txt)

# TBD: This belongs with gcamtool. Currently used only by constraints.py
def  readQueryResult(batchDir, baseline, queryName, years=None, interpolate=False, startYear=0, cache=False):
    """
    Compose the name of the 'standard' result file, read it into a DataFrame and
    return the DataFrame. Data is read from the computed filename
    "{batchDir}/{queryName}-{baseline}.csv".

    :param batchDir: (str) a directory in which the data file resides
    :param baseline: (str) the name of a baseline scenario
    :param queryName: (str) the name of a batch query.
    :param years: (iterable of two values coercible to int) the year columns to keep;
      others are dropped
    :param interpolate: (bool) If True, interpolate annual values between time-steps
    :param startYear: (int) If interpolating, the year to begin interpolation
    :param cache: (bool) If True, files will be sought in and saved to a CSV cache
    :return: (DataFrame) the data in the computed filename.
    """
    pathname = pathjoin(batchDir, '%s-%s.csv' % (queryName, baseline))
    df= readCsv(pathname, years=years, interpolate=interpolate, startYear=startYear, cache=cache)
    return df

# TBD: not used, but may be useful
def getRegionList(workspace=None):
    """
    Get a list of the defined region names.

    :param workspace: the path to a ``Main_User_Workspace`` directory that
      has the file
      ``input/gcam-data-system/_common/mappings/GCAM_region_names.csv``,
      or ``None``, in which case the value of config variable
      ``GCAM.SourceWorkspace`` (if defined) is used. If `workspace` is
      empty or ``None``, and the config variable ``GCAM.SourceWorkspace`` is
      empty (the default value), the built-in default 32-region list is returned.
    :return: a list of strings with the names of the defined regions
    """
    relpath = 'input/gcam-data-system/_common/mappings/GCAM_region_names.csv'

    workspace = workspace or getParam('GCAM.SourceWorkspace')
    if not workspace:
        return GCAM_32_REGIONS

    path = pathjoin(workspace, relpath)

    _logger.debug("Reading:", path)
    df = readCsv(path, skiprows=3)  # this is a gcam-data-system input file (different format)
    regions = list(df.region)
    return regions

def readRegionMap(filename):
    """
    Read a region map file containing one or more tab-delimited lines of the form
    ``key`` <tab> ``value``, where `key` should be a standard GCAM region and
    `value` the name of the region to map the original to, which can be an
    existing GCAM region or a new name defined by the user.

    :param filename: the name of a file containing region mappings
    :return: a dictionary holding the mappings read from `filename`
    """
    import re
    mapping = {}
    pattern = re.compile('\t+')

    _logger.info("Reading region map '%s'", filename)
    with open(filename) as f:
        lines = f.readlines()

    for line in lines:
        line = line.strip()
        if line[0] == '#':
            continue

        tokens = pattern.split(line)
        if len(tokens) != 2:
            raise FileFormatError("Badly formatted line in region map '%s': %s" % (filename, line))

        mapping[tokens[0]] = tokens[1]

    return mapping

def dropExtraCols(df, inplace=True):
    """
    Drop some columns that GCAM queries sometimes return, but which we generally don't need.
    The columns to drop are taken from from the configuration file variable ``GCAM.ColumnsToDrop``,
    which should be a comma-delimited string. The default value is ``scenario,Notes,Date``.

    :param df: a `DataFrame`_ hold the results of a GCAM query.
    :param inplace: if True, modify `df` in-place; otherwise return a modified copy.
    :return: the original `df` (if inplace=True) or the modified copy.
    """
    columns = df.columns
    # eliminate any extra (empty) columns that appear to be query artifacts
    dropCols = filter(lambda s: s.startswith('Unnamed:'), columns)

    colString = getParam('GCAM.ColumnsToDrop')
    colList = colString and colString.split(',')

    if colString and not colList:
        raise ConfigFileError("The value of %s is '%s'; should be a comma-delimited list of column names")

    unneeded = set(colList)
    existing = set(columns)
    dropCols += existing & unneeded    # drop any columns in both sets

    resultDF = df.drop(dropCols, axis=1, inplace=inplace)
    return resultDF

def _removeLevelByName(rewriteList, levelName):
    """
    If there was a hard-coded rewrite for levelName, delete it so we
    can use the one defined in the query file.
    """
    nodes = rewriteList.xpath("./level[@name='%s']" % levelName)
    for levelElt in nodes:
        rewriteList.remove(levelElt)

def _addRegionMap(regionMap, rewriteList):
    _removeLevelByName(rewriteList, 'region')

    levelElt = ET.Element('level', name='region')
    rewriteList.append(levelElt)

    for fromReg, toReg in regionMap.items():
        rewrite = ET.Element('rewrite', attrib={'from': fromReg, 'to': toReg})
        levelElt.append(rewrite)

def _addRewrites(levelElt, rewriteSet):

    def _appendRewrite(From, to):
        'Helper function to reduce redundancy'
        node = ET.Element('rewrite', attrib={'from': From, 'to': to})
        levelElt.append(node)

    byAEZ = rewriteSet.byAEZ
    byBasin = rewriteSet.byBasin

    for rewrite in rewriteSet.rewrites:
        From = rewrite.From             # "from" is a keyword, thus "From"
        to = rewrite.to

        if byAEZ:
            # Generate a rewrite for each AEZ
            for aez in range(1, NUM_AEZS + 1):
                fromAEZ = From + 'AEZ%02d' % aez
                _appendRewrite(fromAEZ, to)

        # TBD: not sure this makes sense here, actually...
        if byBasin:
            # Generate a rewrite for each basin-region combo? # TBD: GCAM5
            # for basin in ?
            #     fromBasin = 'From +
            #     _appendRewrite(fromAEZ, to)
            pass

        else:
            _appendRewrite(From, to)

def _addRewriteSet(rewriteSetList, rewriteParser, rewriteList, title):
    # TBD: Verify with Pralit.
    # Set to True if any of the mappings requires this, since it apparently we
    # cannot set these independently in different rewrite lists.
    appendValues = False

    for rewriteSetName, level in rewriteSetList:
        rewriteSet = rewriteParser.getRewriteSet(rewriteSetName)
        levelName = level or rewriteSet.level

        _removeLevelByName(rewriteList, levelName)
        levelElt = ET.Element('level', name=levelName)

        _logger.debug("Applying rewriteSet '%s', level '%s' for '%s'", rewriteSetName, levelName, title)

        rewriteList.append(levelElt)
        _addRewrites(levelElt, rewriteSet)
        appendValues = appendValues or rewriteSet.appendValues

    rewriteList.set('append-values', 'true' if appendValues else 'false')

def _findQueryByName(tree, title):
    """
    Try the title and variations thereof to locate the query by name
    """
    # This Xpath supports both Main_Queries-type files and batch query files
    xpathPattern = '/queries//queryGroup/*[@title="{title}"]|/queries/aQuery/*[@title="{title}"]'

    patterns = (None, '_', '-', '[-_]')

    for pattern in patterns:
        altTitle = re.sub(pattern, ' ', title) if pattern else title
        xpath = xpathPattern.format(title=altTitle)
        elts = tree.xpath(xpath)  # returns empty list or list of elements found
        if len(elts) != 0:
            return elts

    return None

def _findOrCreateQueryFile(title, queryPath, regions, outputDir=None, tmpFiles=True,
                           regionMap=None, rewriteSetList=None, rewriteParser=None,
                           delete=True):
    '''
    Find a query with the given title either as a file (with .xml extension) or
    within an XML query file by searching queryPath. If the query with "title" is
    found in an XML query file, extract it to generate a batch query file and
    apply it to the given regions. If outputDir is given, files are written there
    rather than creating temp files that would be deleted when the program exits.
    '''
    sep = os.path.pathsep           # ';' on Windows, ':' on Unix
    items = queryPath.split(sep)

    parser = ET.XMLParser(remove_blank_text=True)

    for item in items:
        if os.path.isdir(item):
            pathname = pathjoin(item, title + '.xml')
            if os.path.isfile(pathname):
                return pathname
            else:
                continue

        # Find the query within an XML query file
        tree = ET.parse(item, parser=parser)
        elts = _findQueryByName(tree, title)

        if elts is None or len(elts) == 0:
            continue # to next item in QueryPath

        _logger.debug("Found query '{}' in {}".format(title, item))

        root = ET.Element("queries")
        aQuery = ET.Element("aQuery")
        root.append(aQuery)
        for region in regions:
            aQuery.append(ET.Element('region', name=region))

        queryElt = elts[0]
        aQuery.append(queryElt)

        if regionMap or rewriteSetList:
            # if a rewrite list element exists already, use it, otherwise create it.
            subtree = ET.ElementTree(element=queryElt)
            found = subtree.xpath('//labelRewriteList')
            if len(found) == 1:
                rewriteList = found[0]
            else:
                # create and insert a labelRewriteList element
                rewriteList = ET.Element('labelRewriteList', attrib={'append-values': 'true'})
                queryElt.append(rewriteList)

            if regionMap:
                _addRegionMap(regionMap, rewriteList)

            if rewriteSetList:
                _addRewriteSet(rewriteSetList, rewriteParser, rewriteList, title)

        # Extract the query into a file to submit to ModelInterface
        if tmpFiles:
            path = getTempFile(suffix='.query.xml', delete=delete)
        else:
            outputDir = outputDir or getParam('GCAM.OutputDir')
            queryDir = pathjoin(outputDir, 'queries')
            mkdirs(queryDir)
            path = pathjoin(queryDir, title + '.xml')

        _logger.debug("Writing extracted query for '%s' to '%s'", title, path)
        tree = ET.ElementTree(root)
        tree.write(path, xml_declaration=True, encoding="UTF-8", pretty_print=True)
        return path


BatchQueryTemplate = """<?xml version="1.0"?>
<!-- WARNING: this file is automatically generated. Manual edits will be overwritten. -->
<ModelInterfaceBatch>
    <class name="ModelInterface.ModelGUI2.DbViewer">
        <command name="XMLDB Batch File">
            <scenario name="{scenario}"/>
            <queryFile>{queryFile}</queryFile>
            <outFile>{csvFile}</outFile>
            <xmldbLocation>{xmldb}</xmldbLocation>
            <batchQueryResultsInDifferentSheets>false</batchQueryResultsInDifferentSheets>
            <batchQueryIncludeCharts>false</batchQueryIncludeCharts>
            <batchQuerySplitRunsInDifferentSheets>false</batchQuerySplitRunsInDifferentSheets>
            <batchQueryReplaceResults>true</batchQueryReplaceResults>
        </command>
    </class>
</ModelInterfaceBatch>
"""

MultiCommandTemplate = """<?xml version="1.0"?>
<!-- WARNING: this file is automatically generated. Manual edits will be overwritten. -->
<ModelInterfaceBatch>
    <class name="ModelInterface.ModelGUI2.DbViewer">
        {batchCommands}
    </class>
</ModelInterfaceBatch>
"""

BatchCommandElement = """
        <command name="XMLDB Batch File">
            <scenario name="{scenario}"/>
            <queryFile>{queryFile}</queryFile>
            <outFile>{csvFile}</outFile>
            <xmldbLocation>{xmldb}</xmldbLocation>
            <batchQueryResultsInDifferentSheets>false</batchQueryResultsInDifferentSheets>
            <batchQueryIncludeCharts>false</batchQueryIncludeCharts>
            <batchQuerySplitRunsInDifferentSheets>false</batchQuerySplitRunsInDifferentSheets>
            <batchQueryReplaceResults>true</batchQueryReplaceResults>
        </command>
"""

#
# TBD: use new (in 4.3) -l flag to MI to route output to log file.
#
def _createJavaCommand(batchFile, miLogFile):
    # javaLibPathArg = '-Djava.library.path="%s"' % javaLibPath if javaLibPath else ""
    # command = 'java %s %s -jar "%s" -b "%s" %s' % (javaArgs, javaLibPathArg, jarFile, batchFile, redirect)

    # e.g., java %(GCAM.MI.JavaArgs)s -cp %(GCAM.MI.ClassPath)s ModelInterface/InterfaceMain -b "{batchFile}"
    template = getParam('GCAM.MI.BatchCommand')
    command = template.format(batchFile=batchFile)
    if miLogFile:
        command += ">> %s 2>&1" % miLogFile

    return command

def _copyToLogFile(logFile, filename, msg=''):
    with open(logFile, 'a') as m:
        with open(filename, 'r') as f:
            m.write(msg)
            map(m.write, f.readlines())


def _createBatchCommandElement(scenario, queryName, queryPath, outputDir=None, tmpFiles=True,
                               xmldb='', csvFile=None, regions=None, regionMap=None,
                               rewriters=None, rewriteParser=None, noDelete=False, saveAs=None):
    """
    Generate a <command> element for use in a multi-query batch file. The indicated
    query will be copied into a temporary file that is referenced by this <command>
    element.

    :param scenario: (str) the name of the scenario to perform the query on
    :param queryName: (str) the name of a query to execute
    :param queryPath: (str) a list of directories or XML filenames, separated
        by a colon (on Unix) or a semi-colon (on Windows)
    :param outputDir: (str) the directory in which to write the .CSV
        with query results. Defaults to value of GCAM.OutputDir.
    :param xmldb: (str) the pathname to the XML database to query, or '' to
        use in-memory DB
    :param csvFile: if None, query results are written to a computed filename.
    :param regions: (iterable of str) the regions you want to include in the query
    :param regionMap: (dict-like) keys are the names of regions that should be rewritten.
        The value is the name of the aggregate region to map into.
    :param rewriters: (list of tuples of (mapName, level)) list of mapping rewrites to
        apply to the query results, based on rewriteSets.xml. If level is specified, it
        overrides the level given in the mapping as defined in rewriteSets.xml.
    :param rewriteParser: (RewriteSetParser instance) parsed representation of
        rewriteSets.xml
    :param noDelete: (bool) if True, temporary files created by this function are
        not deleted (use for debugging)
    :param saveAs (str): alternative name to use to save the query results as
    :return: (str) the generated batch command string
    """
    basename = os.path.basename(queryName)
    mainPart, extension = os.path.splitext(basename)   # strip extension, if any

    # set default here so sphinx doc doesn't list all 32 regions
    regions = regions or GCAM_32_REGIONS

    delete = not noDelete

    # Look for both the literal name as given as well as the name with "-" and "_" replaced with " "
    queryFile = _findOrCreateQueryFile(basename, queryPath, regions, regionMap=regionMap,
                                       rewriteSetList=rewriters, rewriteParser=rewriteParser,
                                       outputDir=outputDir, tmpFiles=tmpFiles, delete=delete)

    if not queryFile:
        raise PygcamException("_createBatchCommand: file for query '%s' was not found in '%s'." % \
                              (basename, queryPath))

    if not csvFile:
        csvFile = "%s-%s.csv" % (saveAs or mainPart, scenario)    # compute default filename
        csvFile = csvFile.replace(' ', '_')                       # eliminate spaces for convenience

    outputDir = outputDir or getParam('GCAM.OutputDir')
    mkdirs(outputDir)
    csvPath = os.path.abspath(pathjoin(outputDir, csvFile))

    batchCommand = BatchCommandElement.format(scenario=scenario, queryFile=queryFile,
                                              csvFile=csvPath, xmldb=xmldb)

    return batchCommand


def createBatchFile(scenario, queries, xmldb='', queryPath=None, outputDir=None,
                    regions=None, regionMap=None, rewriteParser=None,
                    batchFileIn=None, batchFileOut=None,
                    tmpFiles=True, noDelete=False):
    """
    Create an optionally-temporary XML file that will run multiple queries, by extracting
    queries into separate temp files and referencing them from the batch query file.

    :param scenario: (str) the name of the scenario to perform the query on
    :param queries: (list of str query names and/or Query instances)
    :param xmldb: (str) path to XMLDB, or '' to use in-memory DB
    :param queryPath: (str) a list of directories or XML filenames, separated
        by a colon (on Unix) or a semi-colon (on Windows)
    :param outputDir: (str) the directory in which to write the .CSV
        with query results, default is value of GCAM.OutputDir.
    :param regions: (iterable of str) the regions you want to include in the query
    :param regionMap: (dict-like) keys are the names of regions that should be rewritten.
        The value is the name of the aggregate region to map into.
    :param rewriteParser: (RewriteSetParser instance) parsed representation of
        rewriteSets.xml
    :param batchFileIn: (str) the name of a pre-formed batch file to run
    :param batchFileOut: (str) where to write output from batchFileIn, if given
    :param tmpFiles: (bool) if True temporary files are used and deleted when the
        program exits, otherwise normal files are create in outputDir.
    :param noDelete: (bool) if True, temporary files created by this function are
        not deleted (use for debugging)
    :return: (str) the pathname of the temporary batch query file
    """
    from queryFile import Query

    commands = []

    for obj in queries:
        # handle both Query instances and simple query name strings
        if isinstance(obj, Query):
            queryName = obj.name
            rewriters = obj.rewriters
            saveAs    = obj.saveAs

            if not rewriteParser:
                raise PygcamException("No rewriteParser defined. Pass filename as argument to query sub-command or define GCAM.RewriteSetsFile")
        else:
            # Deprecated? Old style, simple list of query names.
            queryName = obj
            rewriters = None
            saveAs    = None

        queryName = queryName.strip()

        # N.B. A side-effect of this is that QueryFile elements can be commented out
        # by inserting a "#" at the start of the name, e.g., <query name="#foo" ...>
        if not queryName or queryName[0] == '#':    # ignore blank lines and comments
            continue

        # Extracts the named query into a temp file and returns XML text referencing the file.
        command = _createBatchCommandElement(scenario, queryName, queryPath, outputDir=outputDir,
                                             xmldb=xmldb, regions=regions, regionMap=regionMap,
                                             rewriters=rewriters, rewriteParser=rewriteParser,
                                             tmpFiles=tmpFiles, noDelete=noDelete, saveAs=saveAs)
        commands.append(command)

    # Add command to run pre-formed batch file, if given
    if batchFileIn:
        command = BatchCommandElement.format(scenario=scenario, queryFile=batchFileIn,
                                             csvFile=batchFileOut, xmldb=xmldb)
        commands.append(command)

    # Create the file batch-query.xml in the same dir as the CSV files. It can't be
    # a temp file because this step runs separately from the step running GCAM, and
    # the batch file would be either deleted prematurely or not at all.
    outputDir = outputDir or getParam('GCAM.OutputDir')
    if tmpFiles:
        batchFile = getTempFile(suffix='.batch.xml', delete=not noDelete, text=True)
    else:
        queryDir = pathjoin(outputDir, 'queries')
        mkdirs(queryDir)
        batchFile = pathjoin(queryDir, 'generated-batch-query.xml')

    batchCommands = ''.join(commands)
    contents = MultiCommandTemplate.format(batchCommands=batchCommands)

    saveToFile(contents, filename=batchFile)
    return batchFile


def runMultiQueryBatch(scenario, queries, xmldb='', queryPath=None, outputDir=None,
                       miLogFile=None, regions=None, regionMap=None, rewriteParser=None,
                       batchFileIn=None, batchFileOut=None, noRun=False, noDelete=False):
    """
    Create a single GCAM XML batch file that runs multiple queries, placing the
    each query's results in a file named of the form {queryName}-{scenario}.csv.

    :param scenario: (str) the name of the scenario to perform the query on
    :param queries: (list of str query names and/or Query instances)
    :param xmldb: (str) path to XMLDB, or '' to use in-memory DB
    :param queryPath: (str) a list of directories or XML filenames, separated
        by a colon (on Unix) or a semi-colon (on Windows)
    :param outputDir: (str) the directory in which to write the .CSV
        with query results, default is value of GCAM.OutputDir.
    :param regions: (iterable of str) the regions you want to include in the query
    :param regionMap: (dict-like) keys are the names of regions that should be rewritten.
        The value is the name of the aggregate region to map into.
    :param rewriteParser: (RewriteSetParser instance) parsed representation of
        rewriteSets.xml
    :param batchFileIn: (str) the name of a pre-formed batch file to run
    :param batchFileOut: (str) where to write output from batchFileIn, if given
    :param noRun: (bool) if True, print the command that would be executed, but
        don't run it.
    :param noDelete: (bool) if True, temporary files created by this function are
        not deleted (use for debugging)
    :return: none
    """
    batchFile = createBatchFile(scenario, queries, xmldb=xmldb, queryPath=queryPath,
                                outputDir=outputDir, regions=regions, regionMap=regionMap,
                                rewriteParser=rewriteParser, noDelete=noDelete,
                                batchFileIn=batchFileIn, batchFileOut=batchFileOut)

    runModelInterface(scenario, outputDir, xmldb=xmldb, batchFile=batchFile,
                      miLogFile=miLogFile, noDelete=noDelete, noRun=noRun)


# TBD: Test queryText and asDataFrame.
def runModelInterface(scenario, outputDir, csvFile=None, batchFile=None,
                      queryFile=None, queryText=None,  xmldb='',
                      miLogFile=None, noDelete=False, noRun=False,
                      asDataFrame=False):
    """
    Run a query file on the XML database given by `xmldb`, saving results
    in a CSV file.

    :param scenario: (str) the name of the scenario to perform the query on
    :param outputDir: (str) the directory in which to write the CSV
        with query results
    :param csvFile: (str) the file to create; ignored if batchFile is not None.
    :param batchFile: (str) the path to an existing batchFile; if not given,
        a temporary one will be created based on the other arguments. In no
        batchFile is specified, a csvFile must be provided.
    :param queryFile: (str) the path to the XML file holding the queries, or None
        if the batchFile is not None, in which case the queryFile should already
        be referenced in the batchFile.
    :param queryText: (str) the text of an XML query. Used only if queryFile and
        batchFile are both None, in which case the queryText is written to a tmp
        file that is used as the queryFile.
    :param xmldb: (str) the path to the XML database or '' to use in-memory DB
    :param miLogFile: (str) optional name of a log file to write ModelInterface output to.
        The value is the name of the aggregate region to map into.
    :param noRun: (bool) if True, the command is printed but not executed
    :param noDelete: (bool) if True, temporary files created by this function are
        not deleted (use for debugging)
    :param asDataFrame: (bool) see return, below
    :return: (str or DataFrame) if asDataFrame is True, return the result as a
        pandas DataFrame, if the query was successful, else None. If asDataFrame
        is False, return the absolute path to the generated CSV file if one was
        specified, else None.
    """
    filesToDelete = []   # so we can remove them when done

    def ignoreVar(name, using='batchFile'):
        _logger.warn('runModelInterface called with both %s and %s; the latter is ignored.', using, name)
        return None # just a convenience to set ignored variable to None

    if batchFile:
        if queryFile:
            queryFile = ignoreVar('queryFile')

        if queryText:
            queryText = ignoreVar('queryText')

        if csvFile:
            csvFile = ignoreVar('csvFile')

    elif queryFile and queryText:
        queryText = ignoreVar('queryText', using='queryFile')

    delete = not noDelete

    try:    # ensure that we delete temp files if needed
        if queryText:
            queryFile = getTempFile(suffix='.query.xml', delete=delete)
            filesToDelete.append(queryFile)
            _logger.debug("Creating temporary query file '%s'", queryFile)

        if csvFile and not batchFile:
            csvPath = os.path.abspath(pathjoin(outputDir, csvFile))
            _logger.debug("Writing results to: %s", csvPath)
        else:
            csvPath = None

        # TBD: test this. Might fail if not batchFile and not csvFile
        if not batchFile:
            # Create a batch file for ModelInterface to invoke the query on the named
            # scenario, and save the output where specified
            batchFile = getTempFile(suffix='.batch.xml', delete=delete)
            filesToDelete.append(batchFile)
            _logger.debug("Creating temporary batch file '%s'", batchFile)

            batchFileText = BatchQueryTemplate.format(scenario=scenario, queryFile=queryFile,
                                                      csvFile=csvPath, xmldb=xmldb)
            saveToFile(batchFileText, filename=batchFile)

        if miLogFile:
            mkdirs(os.path.dirname(miLogFile))
            if queryFile:
                _copyToLogFile(miLogFile, queryFile, "Query file: '%s'\n\n" % queryFile)
            _copyToLogFile(miLogFile, batchFile, "Batch file: '%s'\n\n" % batchFile)

        command = _createJavaCommand(batchFile, miLogFile)

        failed = False

        if noRun:
            print(command)
        else:
            _logger.debug(command)

        if getParamAsBoolean('GCAM.MI.UseVirtualBuffer'):   # deprecated as of GCAM 4.3
            with Xvfb():
                subprocess.call(command, shell=True)
        else:
            subprocess.call(command, shell=True)

        # The java program always exits with 0 status, but when the query fails,
        # it writes an error message to the CSV file. If this occurs, we delete
        # the file.
        if csvPath:
            try:
                with open(csvPath, 'r') as f:
                    line = f.readline()

                if re.search('java.*Exception', line, flags=re.IGNORECASE):
                    failed = True

            except Exception:
                failed = True

            finally:
                if failed:
                    _logger.error("Batch file '%s' failed. Deleting '%s'", queryFile, csvPath)
                    deleteFile(csvPath)
    except:
        raise

    finally:
        for filename in filesToDelete:
            TempFile.remove(filename, raiseError=False)

    if failed:
        return None

    if asDataFrame and csvPath:
        return readCsv(csvPath)

    return csvPath  # might be None if batchFile was provided; user must read file manually

def runBatchQuery(scenario, queryName, queryPath, outputDir, xmldb='',
                  csvFile=None, miLogFile=None, regions=None, regionMap=None,
                  rewriters=None, rewriteParser=None, noRun=False, noDelete=False,
                  saveAs=None):
    """
    Run a single query against GCAM's XML database given by `xmldb` (or
    computed from other parameters), saving the results into a CSV file.

    :param scenario: (str) the name of the scenario to perform the query on
    :param queryName: (str) the name of a query to execute
    :param queryPath: (str) a list of directories or XML filenames, separated
        by a colon (on Unix) or a semi-colon (on Windows)
    :param outputDir: (str) the directory in which to write the .CSV
        with query results
    :param xmldb: (str) the path to the XML database, or '' to use in-memory DB
    :param csvFile: if None, query results are written to a computed filename.
    :param miLogFile: (str) optional name of a log file to write ModelInterface output to.
    :param regions: (iterable of str) the regions you want to include in the query
    :param regionMap: (dict-like) keys are the names of regions that should be rewritten.
        The value is the name of the aggregate region to map into.
    :param rewriters: (list of tuples of (mapName, level)) list of mapping rewrites to
        apply to the query results, based on rewriteSets.xml. If level is specified, it
        overrides the level given in the mapping as defined in rewriteSets.xml.
    :param rewriteParser: (RewriteSetParser instance) parsed representation of
        rewriteSets.xml
    :param noRun: (bool) if True, the command is printed but not executed
    :param noDelete: (bool) if True, temporary files created by this function are
        not deleted (use for debugging)
    :param saveAs (str): alternative name to use to save the query results as
    :return: (str) the absolute path to the generated .CSV file, or None
    """
    basename = os.path.basename(queryName)
    mainPart, extension = os.path.splitext(basename)   # strip extension, if any

    regions = regions or GCAM_32_REGIONS # set default here so it doesn't mess up doc for this method

    delete = not noDelete

    # Look for both the literal name as given as well as the name with "-" and "_" replaced with " "
    filename = _findOrCreateQueryFile(basename, queryPath, regions, regionMap=regionMap,
                                      rewriteSetList=rewriters, rewriteParser=rewriteParser,
                                      delete=delete)
    if not filename:
        raise PygcamException("runBatchQuery: file for query '%s' was not found." % basename)

    if not csvFile:
        csvFile = "%s-%s.csv" % (saveAs or mainPart, scenario)    # compute default filename
        csvFile = csvFile.replace(' ', '_')                       # eliminate spaces for convenience

    csvPath = runModelInterface(scenario, filename, outputDir, csvFile, xmldb=xmldb,
                                miLogFile=miLogFile, noDelete=noDelete, noRun=noRun)
    return csvPath

# Deprecated: there no need for this once multiple queries is working
def _runSingleQueryBatch(scenario, xmldb='', queryNames=None, queryNodes=None, queryPath=None,
                        outputDir=None, rewriteParser=None, miLogFile=None, regions=None,
                        regionMap=None, noRun=False, noDelete=False):
    queryNames = queryNames or []
    queryNodes = queryNodes or []

    for queryName in queryNames:
        queryName = queryName.strip()

        if not queryName or queryName[0] == '#':    # allow blank lines and comments
            continue

        if queryName == 'exit':
            _logger.warn('Found "exit"; exiting batch query processing')
            return

        _logger.info("Processing query '%s'", queryName)

        runBatchQuery(scenario, queryName, queryPath, outputDir, xmldb=xmldb,
                      miLogFile=miLogFile, regions=regions, regionMap=regionMap,
                      noRun=noRun, noDelete=noDelete)

    for queryNode in queryNodes:
        queryName = queryNode.name
        saveAs    = queryNode.saveAs
        rewriters = queryNode.rewriters

        runBatchQuery(scenario, queryName, queryPath, outputDir, xmldb=xmldb,
                      miLogFile=miLogFile, regions=regions, regionMap=regionMap,
                      rewriters=rewriters, rewriteParser=rewriteParser,
                      noRun=noRun, noDelete=noDelete, saveAs=saveAs)

def sumYears(files, skiprows=1, interpolate=False):
    """
    For each file given, sum all values in each year column and create
    a file holding the result. Each resulting filename has the same basename
    as the input file but ending with '-sum.csv'.

    :param files: (list of str) Filenames to process
    :param skiprows: (int) the number of rows to skip prior to column headers
    :param interpolate: (bool) if True, interpolate annual values between time-steps
    :return: none
    """
    csvFiles = map(ensureCSV, files)
    dframes  = map(lambda fname: readCsv(fname, skiprows=skiprows, interpolate=interpolate), csvFiles)

    # TBD: preserve columns that have a single value only? Maybe this collapses into sumYearsByGroup()?
    for df, fname in zip(dframes, csvFiles):
        root, ext = os.path.splitext(fname)
        outFile = root + '-sum' + ext
        yearCols = filter(str.isdigit, df.columns)

        with open(outFile, 'w') as f:
            sums = df[yearCols].sum()
            csvText = sums.to_csv(None)
            f.write("%s\n%s\n" % (outFile, csvText))

# TBD: pass an output directory?
def sumYearsByGroup(groupCol, files, skiprows=1, interpolate=False):
    """
    Group data for each time-step (or interpolated annual values) by the given
    column (with categorical data like region or sector), and sum all
    members of each group to produce a time-series for each group. Equivalent to
    a SQL "group by" operation. For each input file, a new .CSV file is written
    with the name formed by the basename of the original file, followed by
    "-groupby-" and the groupCol. For example, given the file "foobar.csv" and
    groupCol "region", the file "foobar-groupby-region.csv" would be generated.
    Tests that all rows have the same units; otherwise raises an error.

    :param groupCol: (str) the column with categorical data to group by.
    :param files: (list of str) Filenames to process
    :param skiprows: (int) the number of rows to skip prior to column headers
    :param interpolate: (bool) if True, interpolate annual values between time-steps
    :return: none
    :raises CommandLineError: if the rows in the input file don't all have the same units
    """
    import numpy as np

    csvFiles = map(ensureCSV, files)
    dframes  = map(lambda fname: readCsv(fname, skiprows=skiprows, interpolate=interpolate), csvFiles)

    for df, fname in zip(dframes, csvFiles):
        units = df['Units'].unique()
        if len(units) != 1:
            raise CommandlineError("Can't sum results; rows have different units: %s" % units)

        root, ext = os.path.splitext(fname)
        name = groupCol.replace(' ', '_')     # eliminate spaces for general convenience
        outFile = '%s-groupby-%s%s' % (root, name, ext)

        cols = [groupCol] + filter(str.isdigit, df.columns)
        grouped = df[cols].groupby(groupCol)
        df2 = grouped.aggregate(np.sum)
        df2['Units'] = units[0]         # add these units to all rows

        with open(outFile, 'w') as f:
            csvText = df2.to_csv(None)
            label = outFile
            f.write("%s\n%s\n" % (label, csvText))

def csv2xlsx(inFiles, outFile, skiprows=0, interpolate=False, years=None, startYear=0):
    """
    Convert a set of CSV files representing GCAM query results into an XLSX file
    with an index page linked by the file names to the sheets with the results.

    :param inFiles: (list of str) the names of CSV files to read.
    :param outFile: (str) the name of the XLSX file to create
    :param skiprows: (int) the number of rows to skip in the CSV files before processing.
    :param interpolate: (bool) if True, interpolate annual values between time-steps.
    :param years: (str) the years to extract from the CSV files; must be of the form
      XXXX-YYYY, e.g. 2010-2050.
    :param startYear: (int) If interpolating, the year to begin interpolation
    :return: none
    """
    import pandas as pd

    csvFiles = map(ensureCSV, inFiles)

    try:
        dframes  = map(lambda fname: readCsv(fname, skiprows=skiprows, interpolate=interpolate,
                                             years=years, startYear=startYear), csvFiles)
    except Exception as e:
        raise CommandlineError("readCsv failed: %s" % e)

    formatStr = getParam('GCAM.ExcelNumberFormat')

    basenames = map(os.path.basename, csvFiles)
    sheetNum = 1
    outFile = ensureExtension(outFile, '.xlsx')
    with pd.ExcelWriter(outFile, engine='xlsxwriter') as writer:
        workbook = writer.book
        numFormat = workbook.add_format({'num_format': formatStr}) if formatStr else None
        linkFmt   = workbook.add_format({'font_color': 'blue', 'underline': True})

        # Create an index sheet
        indexSheet = workbook.add_worksheet('index')
        indexSheet.write_string(0, 1, 'Links to query results')
        maxlen = max(map(len, basenames))
        indexSheet.set_column('B:B', maxlen)

        for i, name in enumerate(basenames):
            row = i+1
            indexSheet.write(row, 0, row)
            indexSheet.write_url(row, 1, "internal:%d!A1" % row, linkFmt, name)

        for df, fname in zip(dframes, basenames):
            sheetName = str(sheetNum)
            sheetNum += 1
            dropExtraCols(df, inplace=True)
            df.to_excel(writer, index=None, sheet_name=sheetName, startrow=3, startcol=0)
            worksheet = writer.sheets[sheetName]

            if numFormat:
                # get the numerical indices of all column names that are numeric (i.e., years)
                yearIndices = map(lambda pair: pair[0], filter(lambda pair: str.isdigit(pair[1]), enumerate(df.columns)))
                for idx in yearIndices:
                    worksheet.set_column(idx, idx, None, numFormat)

            worksheet.write_string(0, 0, "Filename:")
            worksheet.write_string(0, 1, fname)
            worksheet.write_url(1, 0, "internal:index!A1", linkFmt, "Back to index")


def queryMain(args):
    # """
    # Main driver for query sub-command
    #
    # :param args:
    # :return: none
    # """
    miLogFile   = getParam('GCAM.MI.LogFile')
    outputDir   = args.outputDir or getParam('GCAM.OutputDir')
    groupDir    = args.groupDir
    scenario    = args.scenario
    sandbox     = args.workspace or pathjoin(getParam('GCAM.SandboxDir'), groupDir, scenario)
    xmldb       = args.xmldb     or pathjoin(sandbox, 'output', getParam('GCAM.DbFile'))
    queryPath   = args.queryPath or getParam('GCAM.QueryPath')
    queryFile   = args.queryXmlFile
    regionFile  = args.regionMap or getParam('GCAM.RegionMapFile')
    regions     = args.regions.split(',') if args.regions else GCAM_32_REGIONS
    queryNames  = args.queryName
    noDelete    = args.noDelete
    prequery    = args.prequery
    versionNum  = getParam('GCAM.VersionNumber')
    versionInfo = parse_version_info()
    inMemory        = versionInfo > (4, 2) and getParamAsBoolean('GCAM.InMemoryDatabase')
    internalQueries = versionInfo > (4, 2) and (inMemory or getParamAsBoolean('GCAM.RunQueriesInGCAM'))
    batchMultiple   = internalQueries or getParamAsBoolean('GCAM.BatchMultipleQueries')
    rewriteSetsFile = args.rewriteSetsFile or getParam('GCAM.RewriteSetsFile')
    batchFileIn  = args.batchFile
    batchFileOut = os.path.abspath(pathjoin(outputDir, args.batchOutput))

    # Post-GCAM queries are not possible when using in-memory database.
    # The 'prequery' step writes the XMLDBDriver.properties file used
    # by GCAM to query the in-memory database before exiting.
    if prequery and versionInfo <= (4, 2):
        _logger.info('Skipping pre-query step for gcam-v%s', versionNum)
        return

    if internalQueries and not prequery:
        _logger.info('Skipping post-GCAM query step: GCAM runs queries internally')
        return

    if inMemory and not prequery:
        _logger.info('Skipping post-GCAM query step: using in-memory database')
        return

    if not (xmldb or inMemory):
        raise CommandlineError('Must specify xmldb if not using in-memory database')

    if queryNames:
        _logger.debug("Query names: %s", queryNames)

    queryFileObj = QueryFile.parse(queryFile) if queryFile else None
    queryNodes = queryFileObj.queries if queryFileObj else []
    queries = queryNames + queryNodes

    if not (queryNames or queryFileObj or batchFileIn):
        raise CommandlineError("Error: At least one query name, query XML, or batch file must be specified")

    rewriteParser = RewriteSetParser.parse(rewriteSetsFile) if rewriteSetsFile else None
    regionMap = readRegionMap(regionFile) if regionFile else None

    mkdirs(outputDir)

    # TBD: if setupWorkspace is performed in 'setup' step, can be removed from here
    # When writing the XMLDB to disk, we call ModelInterface separately. When
    # using an in-memory database, GCAM runs the queries for us, so here we
    # just create the XMLDBDriver.properties and batch files and return.
    if args.prequery:
        exeDir = getExeDir(sandbox, chdir=True)
        batchFile = createBatchFile(scenario, queries, queryPath=queryPath, outputDir=outputDir,
                                    regions=regions, regionMap=regionMap, rewriteParser=rewriteParser,
                                    batchFileIn=batchFileIn, batchFileOut=batchFileOut,
                                    tmpFiles=False, noDelete=noDelete) \
            if batchMultiple else ''

        filterFile = getParam('GCAM.FilterFile')
        writeXmldbDriverProperties(inMemory=inMemory, outputDir=exeDir, filterFile=filterFile,
                                   batchFile=batchFile, batchLog='logs/batch-query.log')
        return

    xmldb = os.path.abspath(xmldb)

    if miLogFile:
        miLogFile = os.path.abspath(pathjoin(outputDir, miLogFile))
        deleteFile(miLogFile)       # remove it, if any, to start fresh

    # If not a prequery step, we're running queries post-GCAM, which means a database on disk
    # For now, we support running multiple queries in a single batch file, or the old way,
    # running each one individually. The latter is probably not needed, except for debugging.
    if batchMultiple:
        runMultiQueryBatch(scenario, queries, xmldb=xmldb, queryPath=queryPath, outputDir=outputDir,
                           miLogFile=miLogFile, regions=regions, regionMap=regionMap,
                           batchFileIn=batchFileIn, batchFileOut=batchFileOut,
                           rewriteParser=rewriteParser, noRun=args.noRun, noDelete=noDelete)
    else:
        # (Deprecated) Otherwise run them individually.
        _runSingleQueryBatch(scenario, xmldb=xmldb, queryNames=queryNames, queryNodes=queryNodes,
                             queryPath=queryPath, outputDir=outputDir, rewriteParser=rewriteParser,
                             miLogFile=miLogFile, regions=regions, regionMap=regionMap,
                             noRun=args.noRun, noDelete=args.noDelete)
