"""Script and API to load DM+D databases into an overall DM+D warehouse
database. This only has a subset of the data in the full DM+D databases but is
useful for mapping as it will have full coverage of current and historical
codes.
"""
# Importing the version number (in __init__.py) and the package name
from dmd import (
    __version__,
    __name__ as pkg_name,
    common,
    query,
    orm
)
from dmd.warehouse import (
    orm as worm,
    query as wquery,
    join
)
from sqlalchemy.orm.exc import (
    NoResultFound
)
from sqlalchemy_config import config as cfg
# from zipfile import ZipFile
# from datetime import datetime
# from collections import namedtuple
# from xml.etree import ElementTree
from tqdm import tqdm
# import shutil
# import glob
import argparse
# import os
# import re
# import tempfile
import pprint as pp


_PROG_NAME = 'dmd-warehouse'
"""The name of the program (`str`)
"""
_DESC = __doc__
"""The program description (`str`)
"""
_WH_IDX = 0
"""The index position of the DM+D warehouse rows in any join (`int`)
"""
_DMD_IDX = 1
"""The index position of the DM+D database rows in any join (`int`)
"""
_UPDATE_HASH_IDX = 1
"""The index position of a data hash (used for testing updates) in any join
(`int`)
"""
_UPDATE_DATA_IDX = 2
"""The index position of DM+D data rows (used for setting updated data) in any
join (`int`)
"""
_CODE_DATA_IDX = 1
"""The index position of DM+D data rows (used for adding new codes) in any
join (`int`)
"""
_REG_NAME_TYPE = 'name'
"""The regular name type for a VMP/AMP record (`str`)
"""
_DESC_NAME_TYPE = 'desc'
"""The description (long) name type for a AMP record (`str`)
"""
_SHORT_NAME_TYPE = 'short'
"""The short name type for a VMP/AMP record (`str`)
"""
_PREV_NAME_TYPE = 'prev'
"""The previous name type for a VMP/AMP record (`str`)
"""
_AMP_NAME_FIELDS = [
    ('amp_name', _REG_NAME_TYPE),
    ('amp_desc', _DESC_NAME_TYPE),
    ('amp_name_abbrev', _SHORT_NAME_TYPE),
    ('amp_name_prev', _PREV_NAME_TYPE)
]
"""Actual medicinal product name fields (ORM attribute names - ``[0]``) and
their types ``[1]`` (`list` of `tuple`)
"""
_VMP_NAME_FIELDS = [
    ('vmp_name', _REG_NAME_TYPE),
    ('vmp_name_abbrev', _SHORT_NAME_TYPE),
    ('vmp_name_prev', _PREV_NAME_TYPE)
]
"""Virtual medicinal product name fields (ORM attribute names - ``[0]``) and
their types ``[1]`` (`list` of `tuple`)
"""


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class _JoinUpdateEvaluator(object):
    """
    """
    TYPE = "rows"

    def __init__(self, session):
        """
        """
        self.session = session

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def evaluate(self, version, row, *args, **kwargs):
        """Evaluate a join between the warehouse database and the DM+D database
        that is being processed for any updated DM+D (VMP/AMP) names.

        Parameters
        ----------
        session : `sqlalchemy.Session`
            For interaction with the warehouse database.
        name_fields : `list` of `tuple`
            Each tuple contains: ``[0]`` The field names (ORM attributes) where
            the AMP/VMP code names are located (used when adding new names).
            ``[1]`` The name type - i.e. ``short``, ``desc``, ``prev``.
        version : `dmd.warehouse.orm.DmdVersion`
            The version of the DM+D currently being added.
        row : `list` of `merge_sort.join.JoinRows`
            The joined data, this will have the warehouse rows at ``[0]`` and
            the DM+D rows at ``[1]``.

        Notes
        -----
        This assumes that new names are added else where and that updated
        refers to the set not individual names, so it could only be a single
        name that has been updated but a whole new set is stored for the
        version.
        """
        updated = False

        # The entry is in the warehouse and the current DM+D, so we need to
        # check if it has been updated or not
        if len(row[_WH_IDX]) == 1 and len(row[_DMD_IDX]) == 1:
            # Extract the lists from the joined rows, at this point we know
            # there is only a single row in each
            wh_row = row[_WH_IDX][0]
            dmd_row = row[_DMD_IDX][0]

            # Have there been changes?
            if wh_row[_UPDATE_HASH_IDX] != dmd_row[_UPDATE_HASH_IDX]:
                # So, the DM+D data row should be unique (i.e. any VMP/AMP IDs
                # should be unique)
                if len(dmd_row[_UPDATE_DATA_IDX]) > 1:
                    raise IndexError(
                        "incorrect {2} data rows: '{0}' vs '{1}' "
                        "(WH vs. DM+D)".format(
                            len(wh_row[_UPDATE_DATA_IDX]),
                            len(dmd_row[_UPDATE_DATA_IDX]),
                            self.TYPE
                        )
                    )
                # code clarity
                dmd_data_row = dmd_row[_UPDATE_DATA_IDX][0]

                # We want to get the DmdCode object to associate with the
                # updated DmdNames, I tried just assigning the dmd_id field but
                # it was set to None by SQLAlchemy!?
                dmd_ids = set([])
                for i in wh_row[_UPDATE_DATA_IDX]:
                    dmd_ids.add(i.dmd_code.dmd_id)

                if len(dmd_ids) > 1:
                    for i in wh_row[_UPDATE_DATA_IDX]:
                        print(self.TYPE)
                        print(i)
                        print("CODE")
                        print(i.dmd_code)
                    raise IndexError(
                        "multiple {1} mapping to {0} dmd_ids".format(
                            len(dmd_ids), self.TYPE
                        )
                    )
                # Get the DmdCode object to associate with the updated DmdNames
                dmd_code = wh_row[_UPDATE_DATA_IDX][0].dmd_code

                # Now that we are performing an update any existing warehouse
                # names are no longer the latest
                for whr in wh_row[_UPDATE_DATA_IDX]:
                    if whr.is_latest is True:
                        whr.is_latest = False
                        self.session.add(whr)

                # Get the updated names, note that updated refers to the set
                # not individual names, so it could only be a single name that
                # has been updated but a whole new set is stored for the
                # version
                updated_names = self.get_updated_records(
                    dmd_data_row=dmd_data_row, dmd_code=dmd_code,
                    version=version, is_latest=True
                )
                for n in updated_names:
                    self.session.add(n)
                    updated = True
        return updated

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_updated_records(*args, **kwargs):
        """
        """
        return []


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class VmpNameUpdateEvaluator(_JoinUpdateEvaluator):
    """
    """
    TYPE = "VMP names"
    NAME_FIELDS = _VMP_NAME_FIELDS

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_updated_records(self, *args, dmd_data_row=None, version=None,
                            dmd_code=None, is_latest=False, **kwargs):
        """
        """
        return _get_names_for_code(
            dmd_data_row, self.NAME_FIELDS, version=version, dmd_code=dmd_code,
            is_latest=True
        )


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class AmpNameUpdateEvaluator(VmpNameUpdateEvaluator):
    """
    """
    TYPE = "AMP names"
    NAME_FIELDS = _AMP_NAME_FIELDS


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class VmpUnitDoseUpdateEvaluator(_JoinUpdateEvaluator):
    """
    """
    TYPE = "VMP unit dose"
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_updated_records(self, *args, dmd_data_row=None, version=None,
                            dmd_code=None, is_latest=False, **kwargs):
        """
        """
        return [
            _get_unit_dose_for_code(
                dmd_code, dmd_data_row, version=version, is_latest=is_latest
            )
        ]


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class AmpUnitDoseUpdateEvaluator(VmpUnitDoseUpdateEvaluator):
    """
    """
    TYPE = "AMP unit dose"

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_updated_records(self, *args, dmd_data_row=None, **kwargs):
        """
        """
        return super().get_updated_records(
            *args, dmd_data_row=dmd_data_row.virt_med_prod, **kwargs
        )


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class VmpIngredientUpdateEvaluator(_JoinUpdateEvaluator):
    """
    """
    TYPE = "VMP ingredients"

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_updated_records(self, *args, dmd_data_row=None, version=None,
                            dmd_code=None, is_latest=False, **kwargs):
        """
        """
        return _get_ingredients_for_code(
                dmd_code, dmd_data_row, version=version, is_latest=is_latest
        )


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class AmpIngredientUpdateEvaluator(VmpUnitDoseUpdateEvaluator):
    """
    """
    TYPE = "AMP ingredients"

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def get_updated_records(self, *args, dmd_data_row=None, **kwargs):
        """
        """
        return super().get_updated_records(
            *args, dmd_data_row=dmd_data_row.virt_med_prod, **kwargs
        )


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def main():
    """The main entry point for the script.
    """
    # Initialise and parse the command line arguments
    parser = _init_cmd_args()
    args = _parse_cmd_args(parser)

    # Start a msg output, this will respect the verbosity and will output to
    # STDERR
    m = common.Msg(prefix='', verbose=args.verbose, file=common.MSG_OUT)
    m.msg("= {2} ({0} v{1}) =".format(
        pkg_name, __version__, _PROG_NAME
    ))
    m.prefix = common.MSG_PREFIX
    m.msg_atts(args)

    # Get a sessionmaker to create sessions to interact with the database
    wh_sm = cfg.get_sessionmaker(
        args.whurl, common._DEFAULT_PREFIX, url_arg=args.whurl,
        config_arg=args.config, config_env=None,
        config_default=common._DEFAULT_CONFIG
    )

    if args.option == 'add':
        # Make sure the database exists (or does not exist)
        cfg.create_db(wh_sm, exists=not args.create)
        _create_tables(wh_sm, exists=not args.create)

        # Get a sessionmaker to create sessions to interact with the database
        add_sm = cfg.get_sessionmaker(
            args.addurl, common._DEFAULT_PREFIX, url_arg=args.addurl,
            config_arg=args.config, config_env=None,
            config_default=common._DEFAULT_CONFIG
        )

        # The main entry point for adding a database
        add_database(wh_sm, add_sm, verbose=args.verbose, init=args.create)
    elif args.option == 'list':
        # Make sure the database exists (or does not exist)
        cfg.create_db(wh_sm, exists=not args.create)
        _create_tables(wh_sm, exists=not args.create)

        all_dbs = []
        with open(args.list_file) as infile:
            for row in infile:
                all_dbs.append(row.strip())

        if len(set(all_dbs)) != len(all_dbs):
            raise ValueError("duplicated entries in database list")

        wh_session = wh_sm()
        try:
            dbs_to_add = get_missing_dbs(
                wh_session,
                args.config,
                all_dbs,
                verbose=args.verbose
            )
        finally:
            wh_session.close()

        m.msg("need to add {0} databases".format(len(dbs_to_add)))
        for conn_str, day_no, version_name in dbs_to_add:
            m.msg("adding {0} ({1} days)".format(version_name, day_no))
            dmd_sm = cfg.get_sessionmaker(
                conn_str, common._DEFAULT_PREFIX, url_arg=conn_str,
                config_arg=args.config, config_env=None,
                config_default=common._DEFAULT_CONFIG
            )
            # The main entry point for adding a database
            add_database(wh_sm, dmd_sm, verbose=args.verbose, init=args.create)
    elif args.option == 'remove':
        remove_database(args.name)
    else:
        raise ValueError("unknown option: {0}".format(args.option))
    m.msg("*** END ***")


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def get_missing_dbs(session, config, conn_list, verbose=False):
    """Identify DM+D databases not yet loaded into the warehouse.

    Iterates over a list of database connection strings, checks each
    against the warehouse, and returns those that are missing.
    Databases must be added sequentially; a gap raises ``ValueError``.

    Parameters
    ----------
    session : `sqlalchemy.orm.Session`
        Session connected to the warehouse database.
    config : `str` or ``None``
        Path to the database configuration file.
    conn_list : `list` of `str`
        Connection strings or SQLite paths to check.
    verbose : `bool`, optional, default: ``False``
        Show progress bar.

    Returns
    -------
    dbs_to_add : `list` of `tuple`
        Each tuple is ``(conn_str, day_no, version_name, present)``.

    Raises
    ------
    ValueError
        If a non-sequential gap is detected in the database versions.
    """
    db_info = []
    for row in tqdm(conn_list, desc="[info] testing databases",
                    disable=not verbose, unit=" files",
                    ncols=0, dynamic_ncols=False):
        row = row.strip()

        # Get a sessionmaker to create sessions to interact with the
        # database
        dmd_sm = cfg.get_sessionmaker(
            row, common._DEFAULT_PREFIX, url_arg=row,
            config_arg=config, config_env=None,
            config_default=common._DEFAULT_CONFIG
        )
        dmd_session = dmd_sm()
        day_no = query.get_version_day(dmd_session)
        version_name = query.get_version_name(dmd_session)

        try:
            wquery.get_version_from_days(session, day_no)
            present = True
        except NoResultFound:
            present = False
        db_info.append((row, day_no, version_name, present))
        dmd_session.close()

    db_info.sort(key=lambda x: x[1])

    dbs_to_add = []
    missing_found = False
    first_missing = None
    for row, day_no, version_name, is_present in db_info:
        current_db_str = "{0} ({1} days)".format(version_name, day_no)
        if is_present is True and missing_found is True:
            raise ValueError(
                "you are attemting to add an old database to the "
                "warehouse. Databases must be added sequentially."
                " first_missing '{0}', current '{1}'".format(
                    first_missing, current_db_str
                )
            )
        elif is_present is False:
            missing_found = True
            first_missing = current_db_str
            dbs_to_add.append((row, day_no, version_name))
    return dbs_to_add


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def add_database(wh_sm, dmd_sm, verbose=False, init=False):
    """Add a database to the warehouse.

    Parameters
    ----------
    wh_sm : `sqlalchemy.Sessionmaker`
        For interaction with the warehouse database.
    add_sm : `sqlalchemy.Sessionmaker`
        For interaction with the database being added.
    verbose : `bool`, optional, default: `False`
        Track progress.
    init : `bool`, optional, default: `False`
        Is the database being initialised for the first time?
    """
    dmd_session = dmd_sm()
    wh_session = wh_sm()

    day_no = query.get_version_day(dmd_session)
    version_name = query.get_version_name(dmd_session)
    # _test_latest(wh_session, day_no)

    # Add the version to the warehouse
    version = _add_version(wh_session, day_no, version_name)

    add_vmps(wh_session, dmd_session, version, verbose=verbose)
    update_vmp_names(wh_session, dmd_session, version, verbose=verbose)
    update_vmp_unit_dose(wh_session, dmd_session, version, verbose=verbose)
    update_vmp_ingredients(wh_session, dmd_session, version, verbose=verbose)
    add_amps(wh_session, dmd_session, version, verbose=verbose)
    update_amp_names(wh_session, dmd_session, version, verbose=verbose)
    update_amp_unit_dose(wh_session, dmd_session, version, verbose=verbose)
    update_amp_ingredients(wh_session, dmd_session, version, verbose=verbose)


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~d
def add_vmps(wh_session, dmd_session, version, verbose=False):
    """Add and commit the ids from the virtual medicinal product table

    Parameters
    ----------
    wh_session : `sqlalchemy.Session`
        For interaction with the warehouse database.
    dmd_session : `sqlalchemy.Session`
        For interaction with the DM+D database being added.
    version : `dmd.warehouse.orm.DmdVersion`
        The version of the DM+D currently being added.
    verbose : `bool`, optional, default: `False`
        Track progress.

    Notes
    -----
    If there are no VMP codes, then this will propagate it with both the
    VMP ID and the previous VMP ID where it is not None. Otherwise, just the
    current VMP IDs are added. For any IDs added, this will add VMP names.
    However, this does not check for any name updated VMP names.
    """
    m = common.Msg(prefix='[info]', verbose=verbose, file=common.MSG_OUT)

    join_func = join.yield_current_vmp_ids

    # Count VMPs only
    if wquery.get_dmd_id_count(wh_session, is_amp=False) == 0:
        join_func = join.yield_all_vmp_ids

    nadded = 0
    j = join.join_on_ids(
        join.yield_dmd_ids(wh_session),
        join_func(dmd_session)
    )
    for i in tqdm(j, desc="[info] testing VMPs...", unit=" VMPs",
                  disable=not m.verbose):
        nadded += _eval_code_join_row(
            wh_session, _VMP_NAME_FIELDS,
            lambda x: x[_DMD_IDX][0][_CODE_DATA_IDX],
            version, i
        )
    m.msg("added {0} VMP IDs".format(nadded))
    wh_session.commit()


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def add_amps(wh_session, dmd_session, version, verbose=False):
    """Add and commit the ids from the actual medicinal product table.

    Parameters
    ----------
    wh_session : `sqlalchemy.Session`
        For interaction with the warehouse database.
    dmd_session : `sqlalchemy.Session`
        For interaction with the DM+D database being added.
    version : `dmd.warehouse.orm.DmdVersion`
        The version of the DM+D currently being added.
    verbose : `bool`, optional, default: `False`
        Track progress.

    Notes
    -----
    For any IDs added, this will add AMP names. However, this does not check
    for any name updated AMP names.
    """
    m = common.Msg(prefix='[info]', verbose=verbose, file=common.MSG_OUT)

    nadded = 0
    j = join.join_on_ids(
        join.yield_dmd_ids(wh_session),
        join.yield_amp_ids(dmd_session)
    )
    for i in tqdm(j, desc="[info] testing AMPs...", unit=" AMPs",
                  disable=not m.verbose):
        nadded += _eval_code_join_row(
            wh_session, _AMP_NAME_FIELDS,
            lambda x: x[_DMD_IDX][0][_CODE_DATA_IDX].virt_med_prod,
            version, i
        )
    m.msg("added {0} AMP IDs".format(nadded))
    wh_session.commit()


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def _eval_code_join_row(session, name_fields, vmp_data_func, version, row):
    """Evaluate a join between the warehouse database and the DM+D database
    that is being processed to determine if the DM+D row needs to be added to
    the warehouse.

    Parameters
    ----------
    session : `sqlalchemy.Session`
        For interaction with the warehouse database.
    name_fields : `list` of `tuple`
        Each tuple contains: ``[0]`` The field names (ORM attributes) where the
        AMP/VMP code names are located (used when adding new names). ``[1]``
        The name type - i.e. ``short``, ``desc``, ``prev``.
    vmp_data_func : `func`
        A function that accepts the ``row`` and will return the vmp data
        (`dmd.orm.VirtMedProd`) this will supply the VMP data used in the
        warehouse entry.
    version : `dmd.warehouse.orm.DmdVersion`
        The version of the DM+D currently being added.
    row : `list` of `merge_sort.join.JoinRows`
        The joined data, this will have the warehouse rows at ``[0]`` and the
        DM+D rows at ``[1]``.

    Notes
    -----
    This will add any AMP/VMP entries that are not present and also add names
    associated with them. This does not check for additional names that have
    been updated.
    """
    added = False

    # Not in the ware house, then we need to add and also add the names.
    # len(row[_DMD_IDX]) may be > 1 when yield_all_vmp_ids produces
    # duplicates via heapq.merge (current ID + previous ID for the same
    # VMP).  We take the first row in that case.
    if len(row[_WH_IDX]) == 0 and len(row[_DMD_IDX]) >= 1:
        # Get the VMP ID (the join field) an the VirtMedProd data row
        code_id = row[_DMD_IDX][0][0]
        rec = row[_DMD_IDX][0][1]
        vmp_data = vmp_data_func(row)

        # Create the DM+D code from the vmp data
        dmd_code = worm.DmdCode(
            dmd_id=code_id, is_amp=False, vmp_id=vmp_data.vmp_id,
            dmd_version=version
        )
        # Now add the names for the code
        dmd_names = _get_names_for_code(
            rec, name_fields, version=version, dmd_code=dmd_code,
            is_latest=True
        )
        # Now add the names for the code
        unit_dose = _get_unit_dose_for_code(
            dmd_code, vmp_data, version=version, is_latest=True
        )

        # Now add the names for the code
        ingredients = _get_ingredients_for_code(
            dmd_code, vmp_data, version=version, is_latest=True
        )

        session.add(dmd_code)
        session.add(unit_dose)
        for i in dmd_names:
            session.add(i)

        for i in ingredients:
            session.add(i)

        added = True
    elif len(row[_DMD_IDX]) == 0 and len(row[_WH_IDX]) == 1:
        pass
    elif len(row[_DMD_IDX]) >= 1 and len(row[_WH_IDX]) == 1:
        pass
    else:
        # Sanity check
        raise IndexError(
            "incorrect join numbers: '{0}' vs '{1}' (WH vs. DM+D)".format(
                len(row[_WH_IDX]), len(row[_DMD_IDX])
            )
        )
    return added


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def update_vmp_names(wh_session, dmd_session, version, verbose=False):
    """Update the warehouse with any VMP names that have changed.

    Parameters
    ----------
    wh_session : `sqlalchemy.Session`
        For interaction with the warehouse database.
    add_session : `sqlalchemy.Session`
        For interaction with the database being added.
    version : `dmd.warehouse.orm.DmdVersion`
        The version of the DM+D currently being added.
    verbose : `bool`, optional, default: `False`
        Track progress.

    Notes
    -----
    The VMP name refers to the any non null strings that are in the
    ``virt_med_prod`` table under the fields: ``vmp_name``, ``vmp_name_prev``,
    ``vmp_name_abbrev``.

    This assumes that new names are added else where and that updated refers to
    the set not individual names, so it could only be a single name that has
    been updated but a whole new set is stored for the version.
    """
    # Set up the join with the warehouse at 0 and the DM+D at 1
    j = join.join_on_ids(
        join.yield_wh_dmd_names(wh_session),
        join.yield_dmd_vmp_names(dmd_session)
    )

    updater = VmpNameUpdateEvaluator(wh_session)
    _update_data(wh_session, dmd_session, version, j, updater, verbose=verbose)


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def update_amp_names(wh_session, dmd_session, version, verbose=False):
    """Update the warehouse with any AMP names that have changed.

    Parameters
    ----------
    wh_session : `sqlalchemy.Session`
        For interaction with the warehouse database.
    add_session : `sqlalchemy.Session`
        For interaction with the database being added.
    version : `dmd.warehouse.orm.DmdVersion`
        The version of the DM+D currently being added.
    verbose : `bool`, optional, default: `False`
        Track progress.

    Notes
    -----
    The AMP name refers to the any non null strings that are in the
    ``act_med_prod`` table under the fields: ``amp_name``, ``amp_name_prev``,
    ``amp_name_abbrev`` and ``amp_desc``.

    This assumes that new names are added else where and that updated refers to
    the set not individual names, so it could only be a single name that has
    been updated but a whole new set is stored for the version.
    """
    # Set up the join with the warehouse at 0 and the DM+D at 1
    j = join.join_on_ids(
        join.yield_wh_dmd_names(wh_session),
        join.yield_dmd_amp_names(dmd_session)
    )

    updater = AmpNameUpdateEvaluator(wh_session)
    _update_data(wh_session, dmd_session, version, j, updater, verbose=verbose)


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def update_vmp_unit_dose(wh_session, dmd_session, version, verbose=False):
    """Update the warehouse with any VMP names that have changed.

    Parameters
    ----------
    wh_session : `sqlalchemy.Session`
        For interaction with the warehouse database.
    add_session : `sqlalchemy.Session`
        For interaction with the database being added.
    version : `dmd.warehouse.orm.DmdVersion`
        The version of the DM+D currently being added.
    verbose : `bool`, optional, default: `False`
        Track progress.

    Notes
    -----
    The VMP name refers to the any non null strings that are in the
    ``virt_med_prod`` table under the fields: ``vmp_name``, ``vmp_name_prev``,
    ``vmp_name_abbrev``.

    This assumes that new names are added else where and that updated refers to
    the set not individual names, so it could only be a single name that has
    been updated but a whole new set is stored for the version.
    """
    # Set up the join with the warehouse at 0 and the DM+D at 1
    j = join.join_on_ids(
        join.yield_wh_dmd_unit_dose(wh_session),
        join.yield_dmd_vmp_unit_dose(dmd_session)
    )

    updater = VmpUnitDoseUpdateEvaluator(wh_session)
    _update_data(wh_session, dmd_session, version, j, updater, verbose=verbose)


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def update_amp_unit_dose(wh_session, dmd_session, version, verbose=False):
    """Update the warehouse with any VMP names that have changed.

    Parameters
    ----------
    wh_session : `sqlalchemy.Session`
        For interaction with the warehouse database.
    add_session : `sqlalchemy.Session`
        For interaction with the database being added.
    version : `dmd.warehouse.orm.DmdVersion`
        The version of the DM+D currently being added.
    verbose : `bool`, optional, default: `False`
        Track progress.

    Notes
    -----
    The VMP name refers to the any non null strings that are in the
    ``virt_med_prod`` table under the fields: ``vmp_name``, ``vmp_name_prev``,
    ``vmp_name_abbrev``.

    This assumes that new names are added else where and that updated refers to
    the set not individual names, so it could only be a single name that has
    been updated but a whole new set is stored for the version.
    """
    # Set up the join with the warehouse at 0 and the DM+D at 1
    j = join.join_on_ids(
        join.yield_wh_dmd_unit_dose(wh_session),
        join.yield_dmd_amp_unit_dose(dmd_session)
    )

    updater = AmpUnitDoseUpdateEvaluator(wh_session)
    _update_data(wh_session, dmd_session, version, j, updater, verbose=verbose)


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def update_vmp_ingredients(wh_session, dmd_session, version, verbose=False):
    """Update the warehouse with any VMP names that have changed.

    Parameters
    ----------
    wh_session : `sqlalchemy.Session`
        For interaction with the warehouse database.
    add_session : `sqlalchemy.Session`
        For interaction with the database being added.
    version : `dmd.warehouse.orm.DmdVersion`
        The version of the DM+D currently being added.
    verbose : `bool`, optional, default: `False`
        Track progress.

    Notes
    -----
    The VMP name refers to the any non null strings that are in the
    ``virt_med_prod`` table under the fields: ``vmp_name``, ``vmp_name_prev``,
    ``vmp_name_abbrev``.

    This assumes that new names are added else where and that updated refers to
    the set not individual names, so it could only be a single name that has
    been updated but a whole new set is stored for the version.
    """
    # Set up the join with the warehouse at 0 and the DM+D at 1
    j = join.join_on_ids(
        join.yield_wh_dmd_ingredients(wh_session),
        join.yield_dmd_vmp_ingredients(dmd_session)
    )

    updater = VmpIngredientUpdateEvaluator(wh_session)
    _update_data(wh_session, dmd_session, version, j, updater, verbose=verbose)


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def update_amp_ingredients(wh_session, dmd_session, version, verbose=False):
    """Update the warehouse with any VMP names that have changed.

    Parameters
    ----------
    wh_session : `sqlalchemy.Session`
        For interaction with the warehouse database.
    add_session : `sqlalchemy.Session`
        For interaction with the database being added.
    version : `dmd.warehouse.orm.DmdVersion`
        The version of the DM+D currently being added.
    verbose : `bool`, optional, default: `False`
        Track progress.

    Notes
    -----
    The VMP name refers to the any non null strings that are in the
    ``virt_med_prod`` table under the fields: ``vmp_name``, ``vmp_name_prev``,
    ``vmp_name_abbrev``.

    This assumes that new names are added else where and that updated refers to
    the set not individual names, so it could only be a single name that has
    been updated but a whole new set is stored for the version.
    """
    # Set up the join with the warehouse at 0 and the DM+D at 1
    j = join.join_on_ids(
        join.yield_wh_dmd_ingredients(wh_session),
        join.yield_dmd_amp_ingredients(dmd_session)
    )

    updater = AmpIngredientUpdateEvaluator(wh_session)
    _update_data(wh_session, dmd_session, version, j, updater, verbose=verbose)


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def _update_data(wh_session, dmd_session, version, data_join, updater,
                 verbose=False):
    """Update the warehouse with any VMP names that have changed.

    Parameters
    ----------
    wh_session : `sqlalchemy.Session`
        For interaction with the warehouse database.
    add_session : `sqlalchemy.Session`
        For interaction with the database being added.
    version : `dmd.warehouse.orm.DmdVersion`
        The version of the DM+D currently being added.
    verbose : `bool`, optional, default: `False`
        Track progress.

    Notes
    -----
    The VMP name refers to the any non null strings that are in the
    ``virt_med_prod`` table under the fields: ``vmp_name``, ``vmp_name_prev``,
    ``vmp_name_abbrev``.

    This assumes that new names are added else where and that updated refers to
    the set not individual names, so it could only be a single name that has
    been updated but a whole new set is stored for the version.
    """
    data_type = updater.TYPE
    nupdated = 0
    m = common.Msg(prefix='[info]', verbose=verbose, file=common.MSG_OUT)

    for row in tqdm(data_join, desc="[info] testing {0}...".format(data_type),
                    unit=" {0}".format(data_type),
                    disable=not m.verbose):
        nupdated += updater.evaluate(version, row)
    m.msg("updated {0} {1}".format(nupdated, data_type))
    wh_session.commit()


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def _get_names_for_code(row, name_fields, version=None, dmd_code=None,
                        is_latest=False):
    """Get ``DmdName`` objects for each of the defined names in a AMP/VMP.

    Parameters
    ----------
    row : `dmd.orm.VirtMedProd` or `dmd.orm.ActMedProd`
        A virtual/actual medicinal product.
    name_fields : `list` of `tuple`
        Each tuple contains: ``[0]`` The field names (ORM attributes) where the
        AMP/VMP code names are located (used when adding new names). ``[1]``
        The name type - i.e. ``short``, ``desc``, ``prev``.
    version : `dmd.warehouse.orm.DmdVersion`, optional, default: `None`
        The version object, if not provided then it is not added.
    dmd_code : `dmd.warehouse.orm.DmdCode`, optional, default: `None`
        The code object, if not provided then it is not added.
    is_latest : `bool`, optional, default: `False`
        The ``is_latest`` flag.

    Returns
    -------
    names : `list` of `dmd.warehouse.orm.DmdName`
        The names that have been generated.
    """
    names = []
    # attribute name, name type
    for a, t in name_fields:
        n = getattr(row, a)
        if n is not None:
            names.append(
                worm.DmdName(
                    dmd_code=dmd_code,
                    dmd_version=version,
                    is_latest=is_latest,
                    dmd_name=n,
                    dmd_name_type=t
                )
            )
    return names


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def _get_unit_dose_for_code(dmd_code, vmp, version=None, is_latest=False):
    """Get ``UnitDose`` objects for each of the defined names in a AMP/VMP.

    Parameters
    ----------
    dmd_code : `dmd.warehouse.orm.DmdCode`
        The code object.
    vmp : `func`
        The virtual medical product relating to the dmd_code.
    version : `dmd.warehouse.orm.DmdVersion`, optional, default: `None`
        The version object, if not provided then it is not added.
    is_latest : `bool`, optional, default: `False`
        The ``is_latest`` flag.

    Returns
    -------
    unit_dose : ``dmd.warehouse.orm.UnitDose`
        The unit dose object.
    """
    try:
        unit_dose_form_type = vmp.unit_meas_ud.unit_meas_desc
    except AttributeError:
        unit_dose_form_type = None

    try:
        unit_dose_form_units = vmp.unit_meas.unit_meas_desc
    except AttributeError:
        unit_dose_form_units = None

    return worm.UnitDose(
        dmd_code=dmd_code,
        dmd_version=version,
        is_latest=is_latest,
        dose_form_ind=vmp.dose_form_ind.dose_form_ind_desc,
        unit_dose_form_type=unit_dose_form_type,
        unit_dose_form_size=vmp.unit_dose_form_size,
        unit_dose_form_units=unit_dose_form_units
    )


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def _get_ingredients_for_code(dmd_code, vmp, version=None, is_latest=False):
    """Get ``Ingredient`` objects that map to the VMP.

    Parameters
    ----------
    dmd_code : `dmd.warehouse.orm.DmdCode`
        The code object.
    vmp : `func`
        The virtual medical product relating to the dmd_code.
    version : `dmd.warehouse.orm.DmdVersion`, optional, default: `None`
        The version object, if not provided then it is not added.
    is_latest : `bool`, optional, default: `False`
        The ``is_latest`` flag.

    Returns
    -------
    unit_dose : ``dmd.warehouse.orm.UnitDose`
        The unit dose object.

    Notes
    -----
    Although the ingredients are mapped to a VMP in the DM+D, in the warehouse
    they will also be assigned to an AMP.
    """
    ingredients = []
    for i in join.get_vmp_ingredients(vmp):
        ingredients.append(
            worm.Ingredient(
                dmd_code=dmd_code,
                dmd_version=version,
                is_latest=is_latest,
                **i
            )
        )
    return ingredients
    # vpis = vmp.virt_prod_ing
    # if not isinstance(vpis, list):
    #     print(vpis)
    #     print(type(vpis))
    #     raise TypeError("expecting list")

    # wh_ings = []
    # for i in vpis:
    #     ing_id = i.vmp_ingredient_id,
    #     ing_obj = i.ingredient.ingredient_name,
    #     i.stren_value_num
    #     i.stren_value_num_unit_meas.unit_meas_desc
    #     i.stren_value_den
    #     i.stren_value_den_unit_meas.unit_meas_desc

    # try:
    #     unit_dose_form_type = vmp.unit_meas_ud.unit_meas_desc
    # except AttributeError:
    #     unit_dose_form_type = None

    # try:
    #     unit_dose_form_units = vmp.unit_meas.unit_meas_desc
    # except AttributeError:
    #     unit_dose_form_units = None

    # return worm.Ingredient(
    #     dmd_code=dmd_code,
    #     dmd_version=version,
    #     is_latest=is_latest,
    #     dose_form_ind=vmp.dose_form_ind.dose_form_ind_desc,
    #     unit_dose_form_type=unit_dose_form_type,
    #     unit_dose_form_size=vmp.unit_dose_form_size,
    #     unit_dose_form_units=unit_dose_form_units
    # )


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def _add_version(session, day_no, version_name):
    """Add the version to the warehouse version table.

    Parameters
    ----------
    session : `sqlalchemy.Session`
        For interaction with the warehouse database.
    days_no : `int`
        The day version of the database being added.
    version_name : `str`
        The version name the database being added.

    Returns
    -------
    added_version : `warehouse_orm.DmdVersion`
        The version that has been added.
    """
    try:
        version = wquery.get_version_from_days(session, day_no)

        # All ready there, error out
        # print("ERROR")
    except NoResultFound:
        # Good to add
        version = worm.DmdVersion(
            version_day=day_no, version_name=version_name
        )
        session.add(version)
        session.commit()
    return version


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def _test_latest(session, cutoff_days):
    """Make sure that the database being added is not an old version.

    Parameters
    ----------
    session : `sqlalchemy.Session`
        For interaction with the warehouse database.
    cutoff_days : `int`
        The cutoff point to look for databases at least this old or newer.

    Raises
    ------
    ValueError
        If the database being added is not newer than what is already in the
        warehouse.
    """
    latest = wquery.get_latest_version_day(session)

    if cutoff_days <= latest:
        output_cut = 3
        newer_dbs = wquery.get_newer_dbs(session, cutoff_days)
        newer_names = [i.version_name for i in newer_dbs]
        if len(newer_names) < output_cut:
            newer_names = ",".join(newer_names)
        else:
            newer_names = "{0},...and {1} other database(s)".format(
                ",".join(newer_names), len(newer_names) - output_cut
            )
        raise ValueError(
            "you are attemting to add an old database to the warehouse."
            " Databases must be added sequentially. Please remove: "
            "{0}".format(newer_names)
        )


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def _create_tables(sm, exists=True):
    """Ensure all the database tables in the warehouse database are created.

    Parameters
    ----------
    sessionmaker : `sqlalchemy.Sessionmaker`
        For interaction with the warehouse database.
    exists : `bool`, optional, default: `True`
        Is the database expected to exist? If so, table exists errors will
        not be raised.
    """
    session = sm()
    try:
        # Now ensure that all the database tables are created
        # note that if the database exists then we do not error on tables
        # existing
        worm.Base.metadata.create_all(
            session.get_bind(), checkfirst=exists
        )
    except Exception:
        # Could not create tables for some reason
        session.close()
        raise


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def remove_database(remove_name):
    """Remove a database from the warehouse.
    """
    pass


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def _init_cmd_args():
    """Initialise the command line arguments and return the parser.

    Returns
    -------
    args : `argparse.ArgumentParser`
        The argparse parser object with arguments added.
    """
    parser = argparse.ArgumentParser(
        description=_DESC
    )

    parser.add_argument('-c', '--config',
                        type=str,
                        default=common._DEFAULT_CONFIG,
                        help="The location of the config file")
    parser.add_argument('-v', '--verbose',  action="store_true",
                        help="give more output")
    parser.add_argument(
        '-s', '--is-config',
        type=str,
        default=common._DEFAULT_SECTION,
        help="The whurl is a section in the config file and not a "
        "URL/file given on the command line"
    )
    subparsers = parser.add_subparsers(dest='option')
    parser_list = subparsers.add_parser(
        'list', help="Add a list databases, one per line, those already "
        "present will be ignored and just the latest ones added unless you "
        "are trying to add a database in a sequence, in which case it will "
        "error out. The list should contain SQLAlchemy database URLs/config"
        " sections or paths in the case of SQLite (this is the only option "
        "that has been tested)"
    )
    parser_list.add_argument(
        'list_file', help='The list of databases to add'
    )
    parser_list.add_argument('-c', '--create',  action="store_true",
                             help="Assume that the warehouse does not exist")

    parser_add = subparsers.add_parser(
        'add', help='Add a DM+D version to the warehouse'
    )
    parser_add.add_argument(
        'addurl',
        type=str,
        help="A connection to the database being added. This should be"
        " an SQLAlchemy connection URL, filename if using SQLite or a"
        " config file section if you do not want to put full connection "
        "parameters on the cmd-line"
    )
    parser_add.add_argument('-c', '--create',  action="store_true",
                            help="Assume that the warehouse does not exist")

    parser_remove = subparsers.add_parser(
        'remove', help='Remove a DM+D version from the warehouse'
    )
    parser_remove.add_argument(
        'name', type=str,
        help="The name of the database to remove, this is the zip file "
        "name without the .zip file extension"
    )
    parser.add_argument(
        'whurl',
        type=str,
        help="A connection to the warehouse database. This should be"
        " an SQLAlchemy connection URL, filename if using SQLite or a"
        " config file section if you do not want to put full connection "
        "parameters on the cmd-line"
    )
    return parser


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def _parse_cmd_args(parser):
    """Parse the command line arguments.

    Parameters
    ----------
    parser : :obj:`argparse.ArgumentParser`
        The argparse parser object with arguments added.

    Returns
    -------
    args : :obj:`argparse.Namespace`
        The argparse namespace object containing the arguments
    """
    args = parser.parse_args()
    return args
