import logging
import numpy as np
import os
import pandas
from pathlib import Path
from datetime import datetime
from typing import List, Dict, Optional, Tuple, Any
from metameq.src.util import extract_config_dict, \
    validate_required_columns_exist, get_extension, \
    load_df_with_best_fit_encoding, update_metadata_df_field, \
    HOSTTYPE_SHORTHAND_KEY, SAMPLETYPE_SHORTHAND_KEY, \
    QC_NOTE_KEY, METADATA_FIELDS_KEY, HOST_TYPE_SPECIFIC_METADATA_KEY, \
    SAMPLE_TYPE_SPECIFIC_METADATA_KEY, \
    DEFAULT_KEY, REQUIRED_KEY, \
    LEAVE_BLANK_VAL, SAMPLE_NAME_KEY, \
    LEAVE_REQUIREDS_BLANK_KEY, OVERWRITE_NON_NANS_KEY, \
    METADATA_TRANSFORMERS_KEY, PRE_TRANSFORMERS_KEY, POST_TRANSFORMERS_KEY, \
    SOURCES_KEY, FUNCTION_KEY, REQUIRED_RAW_METADATA_FIELDS, \
    HOSTTYPE_COL_OPTIONS_KEY, SAMPLETYPE_COL_OPTIONS_KEY
from metameq.src.metadata_configurator import build_full_flat_config_dict
from metameq.src.metadata_validator import validate_metadata_df, \
    format_validation_msgs_as_df, output_validation_msgs
import metameq.src.metadata_transformers as transformers


# columns added to the metadata that are not actually part of it
INTERNAL_COL_KEYS = [HOSTTYPE_SHORTHAND_KEY, SAMPLETYPE_SHORTHAND_KEY,
                     QC_NOTE_KEY]

REQ_PLACEHOLDER = "_METAMEQ_REQUIRED"

# Define a logger for this module
logger = logging.getLogger(__name__)

pandas.set_option("future.no_silent_downcasting", True)

# TODO: find a way to inform user that they *are not allowed* to have a 'sample_id' column
#  (Per Antonio 10/28/24, this is a reserved name for Qiita and may not be
#  in the metadata).


def get_reserved_cols(
        raw_metadata_df: pandas.DataFrame,
        study_specific_config_dict: Dict[str, Any],
        stds_fp: Optional[str] = None) -> List[str]:
    """Get a list of all reserved column names for all host+sample type combinations in the metadata.

    Note that 'reserved' is not the same as 'required'.  Some column names (e.g.,
        irb_institute for human host types) are not *required*, but are *reserved*, so they can
        only be used to name columns that hold standardized info, not for arbitrary metadata.

    Parameters
    ----------
    raw_metadata_df : pandas.DataFrame
        The input metadata DataFrame.
    study_specific_config_dict : Dict[str, Any]
        Study-specific flat-host-type config dictionary.
    stds_fp : Optional[str], default=None
        Path to standards dictionary file. If None, the default standards
        config pulled from the standards.yml file will be used.

    Returns
    -------
    List[str]
        Sorted list of all reserved column names.
        Empty if there are no reserved columns.

    Raises
    ------
    ValueError
        If required columns are missing from the metadata.
    """
    validate_required_columns_exist(
        raw_metadata_df, [HOSTTYPE_SHORTHAND_KEY, SAMPLETYPE_SHORTHAND_KEY],
        "metadata missing required columns")

    # Essentially, mock a minimal metadata valid metadata dataframe and then
    # use extend_metadata_df to add all the required columns to it (either empty
    # or with default values but we don't care about the actual values), then
    # return the list of column names from that extended df.

    # get unique HOSTTYPE_SHORTHAND_KEY, SAMPLETYPE_SHORTHAND_KEY combinations
    temp_df = raw_metadata_df[
        [HOSTTYPE_SHORTHAND_KEY, SAMPLETYPE_SHORTHAND_KEY]].copy()
    temp_df.drop_duplicates(inplace=True)

    # add a bogus SAMPLE_NAME_KEY column to the df that just holds sequential integers
    temp_df[SAMPLE_NAME_KEY] = range(1, len(temp_df) + 1)

    temp_df = _catch_nan_required_fields(temp_df)

    # extend the metadata_df to get all the required columns for all host+sample type combinations;
    # we don't really care about the contents of these columns, just their names.
    # (Likewise, it is not necessary to pass the actual study_specific_transformers_dict so
    # just use None)
    metadata_df, _ = extend_metadata_df(
        temp_df, study_specific_config_dict, None, None, stds_fp)

    return sorted(metadata_df.columns.to_list())


def get_default_column_name(
        col_options_key: str,
        raw_metadata_df: pandas.DataFrame,
        config_dict: Optional[Dict[str, Any]]) -> Optional[str]:
    """Find a column name in the metadata by checking a list of candidate names from the config.

    Looks up ``col_options_key`` in the config dict to get an ordered list of
    candidate column names, then returns the first one that exists in the
    metadata DataFrame. If no candidate matches, returns None.

    Parameters
    ----------
    col_options_key : str
        Key in the config dict whose value is a list of candidate column names
        to check (e.g. ``hosttype_col_options``).
    raw_metadata_df : pandas.DataFrame
        The metadata DataFrame whose columns are checked for matches.
    config_dict : Optional[Dict[str, Any]]
        Configuration dictionary that may contain a list of candidate column
        names under ``col_options_key``. If None, the default config.yml
        packaged with metameq is loaded and used instead.

    Returns
    -------
    Optional[str]
        The first matching column name found in the DataFrame, or None if no
        candidate column name is present in the DataFrame (or if the candidate
        list is empty/missing from the config).
    """
    found_name = None

    # check None instead of falsy because an empty list of options is a valid value
    # that should prevent using the default config dict
    if config_dict is None:
        config_dict = extract_config_dict(None)
    col_options = config_dict.get(col_options_key)
    if col_options:
        for col_name in col_options:
            if col_name in raw_metadata_df.columns:
                found_name = col_name
                break

    return found_name


def id_missing_cols(a_df: pandas.DataFrame) -> List[str]:
    """Identify required columns that are missing from the DataFrame.

    Parameters
    ----------
    a_df : pandas.DataFrame
        The metadata DataFrame to check for missing columns.

    Returns
    -------
    List[str]
        Sorted list of required column names that are missing from the DataFrame.
        Empty if there are no missing columns.
    """
    missing_cols = set(REQUIRED_RAW_METADATA_FIELDS) - set(a_df.columns)
    return sorted(list(missing_cols))


def find_standard_cols(
        a_df: pandas.DataFrame,
        study_specific_config_dict: Dict[str, Any],
        stds_fp: Optional[str] = None,
        suppress_missing_name_err: bool = False) -> List[str]:
    """Find all the standard columns in the metadata DataFrame.

    Parameters
    ----------
    a_df : pandas.DataFrame
        The metadata DataFrame to analyze.
    study_specific_config_dict : Dict[str, Any]
        Study-specific flat-host-type config dictionary.
    stds_fp : Optional[str], default=None
        Path to standards dictionary file. If None, the default standards
        config pulled from the standards.yml file will be used.
    suppress_missing_name_err : bool, default=False
        Whether to suppress errors about missing sample name.

    Returns
    -------
    List[str]
        List of standard column names found in the DataFrame.
        Empty if there are no standard columns.

    Raises
    ------
    ValueError
        If required columns are missing from the metadata.
    """
    err_msg = "metadata missing required columns"
    required_cols = REQUIRED_RAW_METADATA_FIELDS.copy()
    if suppress_missing_name_err:
        # remove the sample name from the required columns list
        required_cols.remove(SAMPLE_NAME_KEY)
    # endif
    validate_required_columns_exist(a_df, required_cols, err_msg)

    # get the intersection of the reserved standard columns and
    # the columns in the input dataframe
    standard_cols = get_reserved_cols(
        a_df, study_specific_config_dict, stds_fp)

    standard_cols_set = (set(standard_cols) - set(INTERNAL_COL_KEYS))

    return list(standard_cols_set & set(a_df.columns))


def find_nonstandard_cols(
        a_df: pandas.DataFrame,
        study_specific_config_dict: Dict[str, Any],
        stds_fp: Optional[str] = None) -> List[str]:
    """Find any non-standard columns in the metadata DataFrame.

    Parameters
    ----------
    a_df : pandas.DataFrame
        The metadata DataFrame to analyze.
    study_specific_config_dict : Dict[str, Any]
        Study-specific flat-host-type config dictionary.
    stds_fp : Optional[str], default=None
        Path to standards dictionary file. If None, the default standards
        config pulled from the standards.yml file will be used.

    Returns
    -------
    List[str]
        List of non-standard column names found in the DataFrame.
        Empty if there are no non-standard columns.

    Raises
    ------
    ValueError
        If required columns are missing from the metadata.
    """
    validate_required_columns_exist(a_df, REQUIRED_RAW_METADATA_FIELDS,
                                    "metadata missing required columns")

    # get the columns in
    standard_cols = get_reserved_cols(
        a_df, study_specific_config_dict, stds_fp)

    return list(set(a_df.columns) - set(standard_cols))


def extend_metadata_df(
        raw_metadata_df: pandas.DataFrame,
        study_specific_config_dict: Optional[Dict[str, Any]],
        study_specific_transformers_dict: Optional[Dict[str, Any]] = None,
        software_config_dict: Optional[Dict[str, Any]] = None,
        stds_fp: Optional[str] = None,
        hosttype_col_name: Optional[str] = None,
        sampletype_col_name: Optional[str] = None
) -> Tuple[pandas.DataFrame, pandas.DataFrame]:
    """Extend a metadata DataFrame based on metadata standards and study-specific configurations.

    Parameters
    ----------
    raw_metadata_df : pandas.DataFrame
        The raw metadata DataFrame to extend.
    study_specific_config_dict : Optional[Dict[str, Any]]
        Study-specific flat-host-type config dictionary.
    study_specific_transformers_dict : Optional[Dict[str, Any]], default=None
        Dictionary of custom transformers for this study (only).
    software_config_dict : Optional[Dict[str, Any]], default=None
        Software configuration dictionary. If None, the default software
        config pulled from the config.yml file will be used.
    stds_fp : Optional[str], default=None
        Path to standards dictionary file. If None, the default standards
        config pulled from the standards.yml file will be used.
    hosttype_col_name : Optional[str], default=None
        Name of the column in raw_metadata_df that contains host type
        values. If provided, its values are copied into the internal
        ``hosttype_shorthand`` column before processing. If None, the
        function checks the config's ``hosttype_col_options`` list for a
        matching column in the DataFrame.
    sampletype_col_name : Optional[str], default=None
        Name of the column in raw_metadata_df that contains sample type
        values. If provided, its values are copied into the internal
        ``sampletype_shorthand`` column before processing. If None, the
        function checks the config's ``sampletype_col_options`` list for
        a matching column in the DataFrame.

    Returns
    -------
    Tuple[pandas.DataFrame, pandas.DataFrame]
        A tuple containing:
            - The extended metadata DataFrame
            - A DataFrame containing validation messages

    Raises
    ------
    ValueError
        If required columns are missing from the metadata, if a specified
        column name is not found in the DataFrame, or if both the internal
        shorthand column and the specified alternate column exist.
    """
    full_flat_config_dict = build_full_flat_config_dict(
        study_specific_config_dict, software_config_dict, stds_fp)

    metadata_df, validation_msgs_df, _ = _extend_metadata_from_full_flat_config(
        raw_metadata_df, full_flat_config_dict,
        study_specific_transformers_dict,
        hosttype_col_name, sampletype_col_name)

    return metadata_df, validation_msgs_df


def extend_metadata_df_from_yamls(
        raw_metadata_df: pandas.DataFrame,
        study_specific_config_fp: Optional[str],
        stds_fp: Optional[str] = None) -> Tuple[pandas.DataFrame, pandas.DataFrame]:
    """Extend metadata using configuration from a study-specific YAML config file.

    Parameters
    ----------
    raw_metadata_df : pandas.DataFrame
        The raw metadata DataFrame to extend.
    study_specific_config_fp : Optional[str]
        Path to the study-specific configuration YAML file.
    stds_fp : Optional[str], default=None
        Path to standards dictionary file. If None, the default standards
        config pulled from the standards.yml file will be used.

    Returns
    -------
    Tuple[pandas.DataFrame, pandas.DataFrame]
        A tuple containing:
            - The extended metadata DataFrame
            - A DataFrame containing validation messages
    """
    # get the study-specific flat-host-type config dictionary from the input yaml file
    study_specific_config_dict = \
        _get_study_specific_config(study_specific_config_fp)

    # extend the metadata DataFrame using the study-specific flat-host-type config dictionary
    metadata_df, validation_msgs_df = \
        extend_metadata_df(raw_metadata_df, study_specific_config_dict,
                           None, None, stds_fp)

    return metadata_df, validation_msgs_df


def get_qc_failures(a_df: pandas.DataFrame) -> pandas.DataFrame:
    """Get rows from the extended metadata DataFrame that have QC failures.

    Parameters
    ----------
    a_df : pandas.DataFrame
        The extended metadata DataFrame to check for QC failures.

    Returns
    -------
    pandas.DataFrame
        A new DataFrame containing only the rows that failed QC checks.
    """
    fails_qc_mask = a_df[QC_NOTE_KEY] != ""
    qc_fails_df = \
        a_df.loc[fails_qc_mask, :].copy()
    return qc_fails_df


# Map QC note strings to the internal column key they relate to
_QC_NOTE_TO_INTERNAL_KEY = {
    "invalid host_type": HOSTTYPE_SHORTHAND_KEY,
    "invalid sample_type": SAMPLETYPE_SHORTHAND_KEY
}


def _qc_failures_to_validation_msgs(
        metadata_df: pandas.DataFrame,
        col_name_mapping: Dict[str, str]) -> list:
    """Convert QC failure rows into a list of validation message dicts.

    Each QC failure row is converted into a validation message record
    whose ``field_name`` uses the user-facing column name (from
    *col_name_mapping*) rather than the internal shorthand name.

    Parameters
    ----------
    metadata_df : pandas.DataFrame
        The extended metadata DataFrame (must contain ``QC_NOTE_KEY``).
    col_name_mapping : Dict[str, str]
        Mapping from internal column keys to user-facing column names
        (as returned by ``_extend_metadata_from_full_flat_config``).

    Returns
    -------
    list
        A list of dicts, each with keys ``sample_name``,
        ``field_name``, ``field_value``, and ``error_message``
        (a list with one string).
    """
    qc_failures_df = get_qc_failures(metadata_df)

    records = []
    for _, row in qc_failures_df.iterrows():
        qc_note = row[QC_NOTE_KEY]
        internal_key = _QC_NOTE_TO_INTERNAL_KEY.get(qc_note)
        if internal_key is not None:
            field_name = col_name_mapping.get(internal_key, internal_key)
            field_value = row[internal_key]
        else:
            field_name = qc_note
            field_value = None
        records.append({
            SAMPLE_NAME_KEY: row[SAMPLE_NAME_KEY],
            "field_name": field_name,
            "field_value": field_value,
            "error_message": [qc_note]
        })

    return records


def write_extended_metadata_from_df(
        raw_metadata_df: pandas.DataFrame,
        study_specific_config_dict: Dict[str, Any],
        out_dir: str,
        out_name_base: str,
        study_specific_transformers_dict: Optional[Dict[str, Any]] = None,
        sep: str = "\t",
        remove_internals: bool = True,
        suppress_empty_fails: bool = False,
        internal_col_names: Optional[List[str]] = None,
        stds_fp: Optional[str] = None) -> pandas.DataFrame:
    """Write extended metadata to files starting from a metadata DataFrame and config dictionary.

    Parameters
    ----------
    raw_metadata_df : pandas.DataFrame
        The raw metadata DataFrame to extend.
    study_specific_config_dict : Dict[str, Any]
        Study-specific configuration dictionary.
    out_dir : str
        Directory where output files will be written.
    out_name_base : str
        Base name for output files.
    study_specific_transformers_dict : Optional[Dict[str, Any]], default=None
        Dictionary of custom transformers.
    sep : str, default="\t"
        Separator to use in output files.
    remove_internals : bool, default=True
        Whether to remove internal columns.
    suppress_empty_fails : bool, default=False
        Whether to suppress empty failure files.
    internal_col_names : Optional[List[str]], default=None
        List of internal column names.
    stds_fp : Optional[str], default=None
        Path to standards dictionary file. If None, the default standards
        config pulled from the standards.yml file will be used.

    Returns
    -------
    pandas.DataFrame
        The extended metadata DataFrame.
    """
    # extend the metadata DataFrame using the study-specific flat-host-type config dictionary
    metadata_df, validation_msgs_df = extend_metadata_df(
        raw_metadata_df, study_specific_config_dict,
        study_specific_transformers_dict, None, stds_fp)

    # write the metadata and validation results to files
    write_metadata_results(
        metadata_df, validation_msgs_df, out_dir, out_name_base,
        sep=sep, remove_internals=remove_internals,
        suppress_empty_fails=suppress_empty_fails,
        internal_col_names=internal_col_names)

    # for good measure, return the extended metadata DataFrame
    return metadata_df


def write_extended_metadata(
        raw_metadata_fp: str,
        study_specific_config_fp: str,
        out_dir: str,
        out_name_base: str,
        sep: str = "\t",
        remove_internals: bool = True,
        suppress_empty_fails: bool = False,
        stds_fp: Optional[str] = None) -> pandas.DataFrame:
    """Write extended metadata to files starting from input file paths to metadata and config.

    Parameters
    ----------
    raw_metadata_fp : str
        Path to the raw metadata file (.csv, .tsv, .txt, or .xlsx).
    study_specific_config_fp : str
        Path to the study-specific configuration YAML file.
    out_dir : str
        Directory where output files will be written.
    out_name_base : str
        Base name for output files.
    sep : str, default="\t"
        Separator to use in output files.
    remove_internals : bool, default=True
        Whether to remove internal columns.
    suppress_empty_fails : bool, default=False
        Whether to suppress empty failure files.
    stds_fp : Optional[str], default=None
        Path to standards dictionary file. If None, the default standards
        config pulled from the standards.yml file will be used.

    Returns
    -------
    pandas.DataFrame
        The extended metadata DataFrame.

    Raises
    ------
    ValueError
        If the input file extension is not recognized.
    """
    raw_metadata_df = _load_metadata_df(raw_metadata_fp)

    # get the study-specific flat-host-type config dictionary from the input yaml file
    study_specific_config_dict = \
        _get_study_specific_config(study_specific_config_fp)

    # write the extended metadata to files
    extended_df = write_extended_metadata_from_df(
        raw_metadata_df, study_specific_config_dict,
        out_dir, out_name_base, sep=sep,
        remove_internals=remove_internals,
        suppress_empty_fails=suppress_empty_fails,
        stds_fp=stds_fp)

    # for good measure, return the extended metadata DataFrame
    return extended_df


def write_validator_metadata(
        raw_metadata_fp: str,
        full_flat_config_dict_fp: str,
        out_dir: str,
        out_name_base: str,
        sep: str = "\t",
        remove_internals: bool = True,
        suppress_empty_fails: bool = False,
        hosttype_col_name: Optional[str] = None,
        sampletype_col_name: Optional[str] = None) -> None:
    """Write extended metadata to files starting from input file paths to metadata and full flat config.

    """
    # load the metadata
    raw_metadata_df = _load_metadata_df(raw_metadata_fp)

    # load the full flat config dictionary from the input yaml file
    full_flat_config_dict = extract_config_dict(full_flat_config_dict_fp)

    # extend the metadata DataFrame using the study-specific flat-host-type config dictionary
    metadata_df, validation_msgs_df, col_name_mapping = \
        _extend_metadata_from_full_flat_config(
            raw_metadata_df, full_flat_config_dict,
            study_specific_transformers_dict=None,
            hosttype_col_name=hosttype_col_name,
            sampletype_col_name=sampletype_col_name)

    # Convert QC failures to validation message records and combine
    # with any cerberus validation messages
    qc_validation_msgs = _qc_failures_to_validation_msgs(
        metadata_df, col_name_mapping)
    qc_validation_msgs_df = format_validation_msgs_as_df(
        qc_validation_msgs)
    combined_validation_msgs_df = pandas.concat(
        [validation_msgs_df, qc_validation_msgs_df], ignore_index=True)
    combined_validation_msgs_df.sort_values(
        by=[SAMPLE_NAME_KEY, "field_name", "error_message"],
        inplace=True)
    combined_validation_msgs_df.reset_index(drop=True, inplace=True)

    # Write the metadata output file, optionally stripping internal cols
    # but keeping ALL rows (no QC-failure row removal)
    output_df = metadata_df
    if remove_internals:
        output_df = _remove_internal_cols(metadata_df)
    _write_df_to_file(output_df, out_dir, out_name_base, sep=sep)

    # Write the combined validation messages file
    output_validation_msgs(
        combined_validation_msgs_df, out_dir, out_name_base,
        sep=",", suppress_empty_fails=suppress_empty_fails)


def write_metadata_results(
        metadata_df: pandas.DataFrame,
        validation_msgs_df: pandas.DataFrame,
        out_dir: str,
        out_name_base: str,
        sep: str = "\t",
        remove_internals: bool = True,
        suppress_empty_fails: bool = False,
        internal_col_names: Optional[List[str]] = None) -> None:
    """Write metadata and validation results to files.

    Parameters
    ----------
    metadata_df : pandas.DataFrame
        The metadata DataFrame to write.
    validation_msgs_df : pandas.DataFrame
        DataFrame containing validation messages.
    out_dir : str
        Directory where output files will be written.
    out_name_base : str
        Base name for output files.
    sep : str, default="\t"
        Separator to use in output files.
    remove_internals : bool, default=True
        Whether to remove internal columns.
    suppress_empty_fails : bool, default=False
        Whether to suppress empty failure files.
    internal_col_names : Optional[List[str]], default=None
        List of internal column names.
    """
    if internal_col_names is None:
        internal_col_names = INTERNAL_COL_KEYS

    _output_metadata_df_to_files(
        metadata_df, out_dir, out_name_base, internal_col_names,
        remove_internals_and_fails=remove_internals, sep=sep,
        suppress_empty_fails=suppress_empty_fails)

    output_validation_msgs(validation_msgs_df, out_dir, out_name_base, sep=",",
                           suppress_empty_fails=suppress_empty_fails)


def _get_study_specific_config(study_specific_config_fp: Optional[str]) -> Optional[Dict[str, Any]]:
    """Load study-specific flat-host-type configuration from a YAML file.

    Parameters
    ----------
    study_specific_config_fp : Optional[str]
        Path to the study-specific configuration YAML file.
        This file should contain study-specific values for top-level settings (e.g., default
        value) and, if necessary, a HOST_TYPE_SPECIFIC_METADATA_KEY holding a *flat*
        dictionary of host types, defining only their study-specific host and sample type
        metadata fields.

    Returns
    -------
    Optional[Dict[str, Any]]
        The loaded flat-host-type configuration dictionary, or None if no file path provided.
    """
    if study_specific_config_fp:
        study_specific_config_dict = \
            extract_config_dict(study_specific_config_fp)
    else:
        study_specific_config_dict = None

    return study_specific_config_dict


def _populate_metadata_df(
        raw_metadata_df: pandas.DataFrame,
        full_flat_config_dict: Dict[str, Any],
        transformer_funcs_dict: Optional[Dict[str, Any]]) -> Tuple[pandas.DataFrame, pandas.DataFrame]:
    """Populate columns and fields in a metadata DataFrame.

    Parameters
    ----------
    raw_metadata_df : pandas.DataFrame
        The raw metadata DataFrame to populate, which must contain at least
        the columns in REQUIRED_RAW_METADATA_FIELDS.
    full_flat_config_dict : Dict[str, Any]
        Fully combined flat-host-type config dictionary.
    transformer_funcs_dict : Optional[Dict[str, Any]]
        Dictionary of transformer functions, keyed by field name,
        with each value being a dict with keys SOURCES_KEY and FUNCTION_KEY,
        which map to lists of source field names for the transformer to use
        and an existing transformer function name, respectively.

    Returns
    -------
    Tuple[pandas.DataFrame, pandas.DataFrame]
        A tuple containing:
            - The populated metadata DataFrame
            - A DataFrame containing validation messages
    """
    metadata_df = raw_metadata_df.copy()
    # Don't try to populate the QC_NOTE_KEY field, since it is an internal field
    update_metadata_df_field(metadata_df, QC_NOTE_KEY, LEAVE_BLANK_VAL)

    # Error for NaNs in sample name, warn for NaNs in host- and sample-type- shorthand fields.
    metadata_df = _catch_nan_required_fields(metadata_df)

    # Apply pre-transformers to the metadata. Pre-transformers run BEFORE host- and sample-type
    # specific generation (which also includes validation), so they can transform raw input fields
    # into values that the config validation expects (for example, converting a study's custom sex
    # format like "M"/"F" into standardized values like "male"/"female" before validation occurs.
    metadata_df = _transform_metadata(
        metadata_df, full_flat_config_dict,
        PRE_TRANSFORMERS_KEY, transformer_funcs_dict)

    # Add specific metadata based on each host type present in the metadata.
    # This step also validates the metadata against the config requirements.
    metadata_df, validation_msgs = _generate_metadata_for_host_types(
        metadata_df, full_flat_config_dict)

    # Apply post-transformers to the metadata. Post-transformers run AFTER host- and sample-type
    # specific generation, so they can use fields that only exist or were only filled in
    # after that step, such as passing through a value filled in by the defaults to another field.
    metadata_df = _transform_metadata(
        metadata_df, full_flat_config_dict,
        POST_TRANSFORMERS_KEY, transformer_funcs_dict)

    # Reorder the metadata columns for better readability.
    metadata_df = _reorder_df(metadata_df, INTERNAL_COL_KEYS)

    # Turn the validation messages into a DataFrame of validation messages for easier use downstream.
    validation_msgs_df = format_validation_msgs_as_df(validation_msgs)

    return metadata_df, validation_msgs_df


def _find_internal_col_source_name(
        raw_metadata_df: pandas.DataFrame, full_flat_config_dict: Dict[str, Any],
        param_key: Optional[str], internal_key: str, options_key: str) -> Optional[str]:
    specified_name = param_key
    if not specified_name:
        specified_name = get_default_column_name(
            options_key, raw_metadata_df, full_flat_config_dict)

    if specified_name:
        if specified_name != internal_key:
            if internal_key in raw_metadata_df.columns:
                logger.warning(
                    f"Metadata contains both '{internal_key}' and "
                    f"'{specified_name}' columns; using '{internal_key}'.")
                return None
            if specified_name not in raw_metadata_df.columns:
                raise ValueError(
                    f"Specified {internal_key} column '{specified_name}' not found in metadata.")
            return specified_name

    # note that if the specified column name is the same as the internal key, we don't need to do anything;
    # if the internal key doesn't exist in the metadata (already), later code will complain, so this
    # function doesn't need to worry about that case.
    return None


def _catch_nan_required_fields(metadata_df: pandas.DataFrame) -> pandas.DataFrame:
    """Error for NaNs in sample name, warn for NaNs in host- and sample-type- shorthand fields.

    Parameters
    ----------
    metadata_df : pandas.DataFrame
        The metadata DataFrame to process.

    Returns
    -------
    pandas.DataFrame
        The processed DataFrame. NaNs in host- and sample-type-shorthand fields are set to "empty".

    Raises
    ------
    ValueError
        If any sample names are NaN.
    """
    # if there are any sample_name fields that are NaN, raise an error
    nan_sample_name_mask = metadata_df[SAMPLE_NAME_KEY].isna()
    if nan_sample_name_mask.any():
        raise ValueError("Metadata contains NaN sample names")

    # if there are any hosttype_shorthand or sampletype_shorthand fields
    # that are NaN, set them to "empty" and raise a warning
    for curr_key in [HOSTTYPE_SHORTHAND_KEY, SAMPLETYPE_SHORTHAND_KEY]:
        nan_mask = metadata_df[curr_key].isna()
        if nan_mask.any():
            metadata_df.loc[nan_mask, curr_key] = "empty"
            logging.warning(f"Metadata contains NaN {curr_key}s; "
                            f"these have been set to 'empty'")

    return metadata_df


def _transform_metadata(
        metadata_df: pandas.DataFrame,
        full_flat_config_dict: Dict[str, Any],
        stage_key: str,
        transformer_funcs_dict: Optional[Dict[str, Any]]) -> pandas.DataFrame:
    """Apply transformations defined in full_flat_config_dict to metadata fields.

    Parameters
    ----------
    metadata_df : pandas.DataFrame
        The metadata DataFrame to transform, which must contain at least
        the columns in REQUIRED_RAW_METADATA_FIELDS.
    full_flat_config_dict : Dict[str, Any]
        Fully combined flat-host-type config dictionary. May contain
        OVERWRITE_NON_NANS_KEY as a global setting, which can be overridden
        by individual transformers.
    stage_key : str
        Key indicating the transformation stage (pre or post).
    transformer_funcs_dict : Optional[Dict[str, Any]]
        Dictionary of transformer functions, keyed by function name.
        If None, only built-in transformers will be available.

    Returns
    -------
    pandas.DataFrame
        The transformed metadata DataFrame.

    Raises
    ------
    ValueError
        If a specified transformer function cannot be found.

    Notes
    -----
    Each transformer may optionally specify its own OVERWRITE_NON_NANS_KEY
    setting, which takes precedence over the global setting.

    If a transformer references source fields that are not present in the
    DataFrame, that transformer is skipped and a warning is logged. This
    allows optional fields to be used as transformer sources without causing
    errors when they are absent.
    """
    if transformer_funcs_dict is None:
        transformer_funcs_dict = {}
    # If the necessary keys aren't already in the config, set them to do-nothing defaults
    overwrite_non_nans = full_flat_config_dict.get(OVERWRITE_NON_NANS_KEY, False)
    metadata_transformers = full_flat_config_dict.get(METADATA_TRANSFORMERS_KEY, None)
    if metadata_transformers:
        stage_transformers = metadata_transformers.get(stage_key, None)
        # If there are transformers for the stage we're at, apply them
        if stage_transformers:
            for curr_target_field, curr_transformer_dict in \
                    stage_transformers.items():
                curr_source_fields = curr_transformer_dict[SOURCES_KEY]
                curr_func_name = curr_transformer_dict[FUNCTION_KEY]
                curr_overwrite_non_nans = curr_transformer_dict.get(
                    OVERWRITE_NON_NANS_KEY, overwrite_non_nans)

                # If any of the source fields for this transformer are missing from
                # the metadata, skip this transformer and log a warning.  This can
                # happen if, for example, there is a transformer set that uses an
                # optional field as a source, and that optional field is not present.
                curr_missing_sources = set(curr_source_fields) - set(metadata_df.columns)
                if curr_missing_sources:
                    logging.warning(
                        f"Transformer '{curr_func_name}' for target field"
                        f" '{curr_target_field}' skipped due to missing source fields: "
                        f"{', '.join(sorted(curr_missing_sources))}")
                    continue

                try:
                    curr_func = transformer_funcs_dict[curr_func_name]
                except KeyError:
                    try:
                        # if the transformer function isn't in the dictionary
                        # that was passed in, probably it is a built-in one,
                        # so look for it in the metameq transformers module
                        # looking into the metameq transformers module
                        curr_func = getattr(transformers, curr_func_name)
                    except AttributeError:
                        raise ValueError(
                            f"Unable to find transformer '{curr_func_name}'")
                    # end try to find in metameq transformers
                # end try to find in input (study-specific) transformers

                # apply the function named curr_func_name to the column(s) of the
                # metadata_df named curr_source_fields to fill curr_target_field
                update_metadata_df_field(metadata_df, curr_target_field,
                                         curr_func, curr_source_fields,
                                         overwrite_non_nans=curr_overwrite_non_nans)
            # next stage transformer
        # end if there are stage transformers for this stage
    # end if there are any metadata transformers

    return metadata_df


def _generate_metadata_for_host_types(
        metadata_df: pandas.DataFrame,
        full_flat_config_dict: Dict[str, Any]) -> Tuple[pandas.DataFrame, List[str]]:
    """Generate metadata for samples of all host types in the DataFrame.

    Parameters
    ----------
    metadata_df : pandas.DataFrame
        The metadata DataFrame to process, which must contain at least
        the columns in REQUIRED_RAW_METADATA_FIELDS.
    full_flat_config_dict : Dict[str, Any]
        Fully combined flat-host-type config dictionary.

    Returns
    -------
    Tuple[pandas.DataFrame, List[str]]
        A tuple containing:
            - The processed DataFrame with specific metadata added to each sample of each host type
            - A list of validation messages
    """

    validation_msgs = []
    host_type_dfs = []
    # For all the host types present in the metadata, generate the specific metadata
    host_type_shorthands = pandas.unique(metadata_df[HOSTTYPE_SHORTHAND_KEY])
    for curr_host_type_shorthand in host_type_shorthands:
        concatted_dfs, curr_validation_msgs = _generate_metadata_for_a_host_type(
                metadata_df, curr_host_type_shorthand, full_flat_config_dict)

        host_type_dfs.append(concatted_dfs)
        validation_msgs.extend(curr_validation_msgs)
    # next host type

    # Concatenate the processed host-type-specific metadata DataFrames into a single output DataFrame
    output_df = pandas.concat(host_type_dfs, ignore_index=True)

    # concatting dfs from different hosts can create large numbers of NAs--
    # for example, if concatting a host-associated df with a control df, where
    # the control df doesn't have values for any of the host-related columns.
    # Fill those NAs with whatever the general default is.
    # NB: passing in the same dict twice here is not a mistake, just a
    # convenience since we don't have a more specific dict at this point.
    output_df = _fill_na_if_default(
        output_df, full_flat_config_dict)

    # TODO: this is setting a value in the output; should it be centralized
    #  so it is easy to find?
    # Replace the LEAVE_BLANK_VAL with an empty string in the output DataFrame
    output_df.replace(LEAVE_BLANK_VAL, "", inplace=True)
    return output_df, validation_msgs


def _generate_metadata_for_a_host_type(
        metadata_df: pandas.DataFrame,
        a_host_type: str,
        full_flat_config_dict: Dict[str, Any]) -> Tuple[pandas.DataFrame, List[str]]:
    """Generate metadata df for samples with a specific host type.

    Parameters
    ----------
    metadata_df : pandas.DataFrame
        The metadata DataFrame to process, which must contain at least
        the columns in REQUIRED_RAW_METADATA_FIELDS.
    a_host_type : str
        The specific host type for which to process samples.
    full_flat_config_dict : Dict[str, Any]
        Fully combined flat-host-type config dictionary.

    Returns
    -------
    Tuple[pandas.DataFrame, List[str]]
        A tuple containing:
            - The processed DataFrame with specific metadata added to each sample of the input host type
            - A list of validation messages
    """
    # get the subset of the metadata DataFrame that contains samples of the input host type
    host_type_mask = \
        metadata_df[HOSTTYPE_SHORTHAND_KEY] == a_host_type
    host_type_df = metadata_df.loc[host_type_mask, :].copy()

    validation_msgs = []
    known_host_shorthands = full_flat_config_dict[HOST_TYPE_SPECIFIC_METADATA_KEY].keys()
    if a_host_type not in known_host_shorthands:
        # if the input host type is not in the config, add a QC note to the metadata
        # for these samples but do not error out; move on to the next host type
        update_metadata_df_field(
            host_type_df, QC_NOTE_KEY, "invalid host_type")
        concatted_df = host_type_df
    else:
        # gather host-type-specific settings and overwrite the global settings with them, if any
        a_host_type_config_dict = \
            full_flat_config_dict[HOST_TYPE_SPECIFIC_METADATA_KEY][a_host_type]

        dfs_to_concat = []
        # loop through each sample type in the metadata for this host type
        found_host_sample_types = \
            pandas.unique(host_type_df[SAMPLETYPE_SHORTHAND_KEY])
        for curr_sample_type in found_host_sample_types:
            # generate the specific metadata for this sample type *in this host type*
            curr_sample_type_df, curr_validation_msgs = \
                _generate_metadata_for_a_sample_type_in_a_host_type(
                    host_type_df, curr_sample_type, a_host_type_config_dict)

            dfs_to_concat.append(curr_sample_type_df)
            validation_msgs.extend(curr_validation_msgs)
        # next sample type in metadata for this host type

        # Concatenate the processed sample-type-specific metadata DataFrames
        # for the host type into a single output DataFrame
        concatted_df = pandas.concat(dfs_to_concat, ignore_index=True)
    # endif host_type is valid

    return concatted_df, validation_msgs


def _generate_metadata_for_a_sample_type_in_a_host_type(
        host_type_metadata_df: pandas.DataFrame,
        a_sample_type: str,
        a_host_type_config_dict: Dict[str, Any]) -> Tuple[pandas.DataFrame, List[str]]:
    """Generate metadata df for samples with a specific sample type within a specific host type.

    Parameters
    ----------
    host_type_metadata_df : pandas.DataFrame
        DataFrame containing metadata samples for a specific host type.
    a_sample_type : str
        The sample type to process.
    a_host_type_config_dict : Dict[str, Any]
        Dictionary containing config for this host type.

    Returns
    -------
    Tuple[pandas.DataFrame, List[str]]
        A tuple containing:
            - The updated metadata DataFrame with sample-type-specific elements added
            - A list of validation messages
    """
    # get the config section for *all* sample types within this host type
    host_sample_types_config_dict = \
        a_host_type_config_dict[SAMPLE_TYPE_SPECIFIC_METADATA_KEY]

    # get df of records for this sample type in this host type
    sample_type_mask = \
        host_type_metadata_df[SAMPLETYPE_SHORTHAND_KEY] == a_sample_type
    sample_type_df = host_type_metadata_df.loc[sample_type_mask, :].copy()

    validation_msgs = []
    known_sample_types = host_sample_types_config_dict.keys()
    if a_sample_type not in known_sample_types:
        # if the input sample type is not in the config, add a QC note to the metadata
        # for these samples but do not error out; move on to the next sample type
        update_metadata_df_field(
            sample_type_df, QC_NOTE_KEY, "invalid sample_type")
    else:
        # Get the already-resolved metadata fields dict for this sample type.
        # The config is pre-resolved: aliases/base types are merged and
        # host metadata is combined.
        sample_type_config = host_sample_types_config_dict[a_sample_type]
        full_sample_type_metadata_fields_dict = \
            sample_type_config.get(METADATA_FIELDS_KEY, {})

        # update the metadata df with the sample type specific metadata fields
        sample_type_df = _update_metadata_from_dict(
            sample_type_df, full_sample_type_metadata_fields_dict,
            dict_is_metadata_fields=True,
            overwrite_non_nans=a_host_type_config_dict[OVERWRITE_NON_NANS_KEY])

        # for fields that are required but not yet filled, replace the placeholder with
        # either an indicator that it should be blank or else
        # fill with NA (replaced with default just below), based on config setting
        leave_reqs_blank = a_host_type_config_dict[LEAVE_REQUIREDS_BLANK_KEY]
        reqs_val = LEAVE_BLANK_VAL if leave_reqs_blank else np.nan
        sample_type_df.replace(
            to_replace=REQ_PLACEHOLDER, value=reqs_val, inplace=True)

        # fill NAs with appropriate default value if any is set
        sample_type_df = _fill_na_if_default(
            sample_type_df, a_host_type_config_dict)

        # validate the metadata df based on the specific requirements
        # for this host+sample type
        validation_msgs = validate_metadata_df(
            sample_type_df, full_sample_type_metadata_fields_dict)

    return sample_type_df, validation_msgs


def _update_metadata_from_dict(
        metadata_df: pandas.DataFrame,
        config_section_dict: Dict[str, Any],
        dict_is_metadata_fields: bool = False,
        overwrite_non_nans: bool = False) -> pandas.DataFrame:
    """Create an updated copy of the metadata DataFrame based on an input dictionary.

    Parameters
    ----------
    metadata_df : pandas.DataFrame
        The metadata DataFrame to update.
    config_section_dict : Dict[str, Any]
        The relevant section of a config dictionary to use.
    dict_is_metadata_fields : bool, default=False
        Whether the config dict contains a METADATA_FIELDS_KEY
        (in which case False) or is itself the contents of
        a METADATA_FIELDS_KEY (in which case True).
    overwrite_non_nans : bool, default=False
        Whether to overwrite non-NaN values with default values.

    Returns
    -------
    pandas.DataFrame
        An updated copy of the metadata DataFrame.
    """
    if not dict_is_metadata_fields:
        metadata_fields_dict = config_section_dict.get(METADATA_FIELDS_KEY)
    else:
        metadata_fields_dict = config_section_dict

    output_df = _update_metadata_from_metadata_fields_dict(
        metadata_df, metadata_fields_dict,
        overwrite_non_nans=overwrite_non_nans)
    return output_df


def _update_metadata_from_metadata_fields_dict(
        metadata_df: pandas.DataFrame,
        metadata_fields_dict: Dict[str, Any],
        overwrite_non_nans: bool) -> pandas.DataFrame:
    """Create an updated copy of the metadata DataFrame based on a metadata fields dictionary.

    Parameters
    ----------
    metadata_df : pandas.DataFrame
        The metadata DataFrame to update.
    metadata_fields_dict : Dict[str, Any]
        Dictionary containing metadata field definitions and required values.
    overwrite_non_nans : bool
        Whether to overwrite non-NaN values with default values.

    Returns
    -------
    pandas.DataFrame
        An updated copy of the metadata DataFrame.
    """
    output_df = metadata_df.copy()

    # loop through each metadata field in the metadata fields dict
    for curr_field_name, curr_field_vals_dict in metadata_fields_dict.items():
        # if the field has a default value (regardless of whether it is
        # required), update the metadata df with it (this includes adding the
        # field if it does not already exist). For existing fields, what exactly
        # will beupdated depends on the value of overwrite_non_nans:
        # if overwrite_non_nans is True, then all values will be updated;
        # if overwrite_non_nans is False, then only NA values will be updated
        # if the field already exists in the metadata; otherwise, the field
        # will be added to the metadata with the default value throughout.
        if DEFAULT_KEY in curr_field_vals_dict:
            curr_default_val = curr_field_vals_dict[DEFAULT_KEY]
            update_metadata_df_field(
                output_df, curr_field_name, curr_default_val,
                overwrite_non_nans=overwrite_non_nans)
        # if the field is required BUT has no default value, then if the field does not
        # already exist in the metadata, add the field to the metadata with a placeholder value.
        elif REQUIRED_KEY in curr_field_vals_dict:
            curr_required_val = curr_field_vals_dict[REQUIRED_KEY]
            if curr_required_val and curr_field_name not in output_df:
                update_metadata_df_field(
                    output_df, curr_field_name, REQ_PLACEHOLDER,
                    overwrite_non_nans=overwrite_non_nans)
        # note that if the field is (a) required, (b) does not have a
        # default value, and (c) IS already in the metadata, it will
        # be left alone, with no changes made to it!
    return output_df


# fill NAs with default value if any is set
def _fill_na_if_default(
        metadata_df: pandas.DataFrame,
        settings_dict: Dict[str, Any]) -> pandas.DataFrame:
    """Fill NaN values in metadata df with default values if available.

    Parameters
    ----------
    metadata_df : pandas.DataFrame
        The metadata DataFrame to process.
    settings_dict : Dict[str, Any]
        Dictionary containing settings.

    Returns
    -------
    pandas.DataFrame
        The updated DataFrame with NaN values filled. Unchanged if no default values are set.
    """
    default_val = settings_dict.get(DEFAULT_KEY)
    if default_val:
        # TODO: this is setting a value in the output; should it be
        #  centralized so it is easy to find?
        metadata_df = \
            metadata_df.fillna(default_val)

    return metadata_df


def _write_df_to_file(
        a_df: pandas.DataFrame,
        out_dir: str,
        out_base: str,
        sep: str = "\t",
        suffix: str = "") -> None:
    """Write a DataFrame to a timestamped file.

    Parameters
    ----------
    a_df : pandas.DataFrame
        The DataFrame to write.
    out_dir : str
        Directory where the output file will be written.
    out_base : str
        Base name for the output file.
    sep : str, default="\t"
        Separator to use in the output file.
    suffix : str, default=""
        Suffix to append to the base name before the extension
        (e.g., "_fails").
    """
    timestamp_str = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
    extension = get_extension(sep)
    out_fp = os.path.join(
        out_dir, f"{timestamp_str}_{out_base}{suffix}.{extension}")
    a_df.to_csv(out_fp, sep=sep, index=False)


def _remove_internal_cols(
        a_df: pandas.DataFrame,
        internal_col_names: Optional[List[str]] = None) -> pandas.DataFrame:
    """Remove internal columns from a DataFrame.

    Parameters
    ----------
    a_df : pandas.DataFrame
        The DataFrame to process.
    internal_col_names : Optional[List[str]], default=None
        List of internal column names to remove. If None, defaults to
        INTERNAL_COL_KEYS.

    Returns
    -------
    pandas.DataFrame
        A copy of the DataFrame with internal columns removed.
    """
    if internal_col_names is None:
        internal_col_names = INTERNAL_COL_KEYS
    return a_df.drop(columns=internal_col_names)


def _output_metadata_df_to_files(
        a_df: pandas.DataFrame,
        out_dir: str,
        out_base: str,
        internal_col_names: List[str],
        sep: str = "\t",
        remove_internals_and_fails: bool = False,
        suppress_empty_fails: bool = False) -> None:
    """Output DataFrame to files, optionally removing internal columns and failures.

    Parameters
    ----------
    a_df : pandas.DataFrame
        The metadata DataFrame to output.
    out_dir : str
        Directory where output files will be written.
    out_base : str
        Base name for output files.
    internal_col_names : List[str]
        List of internal column names that will be moved
        to the end of the DataFrame.
    sep : str, default="tab"
        Separator to use in output files.
    remove_internals_and_fails : bool, default=False
        Whether to remove internal columns and failures.
    suppress_empty_fails : bool, default=False
        Whether to suppress empty failure files.
    """
    # if we've been told to remove the qc fails and the internal columns
    if remove_internals_and_fails:
        # output a file of any qc failures
        qc_fails_df = get_qc_failures(a_df)
        if qc_fails_df.empty:
            # unless we've been told to suppress empty files
            if not suppress_empty_fails:
                # if there are no failures, create an empty file
                # (not even header line) if there are no failures--bc it is easy to
                # eyeball "zero bytes"
                timestamp_str = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
                qc_fails_fp = os.path.join(
                    out_dir, f"{timestamp_str}_{out_base}_fails.csv")
                Path(qc_fails_fp).touch()
            # else, just do nothing
        else:
            _write_df_to_file(qc_fails_df, out_dir, out_base,
                              sep=",", suffix="_fails")

        # then remove the qc fails and the internal columns from the metadata
        # TODO: I'd like to avoid repeating this mask here + in get_qc_failures
        fails_qc_mask = a_df[QC_NOTE_KEY] != ""
        a_df = a_df.loc[~fails_qc_mask, :].copy()
        a_df = _remove_internal_cols(a_df, internal_col_names)

    # output the metadata
    _write_df_to_file(a_df, out_dir, out_base, sep=sep)


def _reorder_df(a_df: pandas.DataFrame, internal_col_names: List[str]) -> pandas.DataFrame:
    """Reorder DataFrame columns according to standard rules.

    Parameters
    ----------
    a_df : pandas.DataFrame
        The DataFrame to reorder.
    internal_col_names : List[str]
        List of internal column names that will be moved to the end of the DataFrame.

    Returns
    -------
    pandas.DataFrame
        A reordered copy of the input DataFrame with:
            - sample_name as the first column
            - remaining columns except for internal columns in alphabetical order
            - internal columns at the end in the order they were provided
    """
    # sort columns alphabetically
    working_df = a_df.copy().reindex(sorted(a_df.columns), axis=1)

    # move the internal columns to the end of the list of cols to output
    col_names = list(working_df)
    for curr_internal_col_name in internal_col_names:
        # TODO: throw an error if the internal col name is not present
        col_names.pop(col_names.index(curr_internal_col_name))
        col_names.append(curr_internal_col_name)

    # move sample name to the first column
    col_names.insert(0, col_names.pop(col_names.index(SAMPLE_NAME_KEY)))
    output_df = working_df.loc[:, col_names].copy()
    return output_df


def _load_metadata_df(raw_metadata_fp: str) -> pandas.DataFrame:
    """Load a metadata file into a DataFrame based on its file extension.

    Parameters
    ----------
    raw_metadata_fp : str
        Path to the raw metadata file (.csv, .tsv, .txt, or .xlsx).

    Returns
    -------
    pandas.DataFrame
        The loaded metadata DataFrame with all columns as strings.

    Raises
    ------
    ValueError
        If the input file extension is not recognized.
    """
    extension = os.path.splitext(raw_metadata_fp)[1]
    if extension == ".csv":
        raw_metadata_df = load_df_with_best_fit_encoding(raw_metadata_fp, ",", str)
    elif extension in (".txt", ".tsv"):
        raw_metadata_df = load_df_with_best_fit_encoding(raw_metadata_fp, "\t", str)
    elif extension == ".xlsx":
        # NB: this loads (only) the first sheet of the input excel file.
        # If needed, can expand with pandas.read_excel sheet_name parameter.
        raw_metadata_df = pandas.read_excel(raw_metadata_fp, dtype=str)
    else:
        raise ValueError("Unrecognized input file extension; "
                         "must be .csv, .tsv, .txt, or .xlsx")
    return raw_metadata_df


def _extend_metadata_from_full_flat_config(
        raw_metadata_df: pandas.DataFrame,
        full_flat_config_dict: Dict[str, Any],
        study_specific_transformers_dict: Optional[Dict[str, Any]],
        hosttype_col_name: Optional[str],
        sampletype_col_name: Optional[str]) -> Tuple[pandas.DataFrame, pandas.DataFrame, Dict[str, str]]:
    """Resolve shorthand columns and populate a metadata DataFrame using a full flat config.

    The metadata df must have metameq-specific host- and sample-type-shorthand columns,
    since these are used to determine which specific metadata to add for each sample.
    If not already present under their internal names, this function checks for alternate
    column names (from the caller or from the config) and copies them into the internal
    shorthand columns before populating the metadata.

    Parameters
    ----------
    raw_metadata_df : pandas.DataFrame
        The raw metadata DataFrame to extend.
    full_flat_config_dict : Dict[str, Any]
        Fully combined flat-host-type config dictionary.
    study_specific_transformers_dict : Optional[Dict[str, Any]]
        Dictionary of custom transformers for this study (only).
    hosttype_col_name : Optional[str]
        Name of the column in raw_metadata_df that contains host type
        values. If provided, its values are copied into the internal
        ``hosttype_shorthand`` column before processing. If None, the
        function checks the config's ``hosttype_col_options`` list for a
        matching column in the DataFrame.
    sampletype_col_name : Optional[str]
        Name of the column in raw_metadata_df that contains sample type
        values. If provided, its values are copied into the internal
        ``sampletype_shorthand`` column before processing. If None, the
        function checks the config's ``sampletype_col_options`` list for
        a matching column in the DataFrame.

    Returns
    -------
    Tuple[pandas.DataFrame, pandas.DataFrame, Dict[str, str]]
        A tuple containing:
            - The extended metadata DataFrame
            - A DataFrame containing validation messages
            - A mapping from internal column keys to the user-facing
              column names (e.g. ``{"hosttype_shorthand": "host_type"}``)

    Raises
    ------
    ValueError
        If required columns are missing from the metadata, if a specified
        column name is not found in the DataFrame, or if both the internal
        shorthand column and the specified alternate column exist.
    """
    col_name_mapping = {}
    needed_cols = [(HOSTTYPE_SHORTHAND_KEY, hosttype_col_name, HOSTTYPE_COL_OPTIONS_KEY),
                   (SAMPLETYPE_SHORTHAND_KEY, sampletype_col_name, SAMPLETYPE_COL_OPTIONS_KEY)]
    for curr_internal_key, curr_param_key, curr_options_key in needed_cols:
        specified_name = _find_internal_col_source_name(
            raw_metadata_df, full_flat_config_dict,
            curr_param_key, curr_internal_key, curr_options_key)
        if specified_name:
            raw_metadata_df[curr_internal_key] = raw_metadata_df[specified_name]
            col_name_mapping[curr_internal_key] = specified_name
        else:
            col_name_mapping[curr_internal_key] = curr_internal_key

    validate_required_columns_exist(
        raw_metadata_df, REQUIRED_RAW_METADATA_FIELDS,
        "metadata missing required columns")

    metadata_df, validation_msgs_df = _populate_metadata_df(
        raw_metadata_df, full_flat_config_dict,
        study_specific_transformers_dict)

    return metadata_df, validation_msgs_df, col_name_mapping
