"""Classes and functions for simulating word-identification or phoneme-discrimination experiments
like those done for IEEE Trans ASLP paper

*** Main Module Classes:

SimPopulation: defines distribution of response probabilities
    in ONE population of potential participants in a speech test.
    Can draw simulated test participants at random from the population.

SimExperiment: defines a complete simulated speech-identification experiment,
    generates a cm_data.StimRespCountSet with simulated responses
    from simulated participants in one or more groups,
    sampled from SimPopulation instance(s).
    The resulting stim-response data layout is defined by a cm_data.ConfMatrixFrame instance,
    just like the data recorded from an experiment with real participants.

SimSubject = defines response probabilities for ONE participant,
    and generates simulated responses.

SimGroup: container for SimSubject instances
    drawn at random from an SimPopulation instance.

*** Version history:

* Version 0.8:
2026-02-18, tested mainly using SimPopulation.from_pc
2026-01-09, first version of general simulator for extended types of forced-choice speech tests.

* Version <=0.7.1:
Could simulate only some specific examples used for tests
like those in the IEEE Trans ASLP paper 2016
"""
# and special function sim_miller_population using MillerNicely data only for IEEE_paper_tests

import numpy as np
import logging
# import copy
from itertools import product
# from scipy.interpolate import interp1d
import pandas as pd

from ConfMatrixCalc.cm_data import ConfMatrixFrame, StimRespCountSet, COUNT_COL
# from ConfMatrixCalc.cm_model import ResponseProbPopulationPoint

from ConfMatrixCalc.cm_display import prob_correct, mutual_information
from ConfMatrixCalc.cm_file import Table

logger = logging.getLogger(__name__)
# logger.setLevel(logging.INFO)

# JEFFREYS_CONC = 0.5
POPULATION_PRIOR_PSEUDO_COUNT = 1.
# = pseudo-count of presentations for each stim,
# with equal response prob across all allowed response categories
#, i.e., POPULATION_PRIOR_PSEUDO_COUNT / n_responses added to observed count

RNG = np.random.default_rng()


# --------------------------------------- Response models
class SimSubject:
    """ONE simulated participant in speech-recognition experiment
    with forced-choice responses.
    """
    def __init__(self, u, cmf, rng=RNG):
        """
        :param u: dict with elements (sg, sr_sg), where
            sg is stim-group label and
            sr_sg is a 3D array with
            sr_sg[t, s, r] = conditional response prob for r-th response category
                given s-th stim in t-th test condition
            sr_sg.shape[0] MUST be same for all sg
        :param cmf: ConfMatrixFrame instance of population
        :param rng: (optional) np.random.Generator instance
        """
        n_tc = [u_sg.shape[0] for u_sg in u.values()]
        if not all(n_tc[0] == n_tc_i for n_tc_i in n_tc):
            raise RuntimeError('All prob arrays must have same shape[0]')
        self.u = u
        self.cmf = cmf
        self.rng = rng

    def response_prob_df(self):
        """Conditional response probabilities in DataFrame format,
        similar to estimated results from cm_model.
        :return: pd.DataFrame with
            index = cmf.test_condition_index()
            columns = cmf.stim_resp_index
        """
        u = {sg: u_sg.reshape((u_sg.shape[0], -1))
             for (sg, u_sg) in self.u.items()}
        u = np.concatenate([u_sg for u_sg in u.values()])  # simplify
        return pd.DataFrame(u,
                            index=self.cmf.test_cond_index(),
                            columns=self.cmf.stim_resp_index)

    def stim_resp_result(self, min_pres=1, max_pres=1, random_stim=False):
        """Simulate a speech test with ONE subject:
        Generate a sequence of stim-response pairs at random
        using response probabilities of self.
        :param min_pres: scalar integer min number of presentations per stim category
        :param max_pres: maximal number of presentations per stim category
        :param random_stim: (optional) boolean
            True -> select stim at random, with prob self.cmf.stim_prob(),
            False -> each stim category presented exactly min_pres times,
                for each participant in each group
        :return: sr_series = a pd.Series instance with tuples (stim_group, stim, resp),
            each representing ONE presentation, as if collected from a real test.
            sr_series.size == len(cmf.stim_index) * n_pres_per_stim
        """
        def gen_stim_cat():
            """Generator of randomly or sequentially selected stimuli
            :return: iterator over tuples (sg, s, s_i), where
                sg is stim-group label,
                s is cmf.stim_groups[sg].stim[s_i]
            """
            if random_stim:
                s_cat = [(sg_key, s, s_i)
                         for (sg_key, sg) in self.cmf.stim_groups.items()
                         for (s_i, s) in enumerate(sg.stim)]
                s_prob = self.cmf.stim_prob().to_numpy()
                # len(s_prob) == len(s_cat), because cmf.stim_index constructed from cmf.stim_groups
                if max_pres > min_pres:
                    n_pres_per_stim = min_pres + self.rng.choice(max_pres - min_pres)
                else:
                    n_pres_per_stim = min_pres
                return (s_cat[self.rng.choice(len(s_prob), p=s_prob)]
                        for _ in range(n_pres_per_stim * len(s_cat)))
            else:
                 return ((sg, s, s_i)
                        for _ in range(min_pres)
                        for (sg, s_sg) in self.cmf.stim_groups.items()
                        for (s_i, s) in enumerate(s_sg.stim))
        # ------------------------------------------------------------------
        tc_list = list(self.cmf.test_conditions())
        sr = [tc + (sg, s, str(self.rng.choice(self.cmf.stim_groups[sg].resp,
                                           p=self.u[sg][tc_i, s_i])))
              for (tc_i, tc) in enumerate(tc_list)
              for (sg, s, s_i) in gen_stim_cat()
              ]
        t_keys = list(self.cmf.test_factors.keys())
        col = t_keys + [self.cmf.stim_group_head, self.cmf.stim_head, self.cmf.resp_head]
        sr = pd.DataFrame.from_records(sr, columns=col)
        # **** keep as DataFrame with stim, resp, if allowed by StimRespCountSet ********
        # **** for now: store as pd.Series, as required by StimRespCountSet:
        sr = sr.groupby(col, sort=False, as_index=True).value_counts(sort=False)
        # = Series of counts, with unique index
        sr.name = 'Count'  # ****** get from cm_data ?, or skip ?
        return sr


# --------------------------------------------------
class SimPopulation:
    """Simulated population model that can generate
    simulated test participants in a speech-recognition test.
    The population specifies a distribution of response probabilities
    as a mixture of Dirichlet distributions
    for each response-probability vector for each stimulus in each test condition.
    All those Dirichlet distributions are defined by population properties
    conc, weight.

    Each random "participant" is a SimSubject instance,
    specifying a fixed dict with response-probabilities,
    drawn at random form the population.
    """
    def __init__(self, cmf, conc, weight, name, rng=RNG):
        """
        :param cmf: a cm_data.ConfMatrixFrame instance
        :param conc: dict with elements (sg, sr), with
            sg in cmf.stim_groups.keys()
            sr is 4D array with
            sr[m, t, s, r] = m-th mixture of concentration param for r-th response,
                given s-th stim in t-th test condition
        :param weight: 1D array of mixture weights, normalized
            weight[m] = weight for m-th mixture component
            weight.shape == (len(conc),)
        :param name: string label identifying this population,
            used only as informative prefix of subject name labels.
        :param rng: (optional) np.random.Generator instance,
            also used by all SimSubject objects drawn from this population.
        """
        sh = [conc_sg.shape[:2] for conc_sg in conc.values()]
        if not all(sh[0] == sh_i for sh_i in sh):
            raise RuntimeError('All conc arrays must have same shape[:2]')
        self.cmf = cmf
        self.conc = conc
        self.weight = weight
        self.name = name
        self.rng = rng

    # def __repr__(self):
    # def __str__(self):

    @classmethod
    def from_pc(cls, pc,
                conc_scale,
                conc_rel_sd,
                cmf,
                test_cond_effects=None,
                n_comp=1, name='', rng=RNG):
        """Create a cls instance with given parameters
        :param pc: dict with elements (sg_key, pc_sg), where
            sg_key = some element in cmf.stim_groups,
            pc_sg = scalar MEAN prob.correct in this stim group,
                with equal probabilities for all incorrect responses.
            Values for any cmf.stim-groups NOT included here are interpolated
            random-uniformly between the smallest and largest specified value.
        :param test_cond_effects: dict with elements (tc, tc_dev), where
            tc is a tuple with one category from each factor in cmf.test_factors,
            tc_dev = scalar in range (-1, +1) = relative DEVIATION from the values defined by pc,
                -> conc values scaled by (1. + tc_dev) for correct responses, no change for incorrect.
            Effect is zero for any tc not specified in test_cond_effects.
        :param conc_scale: scale factor for concentration re mean response probs
        :param conc_rel_sd: scalar deviation of random variations in conc values
            such that std(log(conc)) = log(1. + conc_rel_sd),
            with independent random variations
            across all mixture components, test conditions, stim, resp
        :param cmf: a cm_data.ConfMatrixFrame instance
            defining stim_groups, test conditions
        :param n_comp: (optional) number of mixture components with equal weights
        :param name: (optional) string label identifying this population,
        :param rng: (optional) numpy.random.Generator instance
        :return: a cls instance
        """
        def stim_group_conc(sg_key, pc_sg):
            """Calc conc params for ONE stim group
            :param sg_key: key label for this stim group
            :param pc_sg: scalar prob.correct for this stim group
            :return: c_sg = 4D array with
                c_sg[m, t, s, r] = concentration param of m-th mixture comp
                    for r-th response to s-th stim in t-th test condition
            """
            assert 0. < pc_sg < 1., 'Prob.Correct must be between 0. and 1.'
            (n_stim, n_resp) = cmf.stim_groups[sg_key].shape
            n_tc = cmf.n_test_conditions
            u = np.ones((n_comp, n_tc, n_stim, n_resp)) * (1. - pc_sg) / (n_resp -1)  # error probs
            u[..., range(n_stim), range(n_stim)] = pc_sg
            c = u * conc_scale
            rnd_scale = np.log(1. + conc_rel_sd)
            c *= np.exp(rnd_scale * rng.standard_normal(size=c.shape))
            return c
        # ----------------------------------------------------
        if len(pc) < 1:
            raise RuntimeError('Must have pc for at least ONE stim group')
        if test_cond_effects is None:
            test_cond_effects = dict()
        pc_min = np.min(pc.values())
        pc_max = np.max(pc.values())
        for sg in cmf.stim_groups.keys():
            if sg not in pc.keys():
                pc[sg] = rng.uniform(pc_min, pc_max)  # -> stim_group_conc ? ****
        # try:
        conc = {sg: stim_group_conc(sg,pc[sg])
                for sg in cmf.stim_groups.keys()}
        # except KeyError as e:  # ****** interpolate randomly if not all stim_groups defined ?
        #     raise RuntimeError(f'Missing stim group: ' + str(e))
        for (tc_i, tc) in enumerate(cmf.test_conditions()):
            if tc in test_cond_effects.keys():
                tc_effect = test_cond_effects[tc]
                tc_effect = min(1., max(-1., tc_effect))
                for c_sg in conc.values():
                    n_stim = c_sg.shape[2]
                    # ******* adjust incorrect elements by (1. - tc_effect), too ? *********
                    c_sg[:, tc_i] *= (1. - tc_effect / 2)
                    c_sg[:, tc_i, range(n_stim), range(n_stim)] *= (1. + tc_effect)
                logger.info(f'Effect={tc_effect:+.2f} included for test cond {tc}')
            else:
                logger.debug(f'No effect of test_cond {tc}')
        weight = np.ones(n_comp) / n_comp
        return cls(cmf, conc, weight, name)

    @classmethod
    def from_count_df(cls, df, conc_scale, conc_rel_sd, cmf, n_comp=1,
                      count_col=COUNT_COL, name='', rng=RNG):   # *** ?
        """Create a cls instance from given parameters
        :param df: a pd.DataFrame table (in long format) with
            columns [*cmf.test_factors, cmf.stim_group_head, cmf.stim_head, cmf.resp_head],
            and one column count_col with RELATIVE COUNTS,
            used to define MEAN response probabilities.
            At least ONE category example from each test_factor,
            and at least ONE stim category from each stim-group MUST be specified in some row.
            All (stim, resp) pairs NOT specified in any row will be assigned count = 0.
            All counts will then be increased by
            POPULATION_PRIOR_PSEUDO_COUNT / n_response_alternatives,
            just like the POPULATION_PRIOR_PSEUDO_COUNT is used in cm_model.

            Test conditions NOT specified in any row will be assigned with
            randomly interpolated relative effects,
            between the smallest and largest test-condition effect actually specified.
        :param conc_scale: scale factor for concentration re mean response probs
        :param conc_rel_sd: scalar control of random log-normal variations in conc values
            such that std(log(conc)) = log(1. + conc_rel_sd),
            with independent random variations
            across all mixture components, test conditions, stim, resp
        :param cmf: a cm_data.ConfMatrixFrame instance
            defining stim_groups, test conditions
        :param n_comp: (optional) number of mixture components with equal weights
        :param count_col: (optional) header for count column in input df
        :param name: (optional) string label identifying this population,
        :param rng: (optional) numpy.random.Generator instance
        :return: a cls instance
        """
        raise NotImplementedError

    @classmethod
    def from_model(cls, cmm, conc_scale, conc_rel_sd, cmf, n_comp=1,
                      test_cond_effects=None, name='', rng=RNG):   # *** ?
        """Create a cls instance from given parameters
        :param cmm: a cm_model.ConfMatrixModel previously learned
        :param test_cond_effects: dict with elements (tc, tc_dev),
            for any additional test conditions NOT already included in cmm.
            tc is a tuple with one category from each factor in cmf.test_factors,
            tc_dev = scalar in range (-1, +1) = relative DEVIATION from the values defined by pc,
                -> conc values scaled by (1. + tc_dev) for correct responses, no change for incorrect.
            Effect is zero for any tc not specified in test_cond_effects.
        :param conc_scale: scale factor for concentration re mean response probs
        :param conc_rel_sd: scalar control of random log-normal variations in conc values
            such that std(log(conc)) = log(1. + conc_rel_sd),
            with independent random variations
            across all mixture components, test conditions, stim, resp
        :param cmf: a cm_data.ConfMatrixFrame instance
            defining stim_groups, test conditions,
            that may be superset or subset of those already specified in cmm.cmf.
        :param n_comp: (optional) number of mixture components with equal weights
        :param name: (optional) string label identifying this population,
        :param rng: (optional) numpy.random.Generator instance
        :return: a cls instance
        """
        raise NotImplementedError

    @classmethod
    def from_arrays(cls, p, conc_scale, cmf, name=''):
        """Create a cls instance from given parameters
        :param p: dict {sg: p_sg} for sg in cmf.stim_groups.keys().
            p_sg = 4D array with
            p_sg[m, t, s, r] = non-normalized response prob for
                r-th response to s-th stim in t-th test-cond in m-th mixture component.
            array shapes must match cmf.test_conditions and cmf.stim_groups[sg].shape
        :param cmf: a cm_data.ConfMatrixFrame instance
            defining stim_groups, test conditions
        :return: a cls instance
        """
        def stim_group_conc(x):
            """
            :param x: 4D array of (possibly un-normalized) response probabilities
            x[m, t, sgs, r] = values for m-th mixture component
                in t-th test cond., for sgs-th (stim_group, stim) tuple, r-th response alternative
            :return: 4D conc array conc propto normalized x, scaled by conc_scale
            """
            assert x.ndim == 4, 'SimPopulation.from_arrays must get 4D input arrays'
            return conc_scale * x / np.sum(x, axis=-1, keepdims=True)
        # ----------------------------------------------------------
        try:
            conc = {sg: stim_group_conc(p[sg])
                    for sg in cmf.stim_groups.keys()}
        except KeyError as e:
            raise RuntimeError(f'Missing stim group: ' + str(e))
        sh = [conc_sg.shape[:2] for conc_sg in conc.values()]
        if not all(sh[0] == sh_i for sh_i in sh):
            raise RuntimeError('All conc arrays must have same shape[:2]')
        n_mix = sh[0][0]
        weight = np.ones(n_mix) / n_mix
        return cls(cmf, conc, weight, name)

    def mean(self):
        """Mean response probability
        :return: m = dict with elements (sg, u_sg), where
            m_keys = cmf.stim_group.keys(), and each
            u_sg = 3D array with
                u_sg[t, s, r] = mean conditional prob for r-th response category,
                given s-th stim in t-th test condition
        """
        return {sg: np.tensordot(self.weight,
                                 conc_sg / np.sum(conc_sg,
                                                  axis=-1, keepdims=True),
                                 axes=1)
                for (sg, conc_sg) in self.conc.items()}

    def conc_df(self):
        """
        :return: pd.DataFrame instance with
            index = (sample, test-cond), defined by self._rvs_index()
            columns = (stim, resp), as defined by self.cmf.stim_resp_index_OLD()
        """
        a = self.conc.copy()
        a = {sg: a_sg.reshape((*a_sg.shape[:2], -1))
             for (sg, a_sg) in a.items()}
        a = np.concat([a_sg for a_sg in a.values()], axis=-1)
        return pd.DataFrame(a.reshape((-1, a.shape[-1])),
                            columns=self.cmf.stim_resp_index,
                            index=self.cmf.rvs_index(len(a)))   #self._rvs_index(len(a)))

    def rvs(self, n_samples):
        """Generate samples of response-probability matrices
        from self = a mixture of Dirichlet distribution
        :param n_samples: scalar number of random samples
        :return: u = dict with elements (sg, u_sg), where
            sg is stim-group key label from self.cmf.stim_groups.keys()
            u_sg is 4D array of response-probability samples of confusion matrices
             u[n, t, s, r] = n-th matrix sample of response prob
                for r-th response to s-th stimulus in t-th test condition
             u.shape == (n_samples, *self.conc.shape[1:])
        """
        def rvs_stim_group(c):
            """Draw random samples from Dirichlet mixture distribution
            for ONE stim group.
            :param c: one 4D array of concentration samples = ONE of self.conc.values()
            :return: u = 4D array of sample matrices, with
                u[n, :, :, :] = n-th matrix sample of response prob
                u.shape == (n_samples, *c.shape[1:])
            """
            ns = self.rng.multinomial(n_samples, pvals=self.weight)
            # NOTE: np.random.Generator.dirichlet can use only 1D conc vector:
            u = [np.stack([np.stack([self.rng.dirichlet(alpha=c_mts, size=n_m)  # shape == (n_m, n_resp)
                                  for c_mts in c_mt], axis=1)  # stacked by s: shape == (n_m, n_stim, n_resp)
                                 for c_mt in c_m], axis=1)  # stacked by t: shape == (n_m, n_t, n_stim, n_resp)
                                for (c_m, n_m) in zip(c, ns) if n_m > 0]
            u = np.concatenate(u, axis=0)   # shape == (n_samples = sum_n n_m, n_t, n_stim, n_resp)
            self.rng.shuffle(u, axis=0)
            return u
        # --------------------------------------------------------------------
        return {sg: rvs_stim_group(conc_sg)
                for (sg, conc_sg) in self.conc.items()}

    def rvs_df(self, n_samples):
        """Generate samples of response-probability matrices
        from self = a mixture of Dirichlet distribution
        :param n_samples: scalar number of random samples
        :return: u = pd.DataFrame with response probabilities, stored with
            index = pd.MultiIndex with levels (sample, *test-conditions()),
            columns = cmf.stim_resp_index = pd.MultiIndex (stim_group, stim, resp)
        """
        u = self.rvs(n_samples)
        # = dict with u.keys() == self.cmf.stim_groups.keys()
        u = [u_sg.reshape((-1, u_sg.shape[-2] * u_sg.shape[-1])) for u_sg in u.values()]
        u = np.concat(u, axis=-1)
        return pd.DataFrame(u,
                            index=self.cmf.rvs_index(n_samples),  # self._rvs_index(n_samples),
                            columns=self.cmf.stim_resp_index)


    def random_group(self, n_participants=1):
        """Generate simulated subjects with response probabilities drawn from self
        suitable for a planned experiment.
        :param n_participants: number of randomly drawn participants from self
        :return: a SimGroup instance, containing generated participants,
            each with properties drawn from self.
        """
        return SimGroup(self,
                        {str(self.name) + f'_S{i}': self.random_subject()
                         for i in range(n_participants)})

    def random_subject(self):
        """Draw ONE random participant from self
        :return: a SimSubject instance
        """
        def random_sg_prob(conc):
            """Draw ONE prob array from given conc array
            :param conc: 3D array with
                conc[t, s, r] = conc param for r-th response to s-th stim in t-th test cond
            :return: 3D array u with
                u[t, s, :] = random Dirichlet response-prob vector for s-th stim in t-th test cond
                u.shape == conc.shape
            """
            u = np.stack([np.stack([self.rng.dirichlet(c_ts) # one 1D random vector: shape == (n_resp,)
                                    for c_ts in c_t], axis=0)   # stacked by s: shape == (n_stim, n_resp)
                          for c_t in conc], axis=0)     # stacked by t: shape == (n_tc, n_stim, n_resp)
            return u
        # ---------------------------------------------------
        rnd_m = self.rng.choice(len(self.weight), p=self.weight)
        # = index of ONE randomly selected mixture component
        u = {sg: random_sg_prob(c_sg[rnd_m])
             for (sg, c_sg) in self.conc.items()}
        return SimSubject(u, self.cmf,self.rng)

    def pc_mi_quantiles(self, q, n_samples=2000):
        """Calculate quantiles of true distribution of
        Prob.Correct and Mutual Information
        in the population defined by self.
        :param q: sequence of one or more quantiles
        :param n_samples: (optional) integer number of samples for calc
        :return: tuple (pc, mi), where both are
            cm_file.Table(pd.DataFrame) instances with
            one row for each test condition,
            one column for each requested quantile
        """
        u = self.rvs_df(n_samples)
        res = []    # space for two results
        for meas_fcn in [prob_correct, mutual_information]:
            m_ds = meas_fcn(u, self.cmf)
            m_ds = m_ds.groupby(level=list(self.cmf.test_factors.keys()),
                                sort=False, group_keys=True).quantile(q).unstack(level=-1)
            m_ds.columns.name = 'Quantile'
            res.append(Table(m_ds))
        return res



# --------------------------------------------
class SimGroup:
    """Group of test participants drawn from ONE given population
    """
    def __init__(self, pop, participants):
        """
        :param pop: a SimPopulation instance,
            from which participants have been drawn at random
        :param participants: dict with items (s_id, s_sim)
            s_id = string name for the subject
            s_sim = a SimSubject instance with pre-defined response probabilities
        """
        self.pop = pop
        self.participants = participants


# -------------------------------------------------------------------------
class SimExperiment:
    """Defines a simulated speech-recognition data-collection experiment,
    with one or more groups of simulated participants, with
    each group generated from a SimPopulation instance.

    Method gen_dataset() generates a complete cm_data.StimRespDataSet instance
    with stim-response for all participants in all groups.

    The experimental procedure is defined by
    cmf = a cm_data.ConfMatrixFrame instance
    """
    def __init__(self, cmf, groups):
        """
        :param cmf: ConfMatrixFrame instance, defining experimental parameters
        :param groups: dict with elements (g_id, g_sim),
            g_id is a tuple with one or several group-category labels,
                one for each element in cmf.group_head
            g_sim = a SimGroup instance
        """
        self.cmf = cmf
        group_keys = [cmf.match_group_key(g) for g in groups.keys()]
        for (gk_new, gk_old) in zip(group_keys, groups.keys()):
            if gk_new != gk_old:
                groups[gk_new] = groups.pop(gk_old)
                logger.warning(f'Group key {gk_old} changed to {(gk_new,)} to match ConfMatrixFrame.')
        self.groups = groups

    def gen_dataset(self, min_pres=1, max_pres=1, random_stim=False):
        """Generate a complete EmaDataSet instance for this experiment,
        with one or more groups of participants.
        :param min_pres: scalar integer min number of presentations per stim category
        :param max_pres: maximal number of presentations per stim category
        :param random_stim: (optional) boolean
            True -> select stim at random, with prob self.cmf.stim_prob(),
            False -> each stim category presented exactly min_pres times,
                for each participant in each group
        :return: a StimRespDataSet instance
        """
        cmd = {g: {s_id: s.stim_resp_result(min_pres, max_pres, random_stim)
                   for (s_id, s) in g_sim.participants.items()}
               for (g, g_sim) in self.groups.items()}
        return StimRespCountSet(self.cmf, cmd)


# --------------------------------------------- module help functions:


# -------------------------------------- TEST:
if __name__ == '__main__':
    from pathlib import Path
    ConfData = Path.home() / 'Documents/LeijonKTH/heartech/ArneForskning/ConfMatrix/ConfData'
    MillerData = ConfData / 'MillerNicely1955'
    # MILLER_FILE = 'Miller1955_SNR_'  # first part of name, rest shows the SNR
    MILLER_SNR = [-12, -6, 0, 6]  # confusion-count data files available for these SNRs

    # ---------- test read_excel with wide format:
    testMillerFile = MillerData / 'Miller1955_SNR_0.xlsx'
    cons = ['p', 't', 'k', 'f', 'th', 's', 'sh', 'b', 'd', 'g', 'v', 'dh', 'z', 'zh', 'm', 'n']
    stim_groups = {'Cons': cons}

    # SNR = np.asarray(SNR).reshape(-1)
    cmf = ConfMatrixFrame.setup(stim_groups,
                                no_response=None,
                                test_factors=dict(SNR=[f'{s}' for s in MILLER_SNR])
                                                  # groups={'Sim': ['Miller']}),
                                                  )

    # ------------------------------- test SimPopulation, SimExperiment

    test_pop_pc = SimPopulation.from_pc({'Cons': 0.6}, conc_scale=10., conc_rel_sd=0.1,
                                        test_cond_effects={('-12',): -0.5, ('6',): +0.5},
                                        cmf=cmf, n_comp=10, name='test_pop_pc')
    print('test_pop_pc=\n', test_pop_pc)
    print('test_pop_pc.conc_df=\n', test_pop_pc.conc_df())

    for (sg, mean_sg) in test_pop_pc.mean().items():
        for (tc_i, tc) in enumerate(cmf.test_conditions()):
            print(f'mean[{sg}][{tc}]=\n' + np.array_str(100*mean_sg[tc_i], precision=1))

    rvs_u = test_pop_pc.rvs(n_samples=10)
    for (sg, u_sg) in rvs_u.items():
        print(f'rvs_u[{sg}]=\n', np.array_str(u_sg, precision=2))
    rvs_df = test_pop_pc.rvs_df(n_samples=10)
    print(rvs_df)

    # ---------- test using the population:
    n_subjects = 20
    nx = 5  # presentations per stim

    groups = {(): test_pop_pc.random_group(n_subjects)}

    cm_exp = SimExperiment(test_pop_pc.cmf, groups)
    sr_set = cm_exp.gen_dataset(min_pres=nx, random_stim=True)
    print('sr_set=\n', sr_set)
    print('sr_set.counts:\n')
    for (g, g_c) in sr_set.gen_response_counts():
        print(f'Group {repr(g)}:')
        for (s, s_c) in g_c:
            print('Subject ' + s + ':\n', s_c.stack(level=[0, 1], future_stack=True).to_string())


    # --------------------- test SimPopulation.from_count_df() using sr_set

    print('Done')

