import numpy as np
import datetime
import os
import glob
import warnings
import copy
import logging
from collections import OrderedDict
from . import spec_tools as st
from . import parse_tools as pt
from configparser import RawConfigParser

__all__ = ["XFasterConfig", "XFaster", "XFasterWarning"]


class XFasterWarning(Warning):
    """
    Warning generated by the XFaster algorithm.
    """

    pass


class XFasterConfig(RawConfigParser):
    """
    ConfigParser subclass for storing command line options and config.
    """

    def __init__(self, defaults=None, default_sec="Uncategorized"):
        """
        Class that tracks command-line options for storage to disk.

        Arguments
        ---------
        defaults : dict
            Dictionary of overall configuration values.
            Eg: locals() at beginning of function, or vars(args) from argparse
        default_sec : string, optional
            The name of the default section in the configuration file.
        """
        super(XFasterConfig, self).__init__(dict_type=OrderedDict)
        self.default_sec = default_sec
        self.add_section(default_sec)
        if defaults is not None:
            self.update(defaults)

    def update(self, options, section=None):
        """
        Update configuration options with a dictionary. Behaves like
        dict.update() for specified section but also clears options of the same
        name from the default section.

        Arguments
        ---------
        options : dict
            The options to update
        section : string, optional
            Name of section to update. Default: self.default_sec
        """
        if section is None:
            section = self.default_sec
        if not self.has_section(section):
            self.add_section(section)
        # change kwargs to be like any other options
        kw = options.pop("kwargs", None)
        if isinstance(kw, dict):
            options.update(kw)
        for k, v in sorted(options.items()):
            self.remove_option(self.default_sec, k)
            self.set(section, k, str(v))

    def sort(self):
        """
        Sort the items in each section of the configuration alphabetically.
        """
        for section, section_items in self.items():
            if sorted(section_items) == list(section_items):
                continue

            section_dict = {k: v for k, v in section_items.items()}

            for k in list(section_items):
                self.remove_option(section, k)

            for k, v in sorted(section_dict.items()):
                self.set(section, k, v)

    def write(self, fp=None, sort=True):
        """
        Write an .ini-format representation of the configuration state.
        Keys are stored alphabetically if `sort` is True.

        Arguments
        ---------
        fp : file object
            If None, write to `sys.stdout`.
        sort : bool
            If True, sort items in each section alphabetically.
        """
        if fp is None:
            import sys

            fp = sys.stdout

        if sort:
            self.sort()
        super(XFasterConfig, self).write(fp)


class XFaster(object):

    checkpoints = [
        "files",
        "masks",
        "kernels",
        "sims_transfer",
        "shape_transfer",
        "transfer",
        "sims",
        "beams",
        "data",
        "sim_data",
        "template_noise",
        "shape",
        "bandpowers",
        "beam_errors",
        "likelihood",
    ]

    # if starting from KEY, force rerun all steps in VALUES
    checkpoint_tree = {
        "files": ["masks"],
        "masks": [
            "kernels",
            "sims_transfer",
            "sims",
            "data",
            "sim_data",
            "template_noise",
        ],
        "kernels": ["transfer"],
        "sims_transfer": ["transfer"],
        "shape_transfer": ["transfer"],
        "transfer": ["bandpowers"],
        "sims": ["bandpowers"],
        "beams": ["transfer"],
        "data": ["bandpowers"],
        "sim_data": ["bandpowers"],
        "template_noise": ["bandpowers"],
        "shape": ["bandpowers"],
        "bandpowers": ["likelihood"],
        "beam_errors": ["likelihood"],
    }

    data_version = 4

    def __init__(
        self,
        config,
        output_root=None,
        output_tag=None,
        verbose="notice",
        debug=False,
        checkpoint=None,
        alm_pixel_weights=False,
        alm_iter=None,
        add_log=False,
    ):
        """
        Initialize an XFaster instance for computing binned power spectra
        using a set of data maps along with signal and noise simulations.

        Arguments
        ---------
        config : string
            Configuration file. If path doesn't exist, assumed
            to be in xfaster/config/<config>
        output_root : string
            Path to data output directory
        output_tag : string
            Tag to use for output data.  Results are typically stored in
            the form ``<output_root>/<output_tag>/<name>_<output_tag>.npz``
        verbose : string
            Verbosity level to use for log messages.  Can be one of
            ["critical", "error", "warning", "notice", "info", "debug", "all"].
        debug : bool
            Store extra data in output files for debugging.
        checkpoint : string
            If output data from this step forward exist on disk, they are
            are re-computed rather than loading from file.
            Options are {checkpoints}.
        alm_pixel_weights : bool
            If True, set the ``use_pixel_weights`` option to True when computing
            map Alms using ``healpy.map2alm``.  If False, sets the
            ``use_weights`` option to True instead.  Note: pixel weights ensure
            accurate Alm computation, but can only be used for analyses where
            ``lmax < 1.5 * nside``.
        alm_iter : int
            If given, set the ``iter`` option to the given value when computing
            map Alms using ``healpy.map2alm``.  Using more iterations improves
            the accuracy of the output Alms.  If not set, uses the default
            number of iterations (3) as defined in healpy.
        add_log : bool
            If True, write log output to a file instead of to STDOUT.
            The log will be in ``<output_root>/run_<output_tag>.log``.
        """
        # verbosity
        self.init_log(
            level=verbose,
            filename=self.get_filename("run", ext=".log") if add_log else None,
        )

        self.debug = debug

        # map tag configuration
        self.load_map_config(config)

        # checkpointing
        if checkpoint is not None:
            if checkpoint not in self.checkpoints:
                raise ValueError(
                    "Invalid checkpoint {}, must be one of {}".format(
                        checkpoint, self.checkpoints
                    )
                )
        self.checkpoint = checkpoint
        self.force_rerun = {cp: False for cp in self.checkpoints}

        # map2alm options
        if alm_pixel_weights:
            import healpy as hp

            if hp.__version__ < "1.12.0":
                self.warn("healpy > 1.11.0 required for alm_pixel_weights, disabling")
                alm_pixel_weights = False
        self.alm_pixel_weights = alm_pixel_weights
        self.alm_iter = alm_iter

        if output_root is None:
            output_root = os.getcwd()
            self.warn("No output root supplied, using {}".format(output_root))

        self.output_root = output_root
        self.output_tag = output_tag
        if self.output_tag is not None:
            self.output_root = os.path.join(self.output_root, self.output_tag)
        if not os.path.exists(self.output_root):
            os.makedirs(self.output_root)

    __init__.__doc__ = __init__.__doc__.format(checkpoints=checkpoints)

    def load_map_config(self, filename):
        """
        Load the input map configuration file.

        The configuration file should be a file that is readable using
        ``ConfigParser``.  It must contain at least a single section called
        "frequencies", with keys for each map tag that may be used by the
        algorithm.  If using the harmonic-domain foreground fitting portions of
        the algorith, the value of each key should be the observing frequency in
        GHz that is appropriate for each tag.  Otherwise, these frequencies can
        be any floating point value.

        Other optional sections include:

        **beam**:
        Beam window specifications for each of the tags in "frequencies".  The
        "beam_product" key should be a path to a .npz file containing a
        dictionary of beam windows keyed by tag.  The "beam_product_error" key
        should be a path to a similar dictionary containing fraction beam error
        envelopes, also keyed by tag.  See ``get_beams`` or ``get_beam_errors``
        for more details.

        **fwhm**:
        If using a Gaussian beam model, this section should contain a list of
        FWHM in arcmin for each such tag in "frequencies".  Keys missing here
        should be present in the beam product file.

        **fwhm_err**:
        If using a Gaussian beam model, this section should contain a list of
        fractional errors on the FWHM for each such tag in "frequencies".  Keys
        missing here should be present in the beam error product file.

        **transfer**:
        If present, this section should contain each of the keys in
        "frequencies", with the value set to "true" if a transfer function
        should be computed for the tag, and "false" otherwise (in which case the
        transfer function will be set to unity for all bins).  This option is
        useful for including, e.g. optimally weighted Planck maps with no
        transfer function in a joint analysis.  If not supplied, it is assumed
        that a transfer function should be computed for every tag in
        "frequencies".

        Arguments
        ----------
        filename : str
            Path to config file
        """

        # Load map configuration file
        assert os.path.exists(filename), "Missing config file {}".format(filename)
        self.config_root = os.path.dirname(os.path.abspath(filename))
        cfg = XFasterConfig()
        cfg.read(filename)

        # dictionary of map frequencies keyed by map tag
        assert "frequencies" in cfg, "Missing [frequencies] section"
        self.dict_freqs = {
            k: cfg.getfloat("frequencies", k) for k in cfg["frequencies"]
        }
        tagset = set(self.dict_freqs)
        assert len(self.dict_freqs) > 0, "At least one map tag is required"

        # beam fwhm for each tag, if not supplied in beam_product
        # converted from arcmin to radians
        if "fwhm" in cfg:
            tags = [k for k in self.dict_freqs if cfg.has_option("fwhm", k)]
            self.fwhm = {k: np.radians(cfg.getfloat("fwhm", k) / 60.0) for k in tags}
        else:
            self.fwhm = {}

        # beam fwhm error for each tag, if not supplied in beam_error_product
        if "fwhm_err" in cfg:
            tags = [k for k in self.dict_freqs if cfg.has_option("fwhm_err", k)]
            self.fwhm_err = {k: cfg.getfloat("fwhm_err", k) for k in tags}
        else:
            self.fwhm_err = {}

        # make sure beam product files exist
        if "beam" in cfg:
            v = cfg["beam"].get("beam_product", None)
            if str(v).lower() != "none":
                if not os.path.exists(v):
                    v = os.path.join(self.config_root, v)
                assert os.path.exists(v), "Missing beam product file {}".format(v)
                self.beam_product = pt.load_compat(v)
                # Only use the fields in the beam product we need
                for btag in set(self.beam_product) - tagset:
                    self.beam_product.pop(btag)
            else:
                self.beam_product = {}

            v = cfg["beam"].get("beam_error_product", None)
            if str(v).lower() != "none":
                if not os.path.exists(v):
                    v = os.path.join(self.config_root, v)
                assert os.path.exists(v), "Missing beam error product file {}".format(v)
                self.beam_error_product = pt.load_compat(v)
                # Only use the fields in the beam product we need
                for btag in set(self.beam_error_product) - tagset:
                    self.beam_error_product.pop(btag)
            else:
                self.beam_error_product = {}
        else:
            self.beam_product = {}
            self.beam_error_product = {}

        # make sure all tags are present in either beam products or fwhm tables
        fwhm_set = set(self.fwhm) | set(self.beam_product)
        assert fwhm_set == tagset, "Missing tags in [fwhm] or beam product"

        if len(self.fwhm_err) or len(self.beam_error_product):
            fwhm_set = set(self.fwhm_err) | set(self.beam_error_product)
            assert (
                fwhm_set == tagset
            ), "Missing tags in [fwhm_err] or beam error product"

        # fit for the transfer function for each tag?
        if "transfer" in cfg:
            tags = [k for k in self.dict_freqs if cfg.has_option("transfer", k)]
            self.fit_transfer = {k: cfg.getboolean("transfer", k) for k in tags}
            assert tagset == set(self.fit_transfer), "Missing tags in [transfer]"
        else:
            # assume true for all tags otherwise
            self.fit_transfer = {k: True for k in self.dict_freqs}

    def init_log(self, level="notice", filename=None):
        """
        Initialize the logger from the input keyword arguments.

        Arguments
        ---------
        level : string, optional, default: "notice"
            Verbosity level.
            Options are "critical", "error", "warning", "notice", "info", "debug", "all".
        filename : string, optional
            Logging output filename.  Default: None (print to sys.stdout)
        """

        # add NOTICE logging level
        logging.NOTICE = 25
        logging.addLevelName(logging.NOTICE, "NOTICE")

        def logger_notice(self, msg, *args, **kwargs):
            """
            Log a message with severity "NOTICE".

            Arguments
            ---------
            msg : str
                Log message
            """
            if self.isEnabledFor(logging.NOTICE):
                self._log(logging.NOTICE, msg, args, **kwargs)

        logging.Logger.notice = logger_notice

        def root_notice(msg, *args, **kwargs):
            """
            Log a message with severity "NOTICE" on the root logger.

            Arguments
            ---------
            msg : str
                Log message
            """
            if len(logging.root.handlers) == 0:
                logging.basicConfig()
            logging.root.notice(msg, *args, **kwargs)

        logging.notice = root_notice

        # create handler
        if filename is None:
            from .batch_tools import get_job_logfile

            filename = get_job_logfile()
        if filename is None:
            handler = logging.StreamHandler()
        else:
            handler = logging.FileHandler(filename)

        # create formatter
        fmt = logging.Formatter(
            fmt="[ %(asctime)s ] %(levelname)s: %(message)s",
            datefmt="%Y-%m-%d %H:%M:%S%Z",
        )
        handler.setFormatter(fmt)

        # configure logger
        self.logger = logging.getLogger("xfaster")
        self.logger.addHandler(handler)

        # set logging level
        if level is None:
            level = "notice"
        elif level.lower() == "all":
            level = "notset"
        self.logger.setLevel(getattr(logging, level.upper()))

    def log(self, message, level=None):
        """
        Log a message with the given logging level.

        Arguments
        ---------
        message : str
            Log message
        level : string, default : None
            Logging level.  Must be one of "critical", "error", "warning",
            "notice", "info", "debug", "all".  If not supplied, "all" is assumed.
        """

        if level is None or level.lower() == "all":
            level = "notset"
        level = getattr(logging, level.upper())

        self.logger.log(level, message)

    def warn(self, message):
        """
        Log a warning message.

        Arguments
        ---------
        message : str
            Warning log message
        """

        warnings.warn(message, XFasterWarning, stacklevel=2)

    def __del__(self):
        """
        Make sure logger is shutdown properly when the object is destroyed.
        """

        # cleanup logging handlers
        for handler in self.logger.handlers[::-1]:
            try:
                handler.acquire()
                handler.flush()
                handler.close()
            except (OSError, ValueError):
                pass
            finally:
                handler.release()

    def _get_data_files(
        self,
        data_type=None,
        data_root=None,
        data_subset=None,
        data_root2=None,
        data_subset2=None,
        config=False,
    ):
        """
        Convenience function for finding all matching map data files.  Used
        internally in ``get_files``.

        This function runs in two modes, depending on the value of ``config``.

        Arguments
        ---------
        data_type : string
            The type of data to use, required if ``config`` is False.
            Otherwise, default to ``'raw'``.
        data_root : string
            Top level path containing subdirectories for data, signal sims,
            noise sims, and masks.
        data_subset : string
            Subset of maps to use for spectrum estimation.  This should be
            a string that is parseable using ``glob`` on the path
            ``data_<data_type>/<data_subset>.fits``.  For example,
            ``'full/*0'`` will expand to read in the 150 GHz and 90GHz maps.
            Maps are then sorted in alphabetical order, and identified
            by their file tag, where each filename is ``map_<tag>.fits``.
        data_root2, data_subset2 : string
            The root and subset for a second set of data.  If either of these is
            keywords is supplied, then the two data sets are treated as two
            halves of a null test.  In this case, XFaster computes the sum and
            difference spectra for each map tag in order to estimate a null
            spectrum.
        config : bool
            If True, return a dictionary of attributes for defining the run
            configuration, such as properly constructed data roots, map tags and
            cross-spectrum pairings.  If False, return a dictionary of map
            filenames and file roots for building the appropriate cross spectra
            with ``get_masked_data()``.

        Returns
        -------
        opts : dict
            Dictionary of configuration options.
        """
        if config:
            num_maps = None
            map_tags_check = None
            null_run = False
            data_type = data_type or "raw"
            if data_root2 is not None or data_subset2 is not None:
                if data_root2 is None:
                    data_root2 = data_root
                if data_subset2 is None:
                    data_subset2 = data_subset
                if (data_root, data_subset) == (data_root2, data_subset2):
                    raise ValueError(
                        "Either data_root2 or data_subset2 must differ "
                        "from data_root/data_subset"
                    )
                null_run = True
        else:
            num_maps = len(self.map_names)
            mt = [os.path.splitext(os.path.basename(f))[0] for f in self.map_names]
            map_tags_check = np.asarray([f.split("_", 1)[1] for f in mt])
            null_run = self.null_run
            data_root = data_root or self.data_root
            data_subset = data_subset or self.data_subset
            if null_run:
                data_root2 = data_root2 or self.data_root2
                data_subset2 = data_subset2 or self.data_subset2

        if data_type is None:
            raise ValueError("Argument `data_type` required")

        sets = [(data_root, data_subset, "")]
        if null_run:
            sets += [(data_root2, data_subset2, "2")]

        out = {}
        if not config:
            out.update(data_type=data_type)

        for droot, dset, suffix in sets:
            if not os.path.exists(droot):
                raise OSError("Missing data root {}".format(droot))

            # find all map files
            map_root = os.path.join(droot, "data_{}".format(data_type))
            map_files = []
            for f in np.atleast_1d(dset.split(",")):
                files = glob.glob(os.path.join(map_root, "{}.fits".format(f)))
                if not len(files):
                    raise OSError("Missing files in data subset {}".format(f))
                map_files.extend(files)
            map_files = sorted(map_files)
            map_files = [f for f in map_files if os.path.basename(f).startswith("map_")]

            if num_maps is None:
                num_maps = len(map_files)
            elif len(map_files) != num_maps:
                raise ValueError(
                    "Found {} maps in root {}/{}, expected {}".format(
                        len(map_files), map_root, dset, num_maps
                    )
                )

            # extract tag for each map
            map_tags = [os.path.splitext(os.path.basename(f))[0] for f in map_files]
            map_tags = np.asarray([f.split("_", 1)[1] for f in map_tags])
            self.log("Map tags: {}".format(map_tags), "debug")

            if map_tags_check is None:
                map_tags_check = map_tags
            else:
                if (map_tags != map_tags_check).any():
                    raise ValueError(
                        "Found map tags {} in root {}/{}, expected {}".format(
                            map_tags, map_root, dset, map_tags_check
                        )
                    )
                if not config:
                    out.update(
                        {
                            "map_root{}".format(suffix): map_root,
                            "map_files{}".format(suffix): map_files,
                        }
                    )
                    self.log(
                        "Found {} map files in {}".format(num_maps, map_root), "info"
                    )
                    self.log("Map files: {}".format(map_files), "debug")
                    continue

            # file names relative to map_root
            map_names = [os.path.relpath(f, map_root) for f in map_files]

            out.update(
                {
                    "data_root{}".format(suffix): droot,
                    "data_subset{}".format(suffix): dset,
                    "map_names{}".format(suffix): map_names,
                }
            )

            if "map_tags" in out:
                continue

            # Also need a list of unique map tags for populating dictionaries
            # in data structures
            map_tags_orig = list(map_tags)  # copy
            map_tags = pt.unique_tags(map_tags)

            # make a list of names corresponding to the order of the cross spectra
            map_pairs = pt.tag_pairs(map_tags)
            map_pairs_orig = pt.tag_pairs(map_tags, index=map_tags_orig)

            # make a dictionary of map frequencies for each unique map tag
            map_freqs = dict(zip(map_tags, [self.dict_freqs[t] for t in map_tags_orig]))
            self.log("Map freqs: {}".format(map_freqs), "debug")

            out.update(
                null_run=null_run,
                num_maps=num_maps,
                map_names=map_names,
                map_tags=map_tags,
                map_pairs=map_pairs,
                map_tags_orig=map_tags_orig,
                map_pairs_orig=map_pairs_orig,
                map_freqs=map_freqs,
            )

        return out

    def _get_mask_files(self, mask_type):
        """
        Convenience function or finding mask file for each map.  Used internally
        in ``get_files()``.

        Arguments
        ---------
        mask_type : string
            The variant of mask to use, e.g. 'rectangle', etc.  We assume a mask
            per file tag in the masks_<mask_type> folder, corresponding to the
            files in data.

        Returns
        -------
        opts : dict
            Dictionary of mask file options, including mask_type, mask_root, and
            mask_files.
        """
        if mask_type is None:
            raise ValueError("Argument `mask_type` required")

        if os.path.splitext(mask_type)[1] == ".fits":
            # use the same mask file for all maps
            mask_file = mask_type
            if not os.path.exists(mask_file):
                mask_file = os.path.join(self.data_root, mask_type)
            if not os.path.exists(mask_file):
                mask_file = os.path.join(self.config_root, mask_type)
            if not os.path.exists(mask_file):
                raise OSError("Missing mask file {}".format(mask_type))

            mask_files = np.tile(mask_file, len(self.map_names))
            mask_root = os.path.dirname(mask_file)
        else:
            # find all masks corresponding to each map tag
            mask_root = os.path.join(self.data_root, "masks_{}".format(mask_type))
            # XXX Do this smarter
            # e.g. allow different masks for different chunks?
            mask_files = [
                os.path.join(mask_root, "mask_{}".format(os.path.basename(f)))
                for f in self.map_names
            ]

        # check that all mask files exist on disk
        for f in mask_files:
            if not os.path.exists(f):
                raise OSError("Missing mask file {}".format(f))
        self.log("Found {} masks in {}".format(len(mask_files), mask_root), "info")
        self.log("Mask files: {}".format(mask_files), "debug")

        return dict(mask_type=mask_type, mask_root=mask_root, mask_files=mask_files)

    def _get_sim_files(
        self,
        name,
        ctype=None,
        subset="*",
        root=None,
        sim_data=False,
        index=None,
    ):
        """
        Convenience function for finding all matching sims per map.  Used
        internally in ``get_files`` for selecting appropriate subsets of
        simulation files.

        Arguments
        ---------
        name : str
            Type of simulation files to collect (signal, noise, foreground,
            template)
        ctype : str
            The type value associated with ``name``.  If supplied, set ``root``
            to ``<name>_<ctype>``, with ``name`` truncated before the first
            underscore.
        subset : str
            Data subset to search for.  See ``get_files`` for details.
        root : str
            Root of the simulation file tree, relative to data_root.  If not
            supplied and ``ctype`` is None, the returned set of variables are
            set to null values.
        sim_data : bool
            If True, build a list of files associated with the
            ``signal_type_sim``, ``noise_type_sim``, etc options to
            ``get_files()``, including the appropriate attribute names.  The
            value of ``subset`` is ignored.  If False, ensure that the same
            number of maps is selected for each map tag.
        index : bool
            If set, return the list of files corresponding to the given
            index.  Ignored if ``sim_data`` is False.

        Returns
        -------
        opts : dict
            Dictionary of sim file options for use with ``get_masked_sims()`` or
            ``get_masked_data()``.
        """
        out = {}
        num_files = None

        if sim_data:
            subset = "*"
            suffix = "_sim"
            match_count = False
        else:
            suffix = ""
            match_count = True
            index = None

        if root is None:
            out["{}_type{}".format(name, suffix)] = ctype
        if not sim_data:
            out["{}_subset".format(name.split("_")[0])] = subset

        if ctype is not None:
            root = "{}_{}".format(name.split("_")[0], ctype)

        for suff in ["", "2"] if self.null_run else [""]:

            if root is None:
                out.update(
                    {
                        "{}_root{}{}".format(name, suffix, suff): None,
                        "{}_files{}{}".format(name, suffix, suff): None,
                        "num_{}{}".format(name, suffix): 0,
                    }
                )
                continue

            data_root = getattr(self, "data_root{}".format(suff))
            map_files = getattr(self, "map_names{}".format(suff))

            root1 = os.path.join(data_root, root)
            all_files = []
            for f in map_files:
                files = sorted(
                    glob.glob(
                        os.path.join(
                            root1, f.replace(".fits", "_{}.fits".format(subset))
                        )
                    )
                )
                nfiles = len(files)
                if not nfiles:
                    raise OSError("Missing {} sims for {}".format(name, f))
                if num_files is None:
                    num_files = out.get("num_{}{}".format(name, suffix), nfiles)
                if num_files != nfiles:
                    if match_count:
                        raise OSError(
                            "Found {} {} sims for map {}, expected {}".format(
                                nfiles, name, f, num_files
                            )
                        )
                    elif nfiles < num_files:
                        num_files = nfiles

                if index is not None:
                    all_files.append(files[index])
                else:
                    all_files.append(files)

            self.log("Found {} {} sims in {}".format(num_files, name, root1), "info")

            if index is None:
                all_files = np.asarray([f[:num_files] for f in all_files])
                self.log(
                    "First {} sim files: {}".format(name, all_files[:, 0].tolist()),
                    "debug",
                )
            else:
                self.log("Selected {} files: {}".format(name, all_files), "debug")

            out.update(
                {
                    "{}_root{}{}".format(name, suffix, suff): root1,
                    "{}_files{}{}".format(name, suffix, suff): all_files,
                    "num_{}{}".format(name, suffix): num_files,
                }
            )

        return out

    def _get_template_files(self, name, ctype=None, suffix=""):
        """
        Convenience function for finding matching template maps per map,
        to be used for template subtraction in ``get_masked_data()``.
        Used internally in ``get_files()``.

        Arguments
        ---------
        name : str
            Type of simulation files to collect (signal, noise, foreground,
            template)
        ctype : str
            The type value associated with ``name``.  If not supplied, the
            returned set of variables are set to null values.
        suffix : str
            Suffix to apply to variable names, e.g. ``"_sim"``.

        Returns
        -------
        opts : dict
            Dictionary of template file options for use with
            ``get_masked_data()``.
        """
        out = {}
        nfiles = None

        root = None
        out["{}_type{}".format(name, suffix)] = ctype
        if ctype is not None:
            root = "{}_{}".format(name.replace("template", "templates"), ctype)

        for group in ["1", "2"]:
            suffix1 = suffix + (group if group == "2" else "")

            if root is None:
                out.update(
                    {
                        "{}_root{}".format(name, suffix1): None,
                        "{}_files{}".format(name, suffix1): None,
                        "num_{}{}".format(name, suffix): 0,
                    }
                )
                continue

            root1 = os.path.join(self.data_root, root, "template{}".format(group))
            files = []
            for f in self.map_names:
                # single template per map
                tf = os.path.join(root1, f)
                if os.path.exists(tf):
                    files.append(tf)
                    continue

                # ensemble of templates per map
                tf = sorted(glob.glob(tf.replace(".fits", "_*.fits")))
                nfiles1 = len(tf)
                if not nfiles1:
                    raise OSError(
                        "Missing temp{} {} files for {}".format(group, name, f)
                    )
                if nfiles is None:
                    nfiles = out.get("num_{}{}".format(name, suffix), nfiles1)
                if nfiles1 != nfiles:
                    raise OSError(
                        "Wrong number of {} sims. Found {} files, expected {}.".format(
                            name, nfiles1, nfiles
                        )
                    )

                files.append(tf)

            files = np.asarray(files)
            if nfiles is None:
                nfiles = len(files)
            if files.shape[-1] != nfiles:
                raise OSError(
                    "Wrong number of {} files. Found {} files, expected {}.".format(
                        name, files.shape[-1], nfiles
                    )
                )

            self.log("Found {} templates in {}".format(nfiles, root1), "info")
            out.update(
                {
                    "{}_root{}".format(name, suffix1): root1,
                    "{}_files{}".format(name, suffix1): files,
                    "num_{}{}".format(name, suffix): nfiles,
                }
            )

        return out

    def _get_reference_files(self, ctype=None):
        """
        Convenience function for finding all reference files per map, to be used
        for reference signal subtraction for null tests in ``get_masked_data()``.
        Used internally in ``get_files()``.

        Arguments
        ---------
        ctype : str
           The reference type to use.  If not supplied, the returned set of
           variables are set to null values.

        Returns
        -------
        opts : dict
            Dictionary of reference file options for use with
            ``get_masked_data()``.
        """
        if ctype is None:
            return {
                "reference_type": None,
                "reference_root": None,
                "reference_files": None,
                "num_reference": 0,
            }

        ref_root = {}
        ref_files = {}
        num_ref = len(self.map_names)
        root = "reference_{}".format(ctype)

        for null_split in ["a", "b"]:
            data_root = self.data_root if null_split == "a" else self.data_root2
            map_names = self.map_names if null_split == "a" else self.map_names2

            for group in ["1", "2"]:
                group1 = "ref{}{}".format(group, null_split)
                root1 = os.path.join(data_root, root, "reference{}".format(group))

                files1 = np.asarray([os.path.join(root1, f) for f in map_names])
                for f in files1:
                    if not os.path.exists(f):
                        raise OSError("Missing ref{} map {}".format(group, f))

                ref_root[group1] = root1
                ref_files[group1] = files1

                self.log(
                    "Found {} reference maps in {}".format(num_ref, ref_root[group1]),
                    "info",
                )
                self.log("Reference files: {}".format(files1), "debug")

        return {
            "reference_type": ctype,
            "reference_root": ref_root,
            "reference_files": ref_files,
            "num_reference": num_ref,
        }

    def get_files(
        self,
        data_root,
        data_subset="full/*0",
        data_root2=None,
        data_subset2=None,
    ):
        """
        Find all files for the given data root and subset.  Finds all files with
        data_type ``'raw'`` that match the subset criterion and stores their
        file tag and frequency information.  Subsequent data selection for
        signal/noise/foreground/template components or other data types should
        match the set of file tags found here. This method is called at the
        ``'files'`` checkpoint.

        The expected data structure is::

            <data_root>
                -> data_<data_type>
                    -> full
                        -> map_<tag>.fits
                        ...
                    -> 1of4 (same filenames as full)
                    -> 2of4 ('')
                    -> 3of4 ('')
                    -> 4of4 ('')
                -> signal_<signal_type>
                   -> spec_signal_<signal_type>.dat
                   -> full
                      -> map_<tag>_####.fits
                      ...
                   -> 1of4 (same filenames as full)
                   -> 2of4 (same filenames as full)
                   -> 3of4 (same filenames as full)
                   -> 4of4 (same filenames as full)
                -> noise_<noise_type> (same filenames as signal_<signal_type>)
                -> masks_<mask_type>
                    -> mask_map_<tag>.fits
                    ...
                -> foreground_<foreground_type_sim>
                    (same filenames as signal_<signal_type>)
                -> templates_<template_type>
                   -> template1
                      (same filenames as data_<data_type>)
                   -> template2
                      (same filenames as data_<data_type>)
                -> reference_<reference_type>
                   -> reference1
                      (same filenames as data_<data_type>)
                   -> reference2
                      (same filenames as data_<data_type>)
            <data_root2> (If provided, same structure as data_root)
                ...

        See ``get_mask_weights``, ``get_masked_data``, ``get_masked_sims``
        for documentation of the various file types.

        Arguments
        ---------
        data_root : string
            Top level path containing subdirectories for data, signal sims,
            noise sims, and masks.
        data_subset : string
            Subset of maps to use for spectrum estimation.  This should be
            a string that is parseable using ``glob`` on the path
            ``data_<data_type>/<data_subset>.fits``.  For example,
            ``'full/*0'`` will expand to read in the 150 GHz and 90GHz maps.
            Maps are then sorted in alphabetical order, and identified
            by their file tag, where each filename is ``map_<tag>.fits``.
        data_root2, data_subset2 : string
            The root and subset for a second set of data.  If either of these is
            keywords is supplied, then the two data sets are treated as two
            halves of a null test.  In this case, XFaster computes the sum and
            difference spectra for each map tag in order to estimate a null
            spectrum.

        Returns
        -------
        file_settings : dict
            A dictionary of file settings used throughout the run.
            These are stored in full as ``<output_root>/files_<output_tag>.npz``,
            and a subset are added to the run configuration file
            ``<output_root>/config_<output_tag>.txt``.
        """

        null_run = data_root2 is not None or data_subset2 is not None

        opts = dict(data_subset=data_subset, null_run=null_run)
        save_attrs = [
            "data_root",
            "data_subset",
            "null_run",
            "num_maps",
            "map_names",
            "map_tags",
            "map_pairs",
            "map_tags_orig",
            "map_pairs_orig",
            "map_freqs",
        ]
        if null_run:
            opts.update(data_subset2=data_subset2)
            save_attrs += ["data_root2", "data_subset2", "map_names2"]

        save_name = "files"
        ret = self.load_data(
            save_name, "files", fields=save_attrs, to_attrs=True, value_ref=opts
        )

        if ret is not None:
            return ret

        ret = self._get_data_files(
            data_root=data_root,
            data_subset=data_subset,
            data_root2=data_root2,
            data_subset2=data_subset2,
            config=True,
        )
        for k in save_attrs:
            setattr(self, k, ret[k])

        return self.save_data(save_name, from_attrs=save_attrs)

    def get_map(self, filename, check_nside=True, cache=False, **kwargs):
        """
        Load an input map from file or from an internal cache.  Maps are
        checked to make sure they all have a consistent size, and optionally
        cached to limit disk I/O.

        Arguments
        ---------
        filename : string
            Path to file on disk.
        check_nside : bool
            If True (default), make sure that all maps have the same ``nside``,
            and that it satisfies ``lmax <= 4 * nside``.
        cache : bool
            If True, cache the map in memory to avoid rereading from disk.
            Use this for maps that are used multiple times by the algoritm
            (e.g. masks).

        Any remaining arguments are passed to ``healpy.read_map``.

        Returns
        -------
        map : array_like
            2D map array containing 1 (T) or 3 (T/Q/U) maps.
            If the XFaster class was initialized with ``pol = True``, this
            returns a 2D array of T/Q/U maps from the file. Otherwise a
            (1, npix) array is returned containing only the T map.
        """
        import healpy as hp

        # initialize map cache
        if not hasattr(self, "_map_cache"):
            self._map_cache = dict()
        if kwargs.pop("reset", False):
            self._map_cache.pop(filename, None)

        # return a copy from cache if found
        if cache and filename in self._map_cache:
            return np.copy(self._map_cache[filename])

        kwargs.setdefault("field", [0, 1, 2] if self.pol else [0])
        kwargs.setdefault("dtype", None)

        self.log("Reading map from {}".format(filename), "all")
        m = np.atleast_2d(hp.read_map(filename, **kwargs))
        m[hp.mask_bad(m)] = 0
        m[np.isnan(m)] = 0

        if check_nside:

            if not hasattr(self, "nside"):
                self.nside = None

            # check nside
            nside = hp.get_nside(m)
            if self.nside is None:
                self.nside = nside
            else:
                if nside != self.nside:
                    raise ValueError(
                        "Input map {} has nside {} expected {}".format(
                            filename, nside, self.nside
                        )
                    )

            # check npix
            npix = hp.nside2npix(nside)
            if getattr(self, "npix", None) is None:
                self.npix = npix
            else:
                if npix != self.npix:
                    raise ValueError(
                        "Input map {} has npix {} expected {}".format(
                            filename, npix, self.npix
                        )
                    )

            # check lmax
            if self.lmax is None:
                self.lmax = 4 * self.nside
            elif self.lmax > 4 * self.nside:
                self.warn(
                    "lmax {} may be too large for nside {}".format(
                        self.lmax, self.nside
                    )
                )

        if cache:
            self._map_cache[filename] = m
        return m

    def get_mask(self, filename, cache=True, check_lims=True, **kwargs):
        """
        Load an input mask from file or from an internal cache.
        See ``XFaster.get_map`` for details.

        Arguments
        ---------
        filename : string
            Path to mask file on disk.
        cache : bool
            This option defaults to True, since masks are typically used
            for all data and sims for a given map tag.
        check_lims : bool
            If True, values in the mask outside of [0,1] are fixed to
            these limits.

        Any remaining arguments are passed to ``XFaster.get_map``.

        Returns
        -------
        mask : array_like
            2D array containing 1 (T) or 2 (T/P) maps;  If the XFaster class
            was initialized with ``pol_mask = True``, this returns a 2D array
            containing both T and P masks.  Otherwise, a (1, npix) is
            returned containing only the T map.
        """

        fields_mask = [0, 1] if self.pol_mask else [0]
        kwargs.setdefault("field", fields_mask)
        m = self.get_map(filename, cache=cache, **kwargs)
        if check_lims:
            m[m < 0] = 0
            m[m > 1] = 1
        return m

    def get_filename(
        self,
        name,
        ext=".npz",
        map_tag=None,
        iter_index=None,
        extra_tag=None,
        data_opts=False,
        bp_opts=False,
    ):
        """
        Define a standard output file path to read or write.

        Arguments
        ---------
        name : string
            String name of output type.  E.g. 'data_xcorr' for data
            cross-correlation spectra.
            If an output tag is set, the name is appended with
            '_<output_tag>'.
        ext : string
            File extension.  The default ('.npz') is used for storing
            output data dictionaries.
        map_tag : string
            If supplied, the name is appended with '_map_<map_tag>'.
            Use this argument when storing output data in a loop over
            input maps.
        iter_index : int
            If supplied, the name is appended with '_iter<iter_index>'.
        extra_tag : string
            If supplied the extra tag is appended to the name as is.
        data_opts : bool
            If True, the output filename is constructed by checking the
            following list of options used in constructing data cross-spectra:
            ensemble_mean, ensemble_median, sim_index, sim_type, data_type,
            template_type, reference_type.
        bp_opts : bool
            If True, also check the following attributes (in addition to those
            checked if ``data_opts`` is True): weighted_bins, return_cls.

        Returns
        -------
        filename : string
            Output filename as ``<output_root>/<name><ext>``, where
            <name> is constructed from the above set of options.
        """
        if self.output_root is None:
            return None

        name = [name]
        if data_opts or bp_opts:
            if getattr(self, "ensemble_mean", False):
                name += ["sim_mean"]
            elif getattr(self, "ensemble_median", False):
                name += ["sim_median"]
            else:
                if getattr(self, "sim_type", None) is not None:
                    name += ["sim"]
                    for comp in ["signal", "noise", "foreground", "template"]:
                        if comp not in self.sim_type:
                            continue
                        name += [self.sim_type[comp]]
                        if comp not in self.sim_index:
                            continue
                        name += ["{}{:04d}".format(comp[0], self.sim_index[comp])]
                        if comp == "signal" and "tensor" in self.sim_index:
                            name += ["t{:04d}".format(self.sim_index["tensor"])]
                elif self.data_type != "raw":
                    name += [self.data_type]
                if getattr(self, "template_type", None) is not None:
                    name += ["clean", self.template_type]
                if getattr(self, "reference_type", None) is not None:
                    name += ["ref", self.reference_type]
        if bp_opts:
            if self.weighted_bins:
                name += ["wbins"]
            if getattr(self, "return_cls", False):
                name += ["cl"]

        if map_tag is not None:
            name += ["map", map_tag]
        if iter_index is not None:
            name += ["iter{:03d}".format(iter_index)]
        if extra_tag is not None:
            name += [extra_tag]
        if self.output_tag:
            name += [self.output_tag]

        name = "_".join(name)
        if not ext.startswith("."):
            ext = ".{}".format(ext)
        return os.path.join(self.output_root, "{}{}".format(name, ext))

    def force_rerun_children(self, checkpoint):
        """Trigger rerunning steps that depend on this checkpoint."""
        for step in self.checkpoint_tree.get(checkpoint, []):
            if step not in self.checkpoints:
                raise ValueError(
                    "Invalid checkpoint {}, must be one of {}".format(
                        step, self.checkpoints
                    )
                )
            self.force_rerun[step] = True

    def load_data(
        self,
        name,
        checkpoint,
        fields=None,
        to_attrs=True,
        shape=None,
        shape_ref=None,
        alt_name=None,
        value_ref=None,
        optional=None,
        file_fields=None,
        **file_opts
    ):
        """
        Load xfaster data from an output.npz file on disk.

        This method is called throughout the code at various checkpoints.  If
        the data exist on disk, they are loaded and returned.  If the data are
        missing or otherwise incompatible, they are recomputed by the corresponding
        calling method, and trigger all subsequent data to also be recomputed.  Data
        handling is described in the ``Notes`` section for methods that use this
        functionality.

        Arguments
        ---------
        name : string
            The name of the data set.  The filename is contructed from this
            as ``<output_root>/<name>_<output_tag>.npz``.  If the file is not
            found then the data are recomputed.
        checkpoint : string
            The name of the checkpoint to which this dataset applies.
            If XFaster is initialized at this checkpoint, or if any of the
            file checks enabled with the following options fails, all
            quantities from this point forward are recomputed.
        fields : list of strings
            List of fields that should be present in the data file.
            If any are not found, the entire dataset and all subsequent
            step are recomputed.
        to_attrs : bool or list of bools or strings
            If True, all items in ``fields`` are stored as attributes of the
            parent object. If A list of booleans, must have the same length
            as ``fields``; any field for which this list item is True is then
            stored as an attribute of the object.  If any list item is a string,
            then the corresponding field is stored as an attribute with this
            new name.
        shape : tuple of ints
            If set, the field specified by ``shape_ref`` is checked to have this
            shape.  If this check fails, then all data are recomputed.
        shape_ref : string
            The reference field whose shape is checked against ``shape``.
            If None and ``shape`` is set, use the first field in ``fields``.
        alt_name : string
            Alternative to ``name`` argument that will be read if file matching
            ``name`` doesn't exist.
        value_ref : dict
            Dictionary of reference values that is checked if simply loading a
            file from disk instead of recomputing-- forces rerun of checkpoints
            if loaded dictionary differs from value_ref.
        optional : list of strings
            Fields that, if missing from the data loaded from disk, will not
            trigger force rerunning of any checkpoints.
        file_fields : list of strings
            Fields that, if missing from the data loaded from disk, will be
            searched for in a separate ``files`` checkpoint file.  This is for
            backwards compatibility.

        Remaining options are passed to ``get_filename`` for constructing the
        output file path.

        Returns
        -------
        data : dict
            If all checks above succeed, the requested data are returned.
            If any tests fail, None is returned, and all subsequent calls
            to ``load_data`` also return None to trigger recomputing all data
            that may depend on this dataset.
            The output dictionary has the additional key 'output_file' which
            is set to the path to the data file on disk.
        """

        # checkpointing
        if checkpoint not in self.checkpoints:
            raise ValueError(
                "Invalid checkpoint {}, must be one of {}".format(
                    checkpoint, self.checkpoints
                )
            )

        if self.checkpoint == checkpoint:
            self.force_rerun[checkpoint] = True
        if self.force_rerun[checkpoint]:
            return self.force_rerun_children(checkpoint)

        use_alt = False
        output_file = self.get_filename(name, ext=".npz", **file_opts)
        if not output_file:
            return self.force_rerun_children(checkpoint)
        errmsg = "Error loading {}".format(output_file)
        if not os.path.exists(output_file):
            self.warn("{}: File not found".format(errmsg))
            if alt_name is not None:
                output_file = self.get_filename(alt_name, ext=".npz", **file_opts)
                errmsg = "Error loading {}".format(output_file)
                if not os.path.exists(output_file):
                    self.warn("{}: Alternate file not found".format(errmsg))
                    return self.force_rerun_children(checkpoint)
                else:
                    use_alt = True
            else:
                return self.force_rerun_children(checkpoint)

        try:
            data = pt.load_and_parse(output_file)
        except Exception as e:
            self.warn("{}: {}".format(errmsg, str(e)))
            return self.force_rerun_children(checkpoint)

        if fields is None:
            fields = list(data)

        if not isinstance(fields, list):
            fields = [fields]

        if file_fields is not None:
            file_data = None
            for k in file_fields:
                if k not in fields:
                    fields += [k]
                if k in data:
                    continue
                if file_data is None:
                    file_data = self.load_data("files", "files", to_attrs=False)
                if k in file_data:
                    data[k] = file_data[k]
            # complete file paths with correct data roots
            pt.fix_data_roots(
                data,
                mode="load",
                root=self.data_root,
                root2=getattr(self, "data_root2", None),
            )

        if shape_ref is not None and shape_ref not in fields:
            self.warn("{}: Field {} not found".format(errmsg, shape_ref))
            return self.force_rerun_children(checkpoint)

        if to_attrs is True or to_attrs is False:
            to_attrs = [to_attrs] * len(fields)

        if shape is not None and shape_ref is None:
            shape_ref = fields[0]

        if value_ref is not None:
            value_ref = copy.deepcopy(value_ref)

        ret = dict()
        for field, attr in zip(fields, to_attrs):
            if field not in data:
                if optional is not None and field in optional:
                    data[field] = None
                else:
                    self.warn("{}: Field {} not found".format(errmsg, field))
                    return self.force_rerun_children(checkpoint)
            v = pt.dict_to_arr(data[field])
            try:
                v.shape
            except AttributeError:
                v = np.asarray(v)
            if not v.shape:
                v = v.tolist()
            if shape_ref in [field, attr]:
                if v.shape != tuple(shape):
                    # check if it's just an extra dimension, ie (1,500) vs (500,)
                    passes = False
                    if tuple(shape)[0] == 1:
                        if v.shape == tuple(shape)[1:]:
                            passes = True
                    if not passes:
                        self.warn(
                            "{}: Field {} has shape {}, expected {}".format(
                                errmsg, shape_ref, v.shape, shape
                            )
                        )
                        return self.force_rerun_children(checkpoint)
            if value_ref is not None:
                for k in [field, attr]:
                    vref = value_ref.pop(k, "undef")
                    # turn it into an array if possible to compare
                    try:
                        vref = pt.dict_to_arr(vref)
                    except:
                        pass

                    if str(vref) != "undef" and np.any(v != vref):
                        self.warn(
                            "{}: Field {} has value {}, expected {}".format(
                                errmsg, k, v, vref
                            )
                        )
                        return self.force_rerun_children(checkpoint)
            ret[field] = data[field]
            if attr:
                key = field if attr is True else attr
                if key not in ["output_file", "data_version"]:
                    setattr(self, key, ret[field])

        if value_ref:
            self.warn("{}: Missing reference fields {}".format(errmsg, list(value_ref)))
            return self.force_rerun_children(checkpoint)

        self.log("Loaded input data from {}".format(output_file), "debug")
        if use_alt:
            # copy data to original file name
            ret.update(**file_opts)
            self.save_data(name, **ret)
            for k in file_opts:
                ret.pop(k)
        ret["output_file"] = output_file
        return ret

    def save_data(self, name, from_attrs=[], file_attrs=[], **data):
        """
        Save xfaster data to an output .npz file on disk.

        Arguments
        ---------
        name : string
            The name of the data set.  The filename is contructed from this as
            ``<output_root>/<name>_<output_tag>.npz``.  If the file is not found
            then the data are recomputed.
        from_attrs : list of strings
            A list of object attributes which should be stored in the data file.
        file_attrs : list of strings
            A list of object attributes which contain file paths. The data root
            is stripped from each path to ensure that only relative paths are
            written to disk, such that the data root may be changed in the
            future without triggering rerunning of the entire checkpoint.
        map_tag : str
            Load the dataset corresponding to this map.
            See ``get_filename`` for documentation.
        iter_index : int
            Load the dataset corresponding to this iteration index.
            See ``get_filename`` for documentation.
        bp_opts : bool
            Format output bandpowers file.  See ``get_filename`` for
            documentation.
        extra_tag : str
             Tag to add to file name.

        Any remaining keyword arguments are added to the output dictionary.

        Returns
        -------
        data : dict
            A copy of the data dictionary that was stored to disk.
            The output dictionary has the additional key 'output_file' which
            is set to the path to the data file on disk.
        """
        data["data_version"] = self.data_version

        file_opts = {}
        for opt in ["map_tag", "iter_index", "data_opts", "bp_opts", "extra_tag"]:
            if opt in data:
                file_opts[opt] = data.pop(opt)

        output_file = self.get_filename(name, ext=".npz", **file_opts)
        if not output_file:
            return

        for attr in from_attrs:
            if hasattr(self, attr):
                data[attr] = getattr(self, attr)

        if len(file_attrs):
            data = data.copy()
            file_data = {k: copy.deepcopy(data[k]) for k in file_attrs}
            # strip data roots for storing to disk
            pt.fix_data_roots(
                file_data,
                mode="save",
                root=self.data_root,
                root2=getattr(self, "data_root2", None),
            )
            data.update(file_data)

        pt.save(output_file, **data)
        self.log("Saved output data to {}".format(output_file), "debug")
        data["output_file"] = output_file
        return data

    def save_config(self, cfg):
        """
        Save a configuration file for the current run on disk.
        This method is used by ``xfaster_run`` to store the config
        in ``<output_root>/config_<output_tag>.txt``.

        Arguments
        ---------
        cfg : XFasterConfig or RawConfigParser object
            Config object containing all relevant arguments to save to disk.

        Returns
        -------
        filename : str
            Name of the config file saved to disk.
        """
        filename = self.get_filename("config", ext=".txt")
        if filename is None:
            return

        if not isinstance(cfg, XFasterConfig):
            cfg = XFasterConfig(cfg)

        try:
            creator = os.getlogin()
        except OSError:
            creator = "unknown"
        with open(filename, "w") as f:
            f.write(
                "# Created by {} on {:%Y-%m-%d %H:%M:%S}\n\n".format(
                    creator, datetime.datetime.now()
                )
            )
            cfg.write(f)

        return filename

    def apply_mask(self, m, mask):
        """
        Apply the input mask to the data map, in place.

        If the map is polarized, the appropriate mask is applied
        to the polarization data, depending on whether the mask
        is also polarized.

        Arguments
        ---------
        m : array_like
            Input map (T/Q/U if polarized, T-only if not)
            This array is modified in place.
        mask : array_like
            Mask to apply (T/P if polarized, T-only if not)

        Returns
        -------
        m : array_like
            Input map multiplied by mask.
        """

        m[0] *= mask[0]
        if self.pol:
            m[1:] *= (mask[1] if self.pol_mask else mask[0])[None, :]

        return m

    def map2alm(self, m, pol=None):
        """
        Wrapper for healpy.map2alm.

        Arguments
        ---------
        m : array_like
            Masked input map for which spherical harmonic alms are
            computed.
        pol : bool
            If None, this is set using the value with which the object
            was initialized.

        Returns
        -------
        alms : array_like
            Alms for the input map, computed using the equivalent of
            ``healpy.map2alm (m, lmax, pol=self.pol, use_weights=True)``.
        """
        import healpy as hp

        opts = dict(pol=self.pol if pol is None else pol)
        if self.alm_pixel_weights:
            if self.lmax > 1.5 * self.nside:
                raise RuntimeError(
                    "Cannot use pixel weights for map2alm, lmax {} is > "
                    "1.5 * nside for nside={}".format(self.lmax, self.nside)
                )
            opts.update(use_pixel_weights=True)
        else:
            opts.update(use_weights=True)
            if self.alm_iter is not None:
                opts.update(iter=self.alm_iter)

        return np.asarray(hp.map2alm(m, self.lmax, **opts))

    def alm2cl(self, m1, m2=None, lmin=2, lmax=None, symmetric=True):
        """
        Wrapper for healpy.alm2cl.

        Arguments
        ---------
        m1 : array_like
            Masked alms for map1
        m2 : array_like
            Masked alms for map2
        lmin : int, default: 2
            The minimum ell bin to include in the output Cls.  All ell
            bins below this are nulled out.
        lmax : int
            The maximum ell bin to compute.  If None, this is set to the
            lmax value with which the class was initialized.
        symmetric : bool, default: True
            If True, the average cross spectrum of (m1-x-m2 + m2-x-m1) / 2.
            is computed.

        Returns
        -------
        cls : array_like
            Cross-spectrum of m1-x-m2.
        """
        import healpy as hp

        if lmax is None:
            lmax = self.lmax
        cls = np.asarray(hp.alm2cl(m1, alms2=m2, lmax=lmax))
        if symmetric:
            cls_T = np.asarray(hp.alm2cl(m2, alms2=m1, lmax=lmax))
            cls = (cls + cls_T) / 2.0
        if lmin:
            cls[..., :lmin] = 0
        return np.atleast_2d(cls)

    def get_mask_weights(
        self,
        mask_type="rectangle",
        apply_gcorr=False,
        reload_gcorr=False,
        gcorr_file=None,
    ):
        """
        Compute cross spectra of the masks for each data map.

        Mode counting matrices are also computed and stored for each mask.

        Arguments
        ---------
        mask_type : string
            The variant of mask to use, e.g. 'rectangle', etc.
            We assume a mask per file tag in the mask_<mask_type> folder,
            corresponding to the files in data.
        apply_gcorr : bool
            If True, a correction factor is applied to the g (mode counting)
            matrix.  The correction factor should have been pre-computed for
            each map tag using independent scripts in the code package.
        reload_gcorr : bool
            If True, reload the gcorr file from the masks directory. Useful when
            iteratively solving for the correction terms.
        gcorr_file : str
            If not None, path to gcorr file. Otherwise, use file labeled
            mask_map_<tag>_gcorr.npz in mask directory for signal, or
            mask_map_<tag>_gcorr_null.npz for nulls.

        Notes
        -----
        This method is called at the 'masks' checkpoint, loads or saves a
        data dictionary with the following keys:

            wls:
                mask1-x-mask2 mask cross spectra for every mask pair
            fsky, w1, w2, w4:
                sky fraction and weighted modes per mask product
            gmat:
                mode-counting matrix, computed from ``g = fsky * w2 ** 2 / w4``
        """

        num_maps = self.num_maps
        mask_shape = self.mask_shape
        save_attrs = ["wls", "fsky", "w1", "w2", "w4", "gmat", "nside", "npix", "gcorr"]
        file_attrs = ["mask_type", "mask_root", "mask_files"]
        save_attrs += file_attrs
        save_name = "masks_xcorr"

        opts = {"mask_type": mask_type}
        if self.alm_pixel_weights:
            opts["alm_pixel_weights"] = self.alm_pixel_weights
            save_attrs += ["alm_pixel_weights"]
        elif self.alm_iter is not None:
            opts["alm_iter"] = self.alm_iter
            save_attrs += ["alm_iter"]

        ret = self.load_data(
            save_name,
            "masks",
            fields=save_attrs,
            file_fields=file_attrs,
            optional=["gcorr"],
            to_attrs=True,
            shape=mask_shape,
            shape_ref="wls",
            value_ref=opts,
        )

        def process_gcorr(gcorr_file_in):
            if not hasattr(self, "gcorr"):
                self.gcorr = None
            if apply_gcorr and self.gcorr is None:
                self.gcorr = OrderedDict()

            for tag, mfile in zip(self.map_tags, self.mask_files):
                if not apply_gcorr:
                    continue

                if not reload_gcorr and tag in self.gcorr:
                    continue

                if gcorr_file_in is None:
                    if self.null_run:
                        gcorr_file = mfile.replace(".fits", "_gcorr_null.npz")
                    else:
                        gcorr_file = mfile.replace(".fits", "_gcorr.npz")
                else:
                    gcorr_file = gcorr_file_in

                if not os.path.exists(gcorr_file):
                    self.warn("G correction file {} not found".format(gcorr_file))
                    continue

                gdata = pt.load_and_parse(gcorr_file)
                gcorr = gdata["gcorr"]
                for k, g in gcorr.items():
                    # check bins match g
                    bd0 = self.bin_def["cmb_{}".format(k)]
                    bd = gdata["bin_def"]["cmb_{}".format(k)]
                    if len(bd0) < len(bd):
                        bd = bd[: len(bd0)]
                        gcorr[k] = g[: len(bd)]
                    if not np.all(bd0 == bd):
                        self.warn(
                            "G correction for map {} has incompatible bin def".format(
                                tag
                            )
                        )
                        break
                else:
                    self.log(
                        "Found g correction for map {}: {}".format(tag, gcorr), "debug"
                    )
                    self.gcorr[tag] = gcorr

            # compute ell-by-ell mode counting factor
            gmat_ell = OrderedDict()
            ell = np.arange(self.lmax + 1)
            ellfac = 2.0 * ell + 1.0

            for xname, (m0, m1) in self.map_pairs.items():
                gmat_ell[xname] = OrderedDict()
                if apply_gcorr:
                    gcorr0 = self.gcorr[m0]
                    gcorr1 = self.gcorr[m1]

                for spec in self.specs:
                    gmat_ell[xname][spec] = self.gmat[xname][spec] * ellfac
                    if apply_gcorr:
                        gcorr = np.sqrt(gcorr0[spec] * gcorr1[spec])
                        bd = self.bin_def["cmb_{}".format(spec)]
                        for gc, (start, stop) in zip(gcorr, bd):
                            gmat_ell[xname][spec][start:stop] *= gc

            self.apply_gcorr = apply_gcorr
            self.gmat_ell = gmat_ell

        if ret is not None:
            process_gcorr(gcorr_file)
            if apply_gcorr and (reload_gcorr or ret.get("gcorr", None) is None):
                return self.save_data(
                    save_name, from_attrs=save_attrs, file_attrs=file_attrs
                )
            ret["gcorr"] = self.gcorr
            return ret

        # find mask data files
        ret = self._get_mask_files(mask_type)
        for k, v in ret.items():
            setattr(self, k, v)

        # mask spectra
        wls = OrderedDict()

        pol_dim = 3 if self.pol else 1

        # moments
        fsky = OrderedDict()
        fsky_eff = OrderedDict()
        w1 = OrderedDict()
        w2 = OrderedDict()
        w4 = OrderedDict()
        gmat = OrderedDict()

        cache = dict()

        def process_index(idx):
            """
            A internal convenience function computes/loads alms from/to cache for each mask idx.
            """
            if idx in cache:
                return cache[idx]

            self.log("Computing Alms for mask {}/{}".format(idx + 1, num_maps), "all")

            mask = self.get_mask(self.mask_files[idx])
            mask_alms = self.map2alm(mask, False)

            cache[idx] = (mask_alms, mask)
            return cache[idx]

        spec_inds = {
            "tt": (0, 0),
            "ee": (1, 1),
            "bb": (2, 2),
            "te": (0, 1),
            "eb": (1, 2),
            "tb": (0, 2),
        }

        for xname, (idx, jdx) in pt.tag_pairs(self.map_tags, index=True).items():
            imask_alms, imask = process_index(idx)
            jmask_alms, jmask = process_index(jdx)

            self.log("Computing mask spectra {}x{}".format(idx, jdx), "debug")
            wls[xname] = self.alm2cl(imask_alms, jmask_alms, lmin=0)

            if self.pol_mask:
                # If there is a pol mask in addition to I, copy it to
                # U so no I masks is [I, Q, U] instead of [I, pol]
                imask = np.vstack([imask, imask[1]])
                jmask = np.vstack([jmask, jmask[1]])

            # calculate moments of cross masks
            # this is an array of shape (pol_dim, pol_dim, npix)
            # and contains all combinations of mask products
            mask = np.sqrt(np.einsum("i...,j...->ij...", imask, jmask))
            counts = [np.count_nonzero(x) for x in mask.reshape(-1, mask.shape[-1])]
            counts = np.array(counts).reshape(self.pol_dim, self.pol_dim).astype(float)

            # fsky is the fraction of pixels that are nonzero, independent
            # of weight
            fsky[xname] = counts / self.npix
            w1[xname] = np.sum(mask, axis=-1) / counts
            w2[xname] = np.sum(mask ** 2, axis=-1) / counts
            w4[xname] = np.sum(mask ** 4, axis=-1) / counts
            # effective fsky takes into account weights between 0 and 1
            fsky_eff[xname] = (
                fsky[xname]
                * w2[xname] ** 2
                / w4[xname]
                * (1.0 + 4.0 * fsky[xname])  # second order correction
            )

            # compute gmat as the average fsky_eff assuming symmetrically
            # computed cross spectra, e.g. TE = (T1 * E2 + T2 * E1) / 2
            gmat[xname] = OrderedDict()
            for spec in self.specs:
                si, sj = spec_inds[spec]
                f = (fsky_eff[xname][si, sj] + fsky_eff[xname][sj, si]) / 2.0
                gmat[xname][spec] = f

        if np.any(np.asarray([f for f in fsky.values()]) > 0.1):
            self.warn(
                "Some fsky are larger than 10% - second order "
                "correction may break down here: {}".format(fsky)
            )

        # store and return
        self.wls = wls
        self.fsky = fsky
        self.w1 = w1
        self.w2 = w2
        self.w4 = w4
        self.gmat = gmat

        process_gcorr(gcorr_file)

        self.log("Fsky: {}".format(self.fsky), "debug")
        self.log("Effective fsky: {}".format(fsky_eff), "debug")
        self.log("Mask moments 1: {}".format(self.w1), "debug")
        self.log("Mask moments 2: {}".format(self.w2), "debug")
        self.log("Mask moments 4: {}".format(self.w4), "debug")
        self.log("G matrix: {}".format(self.gmat), "debug")

        return self.save_data(save_name, from_attrs=save_attrs, file_attrs=file_attrs)

    def get_noise_residuals(self, filename):
        """
        Return a dictionary of ell-by-ell noise residual spectra from an output
        bandpowers file, to be applied to noise Alm's using ``healpy.almxfl``.
        """
        rls = OrderedDict()

        if not os.path.exists(filename):
            filename = os.path.join(self.output_root, filename)
        data = pt.load_and_parse(filename)

        for k, bd in data["bin_def"].items():
            if not k.startswith("res_"):
                continue
            _, specs, map_tag = k.split("_", 2)
            specs = [x + x for x in np.unique(list(specs))]

            res_qb = np.sqrt(1.0 + data["qb"][k])
            (bad,) = np.where(np.isnan(res_qb))
            if len(bad):
                self.warn("Unphysical residuals fit, nulling {} bins {}".format(k, bad))
                res_qb[bad] = 1.0
            rl = pt.expand_qb(res_qb, bd)

            if map_tag not in rls:
                rls[map_tag] = OrderedDict()
            for spec in specs:
                rls[map_tag][spec] = rl

        return rls

    def get_masked_data(
        self,
        data_type="raw",
        template_type=None,
        template_noise_type=None,
        template_alpha=None,
        template_specs=None,
        reference_type=None,
        ensemble_mean=False,
        ensemble_median=False,
        sim=False,
        components=["signal", "noise", "foreground"],
        index=None,
        signal_type_sim=None,
        r=None,
        noise_type_sim=None,
        qb_file=None,
        foreground_type_sim=None,
        template_type_sim=None,
        template_alpha_sim=None,
        save_sim=False,
        update_template=False,
    ):
        """
        Compute cross spectra of the data maps.

        If only one dataset is selected, spectra are computed for every
        combination of pairs of data maps. This results in N * (N + 1) / 2
        cross spectra for N maps. A unique mask is used for each input map.

        If two datasets are selected for a null test, then sum and difference
        cross-spectra are computed by summing and differencing the two datasets.
        A unique mask is used for each map in the first dataset, and the same
        mask is applied to the corresponding map in the second dataset, so that
        both halves are masked identically.

        If ``template_type`` and ``template_alpha`` is supplied, the values
        given are applied to an appropriate template, and the result is
        subtracted from the data alms with map tags in the dictionary.  Map alms
        are cached to speed up processing, if this method is called repeatedly
        with different values of ``template_alpha``.

        The remaining options handle subtraction of additional biases from the
        data, or constructing simulated datasets from sim ensembles.

        Arguments
        ---------
        data_type : string
            The type of data to use, default: "raw"
        template_type : string
            Tag for directory (templates_<template_type>) containing templates
            (e.g. a foreground model) to be scaled by a scalar value per map tag
            and subtracted from the data. The directory contains one template
            per map tag.
        template_noise_type : string
            Tag for directory containing template noise sims to be averaged and
            scaled similarly to the templates themselves.  These averaged sims
            are used to debias template cross spectra due to correlations in the
            way the noise ensembles are constructed.  Typically, this would be a
            noise model based on the Planck FFP10 ensemble for each half-mission
            foreground template.  If not supplied, this debiasing step is not
            performed.
        template_alpha : dict
            Dictionary of template scaling factors to apply to foreground
            templates to be subtracted from the data.  Keys should match
            original map tags in the data set.
        template_specs : list
            Which spectra to use for alpha in the likelihood.
        reference_type : string
            If supplied, subtract a reobserved reference signal from each data
            map.  The reference signal maps should be two datasets with
            uncorrelated noise, such as Planck half-mission maps.  This option
            is used for removing expected signal residuals from null tests.
        ensemble_mean : bool
            If True, use the mean of the ``signal_type`` and ``noise_type``
            ensembles, rather than using maps from the ``data_type`` directory
            or any other sim options.  This is useful for testing the behavior
            of the estimator and mapmaker independently of the data.
        ensemble_median : bool
            If True, use the median of the ``signal_type`` and ``noise_type``
            ensembles, rather than using maps from the ``data_type`` directory
            or any other sim options.  This is useful for testing the behavior
            of the estimator and mapmaker independently of the data.
        sim : bool
            If True, construct simulated data maps using the options below.
        components : list of strings
            List of components to include in the simulated data, of signal,
            noise, foreground or template.
        index : int or dict
            If supplied and ``sim`` is True, then simulated data maps are
            constructed from the appropriate index from the sim ensembles
            ``signal_type_sim``, ``noise_type_sim`` and/or
            ``foreground_type_sim``, rather than using maps from the
            ``data_type`` directory.  If an integer, then the same index is used
            for all ensembles.  Otherwise, should be a dictionary keyed by
            component (signal, tensor, noise, foreground).  Additionally, the key
            ``default`` can be used to indicate the index to use for components
            that are not explicitly enumerated in the dictionary.  If not
            supplied, and ``sim`` is True, index 0 is used for all ensembles.
        signal_type_sim : string
            The variant of signal sims to include in the simulated data maps.
            This enables having a different signal sim ensemble to use for
            simulated data than the ensemble from which the signal model
            component is computed.  If this is not supplied and ``r`` is not
            ``None``, then two signal types are searched: 'signal_r0' for a
            scalar component, and 'signal_r1tens' for a tensor component.  The
            two maps are linearly combined with a scalar r value to construct a
            signal map for a simulated dataset.
        r : float
            If supplied, the simulated signal maps are constructed by combining
            ``signal_scalar + r * signal_tensor``, where the scalar maps are
            stored in the directory ``signal_root_sim``, and the tensor maps are
            stored in the directory ``tensor_root_sim``.  A separate sim index
            is assumed for the ``signal`` (scalar) ensemble and the ``tensor``
            ensemble.
        noise_type_sim : string
            The variant of noise sims to include in the simulated data maps.
            This enables having a different noise sim ensemble to use for
            simulated data than the ensemble from which the noise is computed.
        qb_file : str
            If supplied and noise is included in ``components``, correct the
            simulated noise spectra using the noise residuals stored in this
            file.  Typically, this is the output of a separate data run used to
            determine the appropriate noise correction.  See
            ``get_noise_residuals`` for more details.
        foreground_type_sim : string
            The variant of foreground sims to include in the simulated data maps.
        template_type_sim : string
            Tag for directory containing foreground templates, to be scaled by a
            scalar value per map tag and added to the simulated data.  The
            directory contains one template per map tag.
        template_alpha_sim : dict
            Dictionary of template scaling factors to apply to foreground
            templates to be added to the simulated the data.  Keys should match
            original map tags in the data set.
        save_sim : bool
            If True and constructing a simulated dataset using any of the above
            sim options, write the simulated dataset to disk using an
            appropriate ``'data_xcorr.npz'`` filename.  If False, only
            non-simulated datasets are written to disk.
        update_template : bool
            If True, just apply the loaded template with updated
            ``template_alpha`` and ``template_spec`` parameters and return.

        Notes
        -----
        This method is called at the 'data' checkpoint, loads or saves a data
        dictionary with the following spectra:

            cls_data:
                map1-x-map2 cross spectra for every map pair. This contains the
                sum cross spectra if constructing a null test.
            cls_data_clean:
                template-subtracted spectra, if ``template_alpha`` is supplied.
            cls_template:
                template cross spectra necessary to rebuild the
                template-subtracted data when the ``template_alpha`` parameter
                is changed.
            cls_data_null:
                (map1a-map1b)-x-(map2a-map2b) difference cross spectra for every
                map pair, if computing a null test
            cls_ref, cls_ref_null:
                reference cross spectra, if ``reference_type`` is set.
        """
        import healpy as hp

        num_maps = self.num_maps
        data_shape = self.data_shape
        null_run = self.null_run

        if null_run:
            template_type = None
            template_noise_type = None
            template_type_sim = None
        else:
            reference_type = None

        if template_specs is None:
            template_specs = self.specs

        # ensure dictionary
        if (
            template_alpha is None
            or null_run
            or (update_template and self.template_type is None)
            or (not update_template and template_type is None)
        ):
            template_alpha = OrderedDict()
        else:
            # ensure tagged by original tags
            template_alpha = OrderedDict(
                [(k, v) for k, v in template_alpha.items() if k in self.map_tags_orig]
            )

        def apply_template():
            """
            Internal data processing function to have scaled foreground template
            subtracted from data map.
            """
            subtract_template_noise = hasattr(self, "cls_template_noise")
            cls_clean = getattr(self, "cls_data_clean", OrderedDict())

            for spec in set(self.specs) & set(template_specs):
                cls_clean[spec] = copy.deepcopy(self.cls_data[spec])
                if spec not in self.cls_template:
                    continue
                for xname, d in cls_clean[spec].items():
                    if xname not in self.cls_template[spec]:
                        continue
                    m0, m1 = self.map_pairs_orig[xname]
                    alphas = [template_alpha.get(m, None) for m in (m0, m1)]

                    t1, t2, t3 = self.cls_template[spec][xname]

                    if alphas[0] is not None:
                        d -= alphas[0] * t1
                    if alphas[1] is not None:
                        d -= alphas[1] * t2
                        if alphas[0] is not None:
                            d += alphas[0] * alphas[1] * t3
                            # subtract average template noise spectrum to debias
                            if subtract_template_noise:
                                n = self.cls_template_noise["temp1:temp2"][spec][xname]
                                d -= alphas[0] * alphas[1] * n

            self.cls_data_clean = cls_clean
            self.template_alpha = template_alpha

        # change template subtraction coefficients for pre-loaded data
        if update_template:
            apply_template()
            return

        # set sim attributes
        if ensemble_mean or ensemble_median or not sim:
            if ensemble_mean:
                cls_ens = self.cls_sim
                cls_ens_null = self.cls_sim_null if null_run else None
            elif ensemble_median:
                cls_ens = self.cls_med
                cls_ens_null = self.cls_med_null if null_run else None
            sim_type = components = index = r = qb_file = template_alpha_sim = None
        elif sim:
            if index is None:
                index = {}
            else:
                index = index.copy()
            default_index = index.pop("default", 0)
            sim_type = {}
            if (null_run or template_type_sim is None) and "template" in components:
                components.remove("template")
            for comp in ["signal", "noise", "foreground", "template"]:
                type_key = "{}_type_sim".format(comp)
                type_value = locals()[type_key]
                if comp not in components:
                    if comp == "signal":
                        r = None
                    elif comp == "noise":
                        qb_file = None
                    elif comp == "template":
                        template_alpha_sim = None
                    type_value = None
                    continue
                if comp == "signal" and r is not None:
                    type_value = "rp{:03d}".format(int(r * 1000))
                    index.setdefault("tensor", index.get("signal", default_index))
                if type_value is None and comp != "foreground":
                    v = getattr(self, type_key.replace("_sim", ""))
                    type_value = v
                if type_value is None:
                    continue
                sim_type[comp] = type_value
                if comp == "template":
                    if template_alpha_sim is None:
                        template_alpha_sim = template_alpha
                    else:
                        template_alpha_sim = OrderedDict(
                            [
                                (k, v)
                                for k, v in template_alpha_sim.items()
                                if k in self.map_tags_orig
                            ]
                        )
                else:
                    index.setdefault(comp, default_index)

        # Check for output data on disk
        opts = {"data_type": data_type}
        file_attrs = ["data_type", "map_root", "map_files"]
        save_attrs = ["cls_data", "nside"]
        template_fit = template_type is not None
        subtract_reference_signal = null_run and reference_type is not None
        subtract_template_noise = template_noise_type is not None
        if null_run:
            file_attrs += ["map_root2", "map_files2"]
            save_attrs += ["cls_data_null"]
            if reference_type is not None:
                opts["reference_type"] = reference_type
                file_attrs += [
                    "reference_type",
                    "reference_root",
                    "reference_files",
                    "num_reference",
                ]
                save_attrs += [
                    "cls_data_sub",
                    "cls_ref",
                    "cls_data_sub_null",
                    "cls_ref_null",
                ]
        elif template_type is not None:
            opts["template_type"] = template_type
            file_attrs += [
                "template_type",
                "template_root",
                "template_root2",
                "template_files",
                "template_files2",
                "num_template",
            ]
            save_attrs += ["cls_data_clean", "cls_template", "template_alpha"]
        else:
            subtract_template_noise = False

        if sim:
            for comp, comp_type in sim_type.items():
                k = "{}_type_sim".format(comp)
                opts[k] = comp_type
                file_attrs += [
                    "{}_type_sim".format(comp),
                    "{}_root_sim".format(comp),
                    "{}_files_sim".format(comp),
                    "num_{}_sim".format(comp),
                ]
                if null_run or comp == "template":
                    file_attrs += [
                        "{}_root_sim2".format(comp),
                        "{}_files_sim2".format(comp),
                    ]
                if comp == "template":
                    self.template_alpha_sim = template_alpha_sim
                    save_attrs += ["template_alpha_sim"]

        save_attrs += file_attrs

        # set attributes
        self.data_type = data_type
        self.ensemble_mean = ensemble_mean
        self.ensemble_median = ensemble_median
        self.sim_type = sim_type
        self.sim_index = index
        self.template_type = template_type
        self.reference_type = reference_type

        if sim and not save_sim:
            # rerun dependent steps if not saving sims to disk
            save_name = None
            self.force_rerun_children("sim_data")
        else:
            save_name = "sim_data" if sim else "data"
            ret = self.load_data(
                save_name,
                save_name,
                file_fields=file_attrs,
                fields=save_attrs,
                to_attrs=True,
                shape=data_shape,
                shape_ref="cls_data",
                value_ref=opts,
                data_opts=True,
                extra_tag="xcorr",
            )
            if ret is not None:
                if null_run or not template_fit:
                    return ret
                if np.all(template_alpha == self.template_alpha):
                    return ret
                if subtract_template_noise:
                    self.get_masked_template_noise(template_noise_type)
                apply_template()
                return ret

        # set up filenames
        ret = self._get_data_files(data_type=data_type)
        if template_type is not None:
            ret.update(self._get_template_files("template", template_type))
        if null_run and reference_type is not None:
            ret.update(self._get_reference_files(reference_type))

        if sim:
            for comp, comp_type in sim_type.items():
                if comp == "signal" and r is not None:
                    ret.update(
                        self._get_sim_files(
                            "signal",
                            root="signal_r0",
                            sim_data=True,
                            index=index["signal"],
                        )
                    )
                    ret.update(
                        self._get_sim_files(
                            "tensor",
                            root="signal_r1tens",
                            sim_data=True,
                            index=index["tensor"],
                        )
                    )
                elif comp == "template":
                    ret.update(self._get_template_files(comp, comp_type, suffix="_sim"))
                else:
                    ret.update(
                        self._get_sim_files(
                            comp, comp_type, sim_data=True, index=index[comp]
                        )
                    )
            for comp, idx in index.items():
                k = "{}_files".format(comp)
                if k in ret:
                    ret[k] = [f[idx] for f in ret[k]]
                k2 = k + "2"
                if k2 in ret:
                    ret[k2] = [f[idx] for f in ret[k2]]

        for k, v in ret.items():
            setattr(self, k, v)

        # map spectra
        cls = OrderedDict()
        cls_null = OrderedDict() if null_run else None

        # set up template subtraction
        cls_tmp = None
        if template_fit:
            cls_tmp = OrderedDict()
            template_files = list(zip(self.template_files, self.template_files2))
            if subtract_template_noise:
                self.get_masked_template_noise(template_noise_type)

        # set up reference subtraction
        cls_ref = None
        if subtract_reference_signal:
            cls_ref = OrderedDict()
            cls_ref_null = OrderedDict()
            reference_files_split = list(
                zip(
                    self.reference_files[x]
                    for x in ["ref1a", "ref2a", "ref1b", "ref2b"]
                )
            )

        # set up noise residuals
        rls = OrderedDict()
        if sim and qb_file is not None:
            rls = self.get_noise_residuals(qb_file)

        # convenience functions
        def get_map_file(attr, idx, suffix=""):
            files = getattr(self, "{}{}".format(attr, suffix))
            if isinstance(files[0], str):
                return files[idx]
            files = files[idx]
            # list of available file indices
            fidx = [int(os.path.splitext(x.rsplit("_")[-1])[0]) for x in files]
            # sim index for this component
            comp = attr.split("_")[0]
            sidx = index[comp]
            if sidx not in fidx:
                raise ValueError("Missing sim index {} in {}".format(sidx, attr))
            return files[fidx.index(sidx)]

        def get_map_alms(idx, suffix=""):
            mask = self.get_mask(self.mask_files[idx])
            if not sim:
                # data from disk
                f = get_map_file("map_files", idx, suffix)
                m = self.get_map(f)

            else:
                # simulated data constructed from individual components
                shape = (3, self.npix) if self.pol else (self.npix,)
                m = np.zeros(shape, dtype=float)

                if "signal" in sim_type:
                    f = get_map_file("signal_files_sim", idx, suffix)
                    m += self.get_map(f)
                    if r is not None:
                        f = get_map_file("tensor_files_sim", idx, suffix)
                        m += np.sqrt(r) * self.get_map(f)

                if "noise" in sim_type:
                    mn_alms = None
                    rls1 = rls.get(self.map_tags[idx], None)
                    f = get_map_file("noise_files_sim", idx, suffix)
                    mn = self.get_map(f)
                    if rls1 is None:
                        m += mn
                    else:
                        self.apply_mask(mn, mask)
                        mn_alms = self.map2alm(mn, self.pol)
                        for s, spec in enumerate(self.specs):
                            if spec in rls1:
                                mn_alms[s] = hp.almxfl(mn_alms[s], rls1[spec])
                    del mn

                if "foreground" in sim_type:
                    f = get_map_file("foreground_files_sim", idx, suffix)
                    m += self.get_map(f)

                if "template" in sim_type:
                    alpha = template_alpha_sim.get(self.map_tags_orig[idx], None)
                    if alpha is not None:
                        f = get_map_file("template_files_sim", idx)
                        m += alpha * self.get_map(f)

            self.apply_mask(m, mask)
            m_alms = self.map2alm(m, self.pol)

            if sim and "noise" in sim_type and mn_alms is not None:
                # add noise rescaled by residuals
                m_alms += mn_alms

            return m_alms

        cache = dict()

        def process_index(idx):
            """
            A internal convenience function computes/loads alms from/to cache for each data map idx,
            or data map pair for null tests.
            """
            if idx in cache:
                return cache[idx]

            self.log("Computing Alms for map {}/{}".format(idx + 1, num_maps), "all")

            mask = self.get_mask(self.mask_files[idx])
            m_alms = get_map_alms(idx)
            mn_alms = None

            if null_run:
                # sum and diff spectra for null tests
                # XXX should not take average but sum here if we want to
                # compare power with sum...
                m2_alms = get_map_alms(idx, "2")
                m_alms, mn_alms = (m_alms + m2_alms) / 2.0, (m_alms - m2_alms) / 2.0
                del m2_alms

                if subtract_reference_signal:
                    # cache raw data alms and reference alms together
                    mp1ref1, mp1ref2, mp2ref1, mp2ref2 = (
                        self.map2alm(self.apply_mask(self.get_map(f), mask), self.pol)
                        for f in reference_files_split[idx]
                    )
                    m_alms_ref1 = (mp1ref1 + mp2ref1) / 2.0
                    m_alms_ref2 = (mp1ref2 + mp2ref2) / 2.0
                    mn_alms_ref1 = (mp1ref1 - mp2ref1) / 2.0
                    mn_alms_ref2 = (mp1ref2 - mp2ref2) / 2.0
                    m_alms = (m_alms, m_alms_ref1, m_alms_ref2)
                    mn_alms = (mn_alms, mn_alms_ref1, mn_alms_ref2)

            elif template_fit:
                # cache raw data alms and template alms together
                m_alms = [m_alms]
                for tf in template_files[idx]:
                    self.log("Loading template from {}".format(tf), "debug")
                    mt_alms = self.map2alm(
                        self.apply_mask(self.get_map(tf), mask), self.pol
                    )
                    # null out T template
                    if self.pol:
                        mt_alms[0] *= 0
                    m_alms.append(mt_alms)
                m_alms = tuple(m_alms)

            cache[idx] = (m_alms, mn_alms)
            return cache[idx]

        map_pairs = pt.tag_pairs(self.map_tags, index=True)

        for xname, (idx, jdx) in map_pairs.items():

            if self.ensemble_mean or self.ensemble_median:
                for spec in self.specs:
                    cls_dict = cls.setdefault(spec, OrderedDict())
                    cls_dict[xname] = cls_ens[spec][xname]
                    if null_run:
                        cls_dict = cls_null.setdefault(spec, OrderedDict())
                        cls_dict[xname] = cls_ens_null[spec][xname]
                continue

            imap_alms, inull_alms = process_index(idx)
            jmap_alms, jnull_alms = process_index(jdx)

            self.log("Computing spectra {}x{}".format(idx + 1, jdx + 1), "debug")

            # store cross spectra
            if isinstance(imap_alms, tuple) and len(imap_alms) == 3:
                # raw map spectrum component
                cls1 = self.alm2cl(imap_alms[0], jmap_alms[0])

                # average template alms
                imt = (imap_alms[1] + imap_alms[2]) / 2.0
                jmt = (jmap_alms[1] + jmap_alms[2]) / 2.0

                # compute maximally symmetric cross spectra
                t1 = self.alm2cl(imt, jmap_alms[0])  # multiplies alpha_i
                t2 = self.alm2cl(imap_alms[0], jmt)  # multiplies alpha_j
                t3 = (
                    self.alm2cl(imap_alms[1], jmap_alms[2])
                    + self.alm2cl(imap_alms[2], jmap_alms[1])
                ) / 2.0  # multiplies alpha_i * alpha_j

                for s, spec in enumerate(self.specs):
                    # apply template to TE/TB but not TT
                    if not subtract_reference_signal and spec == "tt":
                        continue
                    cls_dict = cls_ref if subtract_reference_signal else cls_tmp
                    cls_dict = cls_dict.setdefault(spec, OrderedDict())
                    cls_dict[xname] = (t1[s], t2[s], t3[s])

                if null_run:
                    # do this again for the null maps
                    cls_null1 = self.alm2cl(inull_alms[0], jnull_alms[0])

                    # average template alms
                    imt = (inull_alms[1] + inull_alms[2]) / 2.0
                    jmt = (jnull_alms[1] + jnull_alms[2]) / 2.0

                    # compute maximally symmetric cross spectra
                    t1 = self.alm2cl(imt, jnull_alms[0])
                    t2 = self.alm2cl(inull_alms[0], jmt)
                    t3 = (
                        self.alm2cl(inull_alms[1], jnull_alms[2])
                        + self.alm2cl(inull_alms[2], jnull_alms[1])
                    ) / 2.0

                    for s, spec in enumerate(self.specs):
                        cls_dict = cls_ref_null.setdefault(spec, OrderedDict())
                        cls_dict[xname] = (t1[s], t2[s], t3[s])

            else:
                cls1 = self.alm2cl(imap_alms, jmap_alms)
                if null_run:
                    cls_null1 = self.alm2cl(inull_alms, jnull_alms)

            for s, spec in enumerate(self.specs):
                cls.setdefault(spec, OrderedDict())[xname] = cls1[s]
                if null_run:
                    cls_null.setdefault(spec, OrderedDict())[xname] = cls_null1[s]

        # store and return
        self.cls_data = cls
        self.cls_data_null = cls_null

        if template_fit:
            self.cls_template = cls_tmp
            apply_template()

        if subtract_reference_signal:
            self.cls_ref = cls_ref
            self.cls_ref_null = cls_ref_null

            cls_data_sub = copy.deepcopy(self.cls_data)
            cls_data_sub_null = copy.deepcopy(self.cls_data_null)

            for spec in self.specs:
                for xname, d in cls_data_sub[spec].items():
                    t1, t2, t3 = self.cls_ref[spec][xname]
                    d += -t1 - t2 + t3

                # do null specs
                for xname, d in cls_data_sub_null[spec].items():
                    t1, t2, t3 = self.cls_ref_null[spec][xname]
                    d += -t1 - t2 + t3

            self.cls_data_sub = cls_data_sub
            self.cls_data_sub_null = cls_data_sub_null

        if not sim or save_sim:
            return self.save_data(
                save_name,
                from_attrs=save_attrs,
                file_attrs=file_attrs,
                extra_tag="xcorr",
                data_opts=True,
            )

    def get_masked_sims(
        self,
        transfer=False,
        signal_type="synfast",
        signal_subset="*",
        noise_type=None,
        noise_subset="*",
        qb_file=None,
    ):
        """
        Compute average signal and noise spectra for a given ensemble of sim
        maps.  The same procedure that is used for computing data cross spectra
        is used for each realization in the sim ensemble, and only the average
        spectra for all realizations are stored.

        See ``get_masked_data`` for more details on how cross spectra are
        computed.

        Arguments
        ---------
        transfer : bool
            If True, use the 'sims_transfer' checkpoint, otherwise use the
            'sims' checkpoint.
        signal_type : string
            The variant of signal simulation to use, typically identified by the
            input spectrum model used to generate it, e.g 'synfast'.  If
            ``transfer`` is True, this signal ensemble is used for the transfer
            function calculation.  The directory may also contain the input Cl
            spectrum as a ``camb`` file, to be used for constructing the
            kernel-convolved signal model.
        signal_subset : string
            Subset of map tags to use for spectrum estimation for signal
            sims.  This should be a string that is parseable using ``glob``
            that is added onto the data_subset path to indicate which sims
            to use. For example, for all, use ``'*'``. For the first 300 sims,
            use ``'0[0-2]*'``.
        noise_type: string
            The variant of noise simulation to use, e.g. 'gaussian',
            'stationary', etc.  The directory should contain the same number
            of simulations for each map tag.
        noise_subset : string
            Subset of map tags to use for spectrum estimation for noise
            sims.  This should be a string that is parseable using ``glob``
            that is added onto the data_subset path to indicate which sims
            to use. For example, for all, use ``'*'``. For the first 300 sims,
            use ``'0[0-2]*'``.
        qb_file : string
            Pointer to a bandpowers.npz file in the output directory, used
            to correct the ensemble mean noise spectrum by the appropriate
            residual terms.  See ``get_noise_residuals`` for details.

        Notes
        -----
        This method is called at the 'sims' or 'sims_transfer' checkpoint, and
        loads or saves a data dictionary with the following keys:

            cls_signal, cls_signal_null:
                Mean signal spectra
            cls_noise, cls_noise_null:
                Mean noise spectra
            cls_sim, cls_sim_null:
                Mean signal+noise spectra
            cls_med, cls_med_null:
                Median signal+noise spectra
            cls_res, cls_res_null:
                NxN, SxN and NxS spectra for computing noise residuals

        For null tests, difference spectra of the two null halves are stored
        in the corresponding `*_null` keys, and summed spectra are stored
        in the normal keys.  Note that these differ from standard non-null
        spectra by a factor of 2.
        """
        import healpy as hp

        mask_files = self.mask_files
        map_tags = self.map_tags
        map_pairs = pt.tag_pairs(map_tags, index=True)
        num_maps = self.num_maps
        num_corr = self.num_corr
        data_shape = self.data_shape
        null_run = self.null_run

        opts = dict(signal_type=signal_type, signal_subset=signal_subset)
        file_attrs = [
            "signal_type",
            "signal_subset",
            "signal_root",
            "signal_files",
            "num_signal",
        ]
        save_attrs = ["cls_signal", "cls_sim", "cls_med"]
        if null_run:
            file_attrs += ["signal_root2", "signal_files2"]
            save_attrs += ["cls_signal_null", "cls_sim_null", "cls_med_null"]

        do_noise = noise_type is not None
        if do_noise:
            opts.update(noise_type=noise_type, noise_subset=noise_subset)
            if qb_file is not None:
                opts["qb_file"] = qb_file
            file_attrs += [
                "noise_type",
                "noise_subset",
                "noise_root",
                "noise_files",
                "num_noise",
            ]
            save_attrs += ["cls_noise", "cls_res"]
            if null_run:
                file_attrs += ["noise_root2", "noise_files2"]
                save_attrs += ["cls_noise_null", "cls_res_null"]

        save_attrs += file_attrs

        alt_name = None
        save_name = "sims_xcorr_{}".format(signal_type)
        if do_noise:
            alt_name = save_name
            save_name = "{}_{}".format(save_name, noise_type)
        cp = "sims_transfer" if transfer else "sims"

        # don't force rerun the signal sims if previously loaded sims are of the
        # same type
        if hasattr(self, "signal_type") and self.signal_type == signal_type:
            self.force_rerun[cp] = False

        ret = self.load_data(
            save_name,
            cp,
            file_fields=file_attrs,
            fields=save_attrs,
            to_attrs=True,
            shape=data_shape,
            shape_ref="cls_signal",
            value_ref=opts,
            alt_name=alt_name,
        )
        if ret is not None:
            if not do_noise:
                self.cls_noise = self.cls_noise_null = None
                ret["cls_noise"] = ret["cls_noise_null"] = None
            return ret

        # find sim ensemble files
        ret = {}
        ret.update(self._get_sim_files("signal", signal_type, signal_subset))
        if do_noise:
            ret.update(self._get_sim_files("noise", noise_type, noise_subset))
        for k, v in ret.items():
            setattr(self, k, v)

        signal_files = self.signal_files
        signal_files2 = self.signal_files2 if null_run else None
        num_signal = self.num_signal

        if do_noise:
            noise_files = self.noise_files
            noise_files2 = self.noise_files2 if null_run else None
            num_noise = self.num_noise
        else:
            num_noise = 0

        # process signal, noise, and S+N
        cls_sig = OrderedDict()
        cls_null_sig = OrderedDict() if null_run else None
        cls_noise = OrderedDict() if do_noise else None
        cls_null_noise = OrderedDict() if null_run and do_noise else None
        cls_tot = OrderedDict()
        cls_null_tot = OrderedDict() if null_run else None
        cls_med = OrderedDict()
        cls_null_med = OrderedDict() if null_run else None

        ### Noise iteration from res fit fields
        cls_res = OrderedDict() if do_noise else None
        cls_null_res = OrderedDict() if null_run and do_noise else None
        if do_noise:
            for k in ["nxn0", "nxn1", "sxn0", "sxn1", "nxs0", "nxs1"]:
                cls_res[k] = OrderedDict()
                if null_run:
                    cls_null_res[k] = OrderedDict()

        if num_noise != 0:
            nsim_min = min([num_signal, num_noise])
        else:
            nsim_min = num_signal
        nsim_max = max([num_signal, num_noise])
        cls_all = np.zeros(
            [nsim_max, len(map_pairs.items()), len(self.specs), self.lmax + 1]
        )
        if null_run:
            cls_all_null = np.zeros_like(cls_all)

        rls = OrderedDict()
        if do_noise and qb_file is not None:
            rls = self.get_noise_residuals(qb_file)

        # convenience functions
        def get_map_alms(filename, mask, rls=None):
            m = self.get_map(filename)
            self.apply_mask(m, mask)
            m_alms = self.map2alm(m, self.pol)

            if rls is not None:
                for s, spec in enumerate(self.specs):
                    if spec in rls:
                        m_alms[s] = hp.almxfl(m_alms[s], rls[spec])

            return m_alms

        def process_index(files, files2, idx, idx2=None, cache=None, rls=None):
            """
            Compute alms of masked input map
            """
            if cache is None:
                cache = {}

            if idx in cache:
                return cache[idx]

            filename = files[idx]
            if idx2 is None:
                self.log(
                    "Computing Alms for map {}/{}".format(idx + 1, num_maps), "all"
                )
            else:
                self.log(
                    "Computing Alms for sim {} of map {}/{}".format(
                        idx2, idx + 1, num_maps
                    ),
                    "all",
                )
                filename = filename[idx2]

            if rls is not None:
                rls = rls.get(map_tags[idx], None)

            mask = self.get_mask(mask_files[idx])
            m_alms = get_map_alms(filename, mask, rls=rls)
            mn_alms = None

            if null_run:
                # second null half
                filename2 = files2[idx]
                if idx2 is not None:
                    filename2 = filename2[idx2]

                m2_alms = get_map_alms(filename2, mask, rls=rls)
                m_alms, mn_alms = (m_alms + m2_alms) / 2.0, (m_alms - m2_alms) / 2.0

            cache[idx] = (m_alms, mn_alms)
            return cache[idx]

        sig_cache = dict()
        noise_cache = dict()

        for isim in range(nsim_max):
            sig_cache.clear()
            noise_cache.clear()
            for xind, (xname, (idx, jdx)) in enumerate(map_pairs.items()):
                self.log(
                    "Computing spectra {} for signal{} sim {}".format(
                        xname, "+noise" if do_noise else "", isim
                    ),
                    "debug",
                )
                if isim < num_signal:
                    simap_alms, sinull_alms = process_index(
                        signal_files, signal_files2, idx, isim, sig_cache
                    )
                    sjmap_alms, sjnull_alms = process_index(
                        signal_files, signal_files2, jdx, isim, sig_cache
                    )

                    cls1_sig = self.alm2cl(simap_alms, sjmap_alms)
                    if null_run:
                        cls_null1_sig = self.alm2cl(sinull_alms, sjnull_alms)

                    if isim < nsim_min:
                        cls1t = np.copy(cls1_sig)
                        if null_run:
                            cls_null1t = np.copy(cls_null1_sig)

                if do_noise and isim < num_noise:
                    cls1_res = OrderedDict()
                    if null_run:
                        cls_null1_res = OrderedDict()

                    nimap_alms, ninull_alms = process_index(
                        noise_files,
                        noise_files2,
                        idx,
                        isim,
                        noise_cache,
                        rls=rls,
                    )
                    njmap_alms, njnull_alms = process_index(
                        noise_files,
                        noise_files2,
                        jdx,
                        isim,
                        noise_cache,
                        rls=rls,
                    )

                    # need non-symmetric since will potentially modify these
                    # with different residuals for T, E, B
                    for k, cx, cy, cnx, cny in [
                        ("nxn", nimap_alms, njmap_alms, ninull_alms, njnull_alms),
                        ("sxn", simap_alms, njmap_alms, sinull_alms, njnull_alms),
                        ("nxs", nimap_alms, sjmap_alms, ninull_alms, sjnull_alms),
                    ]:
                        k0, k1 = ["{}{}".format(k, i) for i in range(2)]
                        cls1_res[k0] = self.alm2cl(cx, cy, symmetric=False)
                        cls1_res[k1] = self.alm2cl(cy, cx, symmetric=False)
                        if null_run:
                            cls_null1_res[k0] = self.alm2cl(cnx, cny, symmetric=False)
                            cls_null1_res[k1] = self.alm2cl(cny, cnx, symmetric=False)

                    # construct noise model
                    cls1_noise = (cls1_res["nxn0"] + cls1_res["nxn1"]) / 2.0
                    if null_run:
                        cls_null1_noise = (
                            cls_null1_res["nxn0"] + cls_null1_res["nxn1"]
                        ) / 2.0

                    # construct total model
                    if isim < nsim_min:
                        for k in cls1_res:
                            cls1t += cls1_res[k] / 2.0
                            if null_run:
                                cls_null1t += cls_null1_res[k] / 2.0

                quants = []
                if isim < num_signal:
                    quants += [[cls_sig, cls1_sig]]
                    if null_run:
                        quants += [[cls_null_sig, cls_null1_sig]]

                if do_noise and isim < num_noise:
                    quants += [[cls_noise, cls1_noise]]
                    quants += [[cls_res[k], cls1_res[k]] for k in cls_res]
                    if null_run:
                        quants += [[cls_null_noise, cls_null1_noise]]
                        quants += [
                            [cls_null_res[k], cls_null1_res[k]] for k in cls_null_res
                        ]
                if isim < nsim_min:
                    quants += [[cls_tot, cls1t]]
                    if null_run:
                        quants += [[cls_null_tot, cls_null1t]]

                for s, spec in enumerate(self.specs):
                    # running average
                    for q0, q1 in quants:
                        if spec not in q0:
                            q0[spec] = OrderedDict()
                        if xname not in q0[spec]:
                            q0[spec][xname] = np.zeros_like(q1[s])
                        d = q0[spec][xname]
                        d[:] += (q1[s] - d) / float(isim + 1)  # in-place

                    # matrix form for efficient median
                    if isim < nsim_min:
                        cls_all[isim][xind][s] = cls_tot[spec][xname]
                        if null_run:
                            cls_all_null[isim][xind][s] = cls_null_tot[spec][xname]

        cls_med_arr = np.median(cls_all, axis=0)
        for s, spec in enumerate(self.specs):
            cls_med[spec] = OrderedDict()
            for xind, xname in enumerate(map_pairs):
                cls_med[spec][xname] = cls_med_arr[xind][s]

        if null_run:
            cls_null_med_arr = np.median(cls_all_null, axis=0)
            for s, spec in enumerate(self.specs):
                cls_null_med[spec] = OrderedDict()
                for xind, xname in enumerate(map_pairs):
                    cls_null_med[spec][xname] = cls_null_med_arr[xind][s]

        self.cls_signal = cls_sig
        self.cls_signal_null = cls_null_sig
        self.cls_noise = cls_noise
        self.cls_noise_null = cls_null_noise
        self.cls_sim = cls_tot
        self.cls_sim_null = cls_null_tot
        self.cls_med = cls_med
        self.cls_med_null = cls_null_med
        self.cls_res = cls_res
        self.cls_res_null = cls_null_res

        # save and return
        return self.save_data(save_name, from_attrs=save_attrs, file_attrs=file_attrs)

    def get_masked_template_noise(self, template_noise_type):
        """
        Compute all combinations of template noise cross spectra from the ensemble
        of template noise sims.

        Arguments
        ---------
        template_noise_type : string
            Tag for directory containing template noise sims to be averaged and
            scaled similarly to the templates themselves.  These averaged sims
            are used to debias template cross spectra due to correlations in the
            way the noise ensembles are constructed.  Typically, this would be a
            noise model based on the Planck FFP10 ensemble for each half-mission
            foreground template.

        Returns
        -------
        cls_template_noise : OrderedDict
            Dictionary of template noise spectra averaged over all sims, containing
            the following keys: ["temp1:temp1", "temp2:temp2", "temp1:temp2"].  Each entry
            has the same shape structure as the ``cls_data`` attribute.

        Notes
        -----
        This method is called at the 'template_noise' checkpoint, and loads or
        saves a data dictionary containing the ``cls_template_noise`` attribute
        that is returned.
        """
        mask_files = self.mask_files
        map_tags = self.map_tags
        map_pairs = pt.tag_pairs(map_tags, index=True)
        num_maps = self.num_maps
        data_shape = (3 * self.data_shape[0], self.data_shape[1])

        save_attrs = ["cls_template_noise"]
        file_attrs = [
            "template_noise_type",
            "template_noise_root",
            "template_noise_root2",
            "template_noise_files",
            "template_noise_files2",
            "num_template_noise",
        ]
        save_attrs += file_attrs
        save_name = "template_noise_{}".format(template_noise_type)

        ret = self.load_data(
            save_name,
            "template_noise",
            fields=save_attrs,
            file_fields=file_attrs,
            to_attrs=True,
            shape=data_shape,
            shape_ref="cls_template_noise",
        )
        if ret is not None:
            return ret["cls_template_noise"]

        ret = self._get_template_files("template_noise", template_noise_type)
        for k, v in ret.items():
            setattr(self, k, v)
        map_files = [self.template_noise_files, self.template_noise_files2]

        cls_template_noise = OrderedDict()
        for k in ["temp1:temp1", "temp2:temp2", "temp1:temp2"]:
            cls_template_noise[k] = OrderedDict()

        cache = dict()

        # convenience functions
        def process_index(idx, isim):
            """
            Compute alms of masked temp1 and temp2 template noise sims
            """
            if idx in cache:
                return cache[idx]

            self.log(
                "Computing Alms for sim {} of map {}/{}".format(
                    isim, idx + 1, num_maps
                ),
                "all",
            )

            mask = self.get_mask(mask_files[idx])

            m_alms = []
            for f in [x[idx][isim] for x in map_files]:
                m = self.get_map(f)
                self.apply_mask(m, mask)
                m_alms.append(self.map2alm(m, self.pol))

            cache[idx] = m_alms
            return cache[idx]

        for isim in range(self.num_template_noise):
            cache.clear()
            for xname, (idx, jdx) in map_pairs.items():
                self.log(
                    "Computing spectra {} for template noise sim {}".format(
                        xname, isim
                    ),
                    "debug",
                )
                temp1i, temp2i = process_index(idx, isim)
                temp1j, temp2j = process_index(jdx, isim)

                cls1 = OrderedDict()
                cls1["temp1:temp1"] = self.alm2cl(temp1i, temp1j)
                cls1["temp2:temp2"] = self.alm2cl(temp2i, temp2j)
                cls1["temp1:temp2"] = 0.5 * (
                    self.alm2cl(temp1i, temp2j) + self.alm2cl(temp2i, temp1j)
                )

                for q0, q1 in [(cls_template_noise[k], cls1[k]) for k in cls1]:
                    for s, spec in enumerate(self.specs):
                        dd = q0.setdefault(spec, OrderedDict())
                        if xname not in dd:
                            dd[xname] = np.zeros_like(q1[s])
                        d = dd[xname]
                        d[:] += (q1[s] - d) / float(isim + 1)  # in-place

        self.cls_template_noise = cls_template_noise
        self.save_data(save_name, from_attrs=save_attrs, file_attrs=file_attrs)
        return cls_template_noise

    def get_kernels(self, window_lmax=None):
        """
        Compute kernels using the mask cross-spectra.  This follows
        the polspice azimuthal approximation for the kernel computation.

        Arguments
        ---------
        window_lmax : int
            The window within which the kernel is computed about
            each ell bin.

        Notes
        -----
        This method is called at the 'kernels' checkpoint and loads or saves
        the following data keys to disk:

            kern:
                temperature kernel
            pkern, mkern:
                + and - polarization kernels
            xkern:
                temperature/polarization cross term kernel
        """

        if window_lmax is None:
            window_lmax = self.lmax

        save_name = "kernels"
        save_attrs = ["kern", "pkern", "mkern", "xkern", "window_lmax"]
        ret = self.load_data(
            save_name,
            "kernels",
            fields=save_attrs,
            to_attrs=True,
            shape=self.kern_shape,
            shape_ref="kern",
            value_ref={"window_lmax": window_lmax},
        )
        if ret is not None:
            return ret

        kern = OrderedDict()
        if self.pol:
            pkern = OrderedDict()
            mkern = OrderedDict()
            xkern = OrderedDict()
        else:
            pkern = None
            mkern = None
            xkern = None

        lmax = self.lmax
        pol = self.pol
        wls = self.wls

        all_ells = np.arange(2 * lmax + 1)
        for xname in self.map_pairs:
            kern[xname] = np.zeros((lmax + 1, 2 * lmax + 1))
            if pol:
                pkern[xname] = np.zeros((lmax + 1, 2 * lmax + 1))
                mkern[xname] = np.zeros((lmax + 1, 2 * lmax + 1))
                xkern[xname] = np.zeros((lmax + 1, 2 * lmax + 1))

        for l in all_ells[2 : lmax + 1]:
            if np.mod(l, 50) == 0:
                self.log("Computing kernels for ell {}/{}".format(l, lmax), "debug")
            l2 = np.min([2 * lmax + 1, l + lmax + 1])
            # populate upper triangle
            for ll in all_ells[l:l2]:
                j0, j0_lmin, j0_lmax = st.wigner3j(l, 0, ll, 0)
                if pol:
                    j2, j2_lmin, j2_lmax = st.wigner3j(l, 2, ll, -2)

                # only go up to window lmax
                j0_lmax = np.minimum(j0_lmax, window_lmax)

                # computed as in https://arxiv.org/abs/1909.09375
                # equations 128 - 136
                l3 = np.arange(j0_lmin, j0_lmax + 1)
                dl3 = 2.0 * l3 + 1.0
                vk = j0[l3] ** 2 * dl3
                if pol:
                    sign = ((-1.0) ** (l + ll + l3)).astype(int)
                    v = j2[l3] ** 2 * dl3
                    vp = v * (1.0 + sign) / 2.0
                    vm = v * (1.0 - sign) / 2.0
                    vx = j2[l3] * j0[l3] * dl3
                for xname in self.map_pairs:
                    wls1 = wls[xname][:, l3]
                    kern[xname][l, ll] += (vk * wls1[0]).sum(axis=-1)
                    if pol:
                        pkern[xname][l, ll] += (vp * wls1[1]).sum(axis=-1)
                        mkern[xname][l, ll] += (vm * wls1[1]).sum(axis=-1)
                        xkern[xname][l, ll] += (vx * wls1[2]).sum(axis=-1)

        # apply symmetry relation
        for l in all_ells[2 : lmax + 1]:
            ll = np.arange(2 * lmax + 1)
            dll = (2.0 * ll + 1.0) / 4.0 / np.pi
            sll = slice(l, lmax + 1)
            for xname in self.map_pairs:
                # populate lower triangle (wigners are symmetric in l and ll)
                kern[xname][sll, l] = kern[xname][l, sll]
                if pol:
                    pkern[xname][sll, l] = pkern[xname][l, sll]
                    mkern[xname][sll, l] = mkern[xname][l, sll]
                    xkern[xname][sll, l] = xkern[xname][l, sll]
                # apply ell scaling along the axis that we bin over
                kern[xname][l, :] *= dll
                if pol:
                    pkern[xname][l, :] *= dll
                    mkern[xname][l, :] *= dll
                    xkern[xname][l, :] *= dll

        # save and return
        self.kern = kern
        self.pkern = pkern
        self.mkern = mkern
        self.xkern = xkern
        self.window_lmax = window_lmax

        return self.save_data(save_name, from_attrs=save_attrs)

    def get_signal_shape(
        self,
        filename=None,
        r=None,
        component=None,
        flat=None,
        filename_fg=None,
        freq_ref=359.7,
        beta_ref=1.54,
        signal_mask=None,
        transfer=False,
        save=True,
    ):
        """
        Load a shape spectrum for input to the Fisher iteration algorithm.

        If the spectrum is used as input to ``get_transfer``, it must match
        the spectrum used to generate the simulations, in order to compute
        the correct transfer function.

        Alternatively, the spectrum can be computed using CAMB for arbitrary
        values of ``r``, typically used to compute the ``r`` likelihood once
        the bandpowers have been computed.

        Finally, the spectrum can be flat in ell^2 Cl.  This is typically
        used as the input shape for computing bandpowers for a null test.

        Arguments
        ---------
        filename : string
            Filename for a signal spectrum on disk.  If None, and ``r`` is None
            and ``flat`` is False, this will search for a spectrum stored in
            ``signal_<signal_type>/spec_signal_<signal_type>.dat``.  Otherwise,
            if the filename is a relative path and not found, the config
            directory is searched.
        r : float
            If supplied and ``flat`` is False, a spectrum is computed using
            CAMB for the given ``r`` value.  Overrides ``filename``.
        component : 'scalar', 'tensor', 'cmb', 'fg'
            If 'scalar', and ``r`` is not None, return just the r=0 scalar terms
            in the signal model.  If 'tensor', return just the tensor component
            scaled by the input ``r`` value. If 'cmb' or 'fg', return just those
            signal terms.
        flat : float
            If given, a spectrum that is flat in ell^2 Cl is returned, with
            amplitude given by the supplied value. Overrides all other options.
        filename_fg : string
            Filename for a foreground spectrum on disk.  If the filename is a
            relative path and not found, the config directory is searched.  If
            not supplied, a power law dust model spectrum is assumed.
        freq_ref : float
            In GHz, reference frequency for dust model. Dust bandpowers output
            will be at this reference frequency.
        beta_ref : float
            The spectral index of the dust model. This is a fixed value, with
            an additive deviation from this value fit for in foreground fitting
            mode.
        signal_mask: str array
            Include only these spectra, others set to zero.
            Options: TT, EE, BB, TE, EB, TB
        transfer : bool
            If True, this is a transfer function run. If ``filename`` is None
            and ``r`` is None and ``flat`` is False, will search for a signal
            spectrum stored in
            ``signal_<signal_transfer_type>/spec_signal_<signal_transfer_type>.dat``.
        save : bool
            If True, save signal shape dict to disk.

        Returns
        -------
        cls : OrderedDict
            Dictionary keyed by spectrum (cmb_tt, cmb_ee, ... , fg_tt, ...),
            each entry containing a vector of length 2 * lmax + 1.
        """

        lmax_kern = 2 * self.lmax

        specs = list(self.specs)

        comps = list(self.signal_components)
        if (self.null_run or transfer) and "fg" in comps:
            comps.remove("fg")
        if component in self.signal_components:
            comps = [component]
        elif component in ["scalar", "tensor"]:
            comps = ["cmb"]

        if transfer:
            if "eb" in specs:
                specs.remove("eb")
            if "tb" in specs:
                specs.remove("tb")
        tbeb = "eb" in specs and "tb" in specs and r is None

        if save:
            shape = (len(specs) * len(comps), lmax_kern + 1)
            save_name = "shape_transfer" if transfer else "shape"

            opts = dict(signal_mask=signal_mask)
            if "cmb" in comps:
                opts.update(filename=filename, r=r, flat=flat)
            if "fg" in comps:
                opts.update(
                    filename_fg=filename_fg, freq_ref=freq_ref, beta_ref=beta_ref
                )
            ret = self.load_data(
                save_name, save_name, shape_ref="cls_shape", shape=shape, value_ref=opts
            )
            if ret is not None:
                if "cmb" in comps and r is not None:
                    self.r_model = ret["r_model"]
                if "fg" in comps:
                    self.freq_ref = ret["freq_ref"]
                    self.beta_ref = ret["beta_ref"]
                    self.fg_scales = ret["fg_scales"]
                self.cls_shape = ret["cls_shape"]
                return ret["cls_shape"]

        ell = np.arange(lmax_kern + 1)
        ellfac = ell * (ell + 1) / 2.0 / np.pi
        cls_shape = OrderedDict()

        if "cmb" in comps:
            if flat is not None and flat is not False:
                if flat is True:
                    flat = 2e-5
                # flat spectrum for null tests
                for spec in specs:
                    cls_shape["cmb_" + spec] = flat * np.ones_like(ell)

            elif r is not None:
                # cache model components
                if not hasattr(self, "r_model") or self.r_model is None:
                    # scalar CAMB spectrum
                    scal = st.get_camb_cl(r=0, lmax=lmax_kern)
                    # tensor CAMB spectrum for r=1, scales linearly with r
                    tens = st.get_camb_cl(r=1, lmax=lmax_kern, nt=0, spec="tensor")
                    self.r_model = {"scalar": scal, "tensor": tens}
                    if save:
                        opts["r_model"] = self.r_model
                # CAMB spectrum for given r value
                component = str(component).lower()
                if component == "scalar":
                    cls_camb = self.r_model["scalar"]
                elif component == "tensor":
                    cls_camb = r * self.r_model["tensor"]
                else:
                    cls_camb = self.r_model["scalar"] + r * self.r_model["tensor"]
                cls_shape.update({"cmb_" + s: cls for s, cls in zip(specs, cls_camb)})

            else:
                # signal sim model or custom filename
                if filename is None:
                    signal_root = self.signal_root
                    filename = "spec_{}.dat".format(os.path.basename(signal_root))
                    filename = os.path.join(signal_root, filename)
                elif not os.path.exists(filename):
                    filename = os.path.join(self.config_root, filename)
                if not os.path.exists(filename):
                    raise OSError("Missing model file {}".format(filename))

                cls_cmb = st.load_camb_cl(filename, lmax=lmax_kern, pol=self.pol)
                cls_shape.update({"cmb_" + s: cls for s, cls in zip(specs, cls_cmb)})

            # EB and TB flat l^2 * C_l
            if self.pol:
                if tbeb and (flat is None or flat is False):
                    tbeb_flat = np.abs(cls_shape["cmb_bb"][100]) * ellfac[100] * 1e-4
                    tbeb_flat = np.ones_like(cls_shape["cmb_bb"]) * tbeb_flat
                    tbeb_flat[:2] = 0
                    cls_shape["cmb_eb"] = np.copy(tbeb_flat)
                    cls_shape["cmb_tb"] = np.copy(tbeb_flat)
                elif not tbeb and not transfer:
                    cls_shape["cmb_eb"] = np.zeros_like(ell, dtype=float)
                    cls_shape["cmb_tb"] = np.zeros_like(ell, dtype=float)

        if "fg" in comps:
            # dust signal sim model or custom filename
            # XXX optionally load freq_ref and beta_ref from file
            if filename_fg is None:
                # From Planck LIV EE dust
                cls_dust = st.dust_model(ell, lfac=True)
                for spec in specs[: 4 if self.pol else 1]:
                    cls_shape["fg_" + spec] = cls_dust
                self.log(
                    "Added simple foregrounds to cls shape {}".format(list(cls_shape)),
                    "debug",
                )
            else:
                if not os.path.exists(filename_fg):
                    filename_fg = os.path.join(self.config_root, filename_fg)
                if not os.path.exists(filename_fg):
                    raise OSError(
                        "Missing foreground model file {}".format(filename_fg)
                    )

                cls_fg = st.load_camb_cl(filename_fg, lmax=lmax_kern, pol=None)
                cls_shape.update({"fg_" + s: cls for s, cls in zip(specs, cls_fg)})

            if self.pol:
                if tbeb and "fg_eb" not in cls_shape:
                    tbeb_flat = np.abs(cls_shape["fg_ee"][100]) * 1e-4
                    tbeb_flat = np.ones_like(cls_shape["fg_ee"]) * tbeb_flat
                    cls_shape["fg_eb"] = np.copy(tbeb_flat)
                    cls_shape["fg_tb"] = np.copy(tbeb_flat)
                else:
                    cls_shape["fg_eb"] = np.zeros_like(ell, dtype=float)
                    cls_shape["fg_tb"] = np.zeros_like(ell, dtype=float)

            # frequency scaling for cross spectra
            fg_scales = OrderedDict()
            for xname, (m0, m1) in self.map_pairs.items():
                f0 = self.map_freqs[m0]
                f1 = self.map_freqs[m1]
                fg_scales[xname] = st.scale_dust(f0, f1, freq_ref, beta_ref, delta=True)
            self.fg_scales = fg_scales
            self.freq_ref = freq_ref
            self.beta_ref = beta_ref
            if save:
                opts["fg_scales"] = self.fg_scales

        # divide out l^2/2pi
        for cl in cls_shape.values():
            cl[2:] /= ellfac[2:]
            cl[:2] = 0.0

        if signal_mask is not None:
            self.log("Masking {} spectra".format(signal_mask), "debug")
            for csk in cls_shape:
                masked = False
                for smk in signal_mask:
                    if smk.lower() in csk:
                        masked = True
                if not masked:
                    cls_shape[csk] *= 1.0e-12

        if save:
            self.cls_shape = cls_shape
            opts["cls_shape"] = cls_shape
            self.save_data(save_name, **opts)

        return cls_shape

    def get_beams(self, pixwin=True):
        """
        Return beam window functions for all input map tags.

        Arguments
        ---------
        pixwin : bool
            If True, the pixel window function for the map ``nside`` is
            applied to the Gaussian beams.

        Returns
        -------
        beam_windows : dict
            A dictionary of beam window function arrays
            (3 x lmax+1 if pol, 1 x lmax+1 if not) for each map tag

        Notes
        -----
        This method is called at the 'beams' checkpoint and loads or saves
        a dictionary containing just the ``beam_windows`` key to disk.
        """
        import healpy as hp

        lsize = 2 * self.lmax + 1
        nspec = 6 if self.pol else 1
        beam_shape = (self.num_maps * nspec, lsize)

        save_name = "beams"
        cp = "beams"
        ret = self.load_data(
            save_name, cp, fields=["beam_windows"], to_attrs=True, shape=beam_shape
        )
        if ret is not None:
            return ret["beam_windows"]

        beam_windows = OrderedDict()
        beam_windows["tt"] = OrderedDict()
        if self.pol:
            for s in ["ee", "bb", "te", "eb", "tb"]:
                beam_windows[s] = OrderedDict()

        pwl = np.ones((3 if self.pol else 1, lsize))
        if pixwin:
            pwl *= 0.0
            pix = hp.pixwin(self.nside, pol=self.pol)
            if self.pol:
                pixT = pix[0]
                pixP = pix[1]
                end = min(len(pixT), lsize)
                pwl[1, :end] = pixP[:end]
                pwl[2, :end] = np.sqrt(pixT[:end] * pixP[:end])
            else:
                pixT = pix
                end = min(len(pixT), lsize)
            pwl[0, :end] = pixT[:end]

        for tag, otag in zip(self.map_tags, self.map_tags_orig):
            if otag in self.beam_product:
                bl = np.atleast_2d(self.beam_product[otag])[:, :lsize]
            elif otag in self.fwhm:
                bl = hp.gauss_beam(self.fwhm[otag], lsize - 1, self.pol)
                if self.pol:
                    bl = bl.T[[0, 1, 3]]
            else:
                raise ValueError("No beam in config for {}".format(tag))

            blpw = np.atleast_2d(bl * pwl)[:, :lsize]
            beam_windows["tt"][tag] = np.copy(blpw[0])
            if self.pol:
                for s in ["ee", "bb", "eb"]:
                    beam_windows[s][tag] = np.copy(blpw[1])
                for s in ["te", "tb"]:
                    beam_windows[s][tag] = np.copy(blpw[2])

        # save and return
        self.beam_windows = beam_windows
        self.save_data(save_name, from_attrs=["beam_windows"])
        return self.beam_windows

    def get_beam_errors(self):
        """
        Get error envelope to multiply beam by (so, to get beam + 2 sigma error,
        do beam * (1 + 2 * beam_error))
        """

        lsize = 2 * self.lmax + 1
        nspec = 6 if self.pol else 1
        beam_shape = (self.num_maps * nspec, lsize)

        save_name = "beam_errors"
        cp = "beam_errors"

        if hasattr(self, "beam_errors") and not self.force_rerun[cp]:
            return self.beam_errors

        ret = self.load_data(
            save_name, cp, fields=["beam_errors"], to_attrs=True, shape=beam_shape
        )
        if ret is not None:
            return ret["beam_errors"]

        beam_errors = OrderedDict()
        beam_errors["tt"] = OrderedDict()
        if self.pol:
            for s in ["ee", "bb", "te", "eb", "tb"]:
                beam_errors[s] = OrderedDict()

        for tag, otag in zip(self.map_tags, self.map_tags_orig):
            if otag in self.beam_error_product:
                be = self.beam_error_product[otag]
            elif otag in self.fwhm_err:
                # convert error on the FWHM to an envelope error on the beam window
                fwhm = self.fwhm[otag]
                bl = self.beam_windows["tt"][tag]
                blp = hp.gauss_beam(
                    fwhm * (1 - self.fwhm_err[otag]), lsize - 1, self.pol
                )
                blm = hp.gauss_beam(
                    fwhm * (1 + self.fwhm_err[otag]), lsize - 1, self.pol
                )
                if self.pol:
                    bl = np.asarray(
                        [bl, self.beam_windows["ee"][tag], self.beam_windows["te"][tag]]
                    )
                    blp = blp.T[[0, 1, 3]]
                    blm = blm.T[[0, 1, 3]]
                be = (blp - blm) / 2.0 / bl
            else:
                raise ValueError("No beam in config for {}".format(otag))

            be = np.atleast_2d(be)[:, :lsize]
            beam_errors["tt"][tag] = np.copy(be[0])
            if self.pol:
                for s in ["ee", "bb", "eb"]:
                    beam_errors[s][tag] = np.copy(be[1])
                for s in ["te", "tb"]:
                    beam_errors[s][tag] = np.copy(be[2])

        # save and return
        self.beam_errors = beam_errors
        self.save_data(save_name, from_attrs=["beam_errors"])
        return self.beam_errors

    def get_bin_def(
        self,
        lmin=2,
        lmax=500,
        pol=True,
        pol_mask=True,
        tbeb=False,
        bin_width=25,
        weighted_bins=False,
        residual_fit=False,
        res_specs=None,
        bin_width_res=25,
        foreground_fit=False,
        beta_fit=False,
        bin_width_fg=25,
        lmin_fg=None,
        lmax_fg=None,
    ):
        """
        Construct the bin definition array that defines the bins for each output
        spectrum.

        Arguments
        ---------
        lmin : int
            Minimum ell for binned spectra
        lmax : int
            The maximum multipole for which spectra are computed
        pol : bool
            If True, polarized spectra are computed from the input maps
        pol_mask : bool
            If True, two independent masks are applied to every map:
            one for T maps and one for Q/U maps.
        tbeb : bool
            If True, EB and TB bins are constructed so that these spectra are
            computed by the estimator.  Otherwise, these spectra are fixed at
            zero.
        bin_width : int or list of ints
            Width of each spectrum bin.  If a scalar, the same width is applied
            to all cross spectra.  Otherwise, must be a list of up to six
            elements, listing bin widths for the spectra in the order (TT, EE,
            BB, TE, EB, TB).
        weighted_bins : bool
            If True, use an lfac-weighted binning operator to construct Cbls.
            By default, a flat binning operator is used.
        residual_fit : bool
            If True, fit for (compute bandpower amplitudes for) several wide
            bins of excess noise.
        res_specs : list of strings
            List of spectra which are to be included in the residual fit.  Can
            be individual spectra ('tt', 'ee', 'bb'), or 'eebb' to fit for EE
            and BB residuals simultaneously.  If not supplied, this defaults to
            ['eebb'] for polarized maps, and ['tt'] for unpolarized maps.
        bin_width_res : int or list of ints
            Width of each residual spectrum bin.  If a scalar, the same width
            is applied to all spectra for all cross spectra.  Otherwise, must
            be a list of up to nspec * nmaps elements, listing bin widths for
            each of the spectra in ``res_specs`` in order, then ordered by map.
        foreground_fit : bool
            If True, construct bin definitions for foreground components as
            well.
        beta_fit : bool
            If True, include ``delta_beta`` in the foreground fitting parameters,
            along with the foreground amplitudes.
        bin_width_fg : int or list of ints
            Width of each foreground spectrum bin.  If a scalar, the same width
            is applied to all cross spectra.  Otherwise, must be a list of up to
            six elements, listing bin widths for the spectra in the order (TT,
            EE, BB, TE, EB, TB).
        lmin_fg : int
            Minimum ell to use for defining foreground bins.  If not set,
            defaults to ``lmin``.
        lmax_fg : int
            Maximum ell to use for defining foreground bins.  If not set,
            defaults to ``lmax``.

        Returns
        -------
        bin_def : dict
            The bin definition dictionary.  Each key contains a Nx2 array
            that defines the left and right edges for each bin of the
            corresponding spectrum.
        """
        self.lmax = lmax
        self.pol = pol
        self.pol_dim = 3 if self.pol else 1
        self.pol_mask = pol_mask if self.pol else False

        # count all the things
        self.num_corr = pt.num_corr(self.num_maps)
        self.num_spec = pt.num_corr(3 if self.pol else 1)
        self.num_spec_mask = pt.num_corr(2 if self.pol_mask else 1)
        self.num_kern = self.num_corr
        self.data_shape = (self.num_corr * self.num_spec, self.lmax + 1)
        self.mask_shape = (self.num_corr * self.num_spec_mask, self.lmax + 1)
        self.kern_shape = (self.num_kern * (self.lmax + 1), 2 * self.lmax + 1)

        npol = (6 if tbeb else 4) if self.pol else 1
        specs = ["tt", "ee", "bb", "te", "eb", "tb"][:npol]

        if np.isscalar(bin_width) or len(bin_width) == 1:
            bin_width = [bin_width] * npol
        bin_width = np.asarray(bin_width).ravel()[:npol]

        bwerr = "EE and BB must have the same bin width (for mixing)"
        if self.pol and bin_width[1] != bin_width[2]:
            raise ValueError(bwerr)

        comps = []
        signal_comps = []

        # Define bins
        nbins_cmb = 0
        bin_def = OrderedDict()
        for spec, bw in zip(specs, bin_width):
            bins = np.arange(lmin, self.lmax, bw)
            bins = np.append(bins, self.lmax + 1)
            bin_def["cmb_{}".format(spec)] = np.column_stack((bins[:-1], bins[1:]))
            nbins_cmb += len(bins) - 1
        comps += ["cmb"]
        signal_comps += ["cmb"]
        self.log("Added {} CMB bins to bin_def".format(nbins_cmb), "debug")

        # Do the same for foreground bins
        nbins_fg = 0
        if foreground_fit:
            if np.isscalar(bin_width_fg) or len(bin_width_fg) == 1:
                bin_width_fg = [bin_width_fg] * npol
            bin_width_fg = np.asarray(bin_width_fg).ravel()[:npol]

            if self.pol and bin_width_fg[1] != bin_width_fg[2]:
                raise ValueError("Foreground {}".format(bwerr))

            lmin_fg = lmin_fg or lmin
            assert lmin_fg >= lmin
            lmax_fg = lmax_fg or self.lmax
            assert lmax_fg <= lmax

            for spec, bw in zip(specs, bin_width_fg):
                bins = np.arange(lmin_fg, lmax_fg, bw)
                bins = np.append(bins, lmax_fg + 1)
                bin_def["fg_{}".format(spec)] = np.column_stack((bins[:-1], bins[1:]))
                nbins_fg += len(bins) - 1
            if beta_fit:
                bin_def["delta_beta"] = np.array([[0, 0]])
            comps += ["fg"]
            signal_comps += ["fg"]
            self.log(
                "Added {} foreground bins to bin_def".format(nbins_fg + int(beta_fit)),
                "debug",
            )

        # Do the same for residual bins
        nbins_res = 0
        if residual_fit:
            if res_specs is None:
                res_specs = ["eebb"] if self.pol else ["tt"]
            res_specs = [s.lower() for s in res_specs]
            nmap = len(self.map_tags)
            nspecr = len(res_specs) * nmap
            if np.isscalar(bin_width_res) or len(bin_width_res) == 1:
                bin_width_res = [bin_width_res] * nspecr
            elif len(bin_width_res) == nspecr // nmap:
                bin_width_res = np.array([bin_width_res] * nmap).ravel()
            bin_width_res = np.asarray(bin_width_res).ravel()[:nspecr].reshape(nmap, -1)

            for tag, bws in zip(self.map_tags, bin_width_res):
                if self.pol and "ee" in res_specs and "bb" in res_specs:
                    if bws[res_specs.index("ee")] != bws[res_specs.index("bb")]:
                        raise ValueError("Residual {} {}".format(tag, bwerr))

                for spec, bw in zip(res_specs, bws):
                    bins = np.arange(lmin, self.lmax, bw)
                    bins = np.append(bins, self.lmax + 1)
                    btag = "res_{}_{}".format(spec, tag)
                    bin_def[btag] = np.column_stack((bins[:-1], bins[1:]))
                    nbins_res += len(bins) - 1

            comps += ["res"]
            self.log("Added {} residual bins to bin_def".format(nbins_res), "debug")

        ell = np.arange(self.lmax + 1)
        lfac = ell * (ell + 1) / 2.0 / np.pi

        bin_weights = OrderedDict()
        for k, bd in bin_def.items():
            bin_weights[k] = []
            for left, right in bd:
                if weighted_bins:
                    w = lfac[left:right] / np.mean(lfac[left:right])
                else:
                    w = 1.0
                bin_weights[k].append(w)

        self.lmin = lmin
        self.nbins_cmb = nbins_cmb
        self.nbins_fg = nbins_fg
        self.nbins_res = nbins_res
        self.bin_def = bin_def
        self.specs = specs
        self.weighted_bins = weighted_bins
        self.bin_weights = bin_weights
        self.components = comps
        self.signal_components = signal_comps

        return self.bin_def

    def kernel_precalc(self, map_tag=None, transfer=False):
        """
        Compute the mixing kernels M_ll' = K_ll' * F_l' * B_l'^2.  Called by
        ``bin_cl_template`` to pre-compute kernel terms.

        Arguments
        ---------
        map_tag : str
            If supplied, the kernels are computed only for the given map tag
            (or cross if map_tag is map_tag1:map_tag2).
            Otherwise, it is computed for all maps and crosses.
        transfer : bool
            If True, set transfer function to 1 to solve for transfer function
            qbs.

        Returns
        -------
        mll : OrderedDict
            Dictionary of M_ll' matrices, keyed by component spec and xname.
        """

        map_pairs = None
        if map_tag is not None:
            if map_tag in self.map_pairs:
                map_pairs = {map_tag: self.map_pairs[map_tag]}
                map_tags = list(set(self.map_pairs[map_tag]))
            else:
                map_tags = [map_tag]
                map_pairs = pt.tag_pairs(map_tags)
        else:
            map_tags = self.map_tags
            map_pairs = self.map_pairs

        specs = list(self.specs)
        lmax = self.lmax  # 2 * lmax

        lk = slice(0, lmax + 1)
        mll = OrderedDict()

        if transfer:
            comps = ["cmb"]
        else:
            comps = list(self.signal_components)

        for stag in self.bin_def:
            comp, spec = stag.split("_", 1)
            if comp not in comps:
                continue

            mll[stag] = OrderedDict()
            if spec in ["ee", "bb"]:
                mstag = "{}_mix".format(stag)
                mll[mstag] = OrderedDict()

            bw = self.beam_windows[spec]
            if not transfer:
                tf = self.transfer[stag]

            for xname, (m0, m1) in map_pairs.items():
                # beams
                fb2 = bw[m0][lk] * bw[m1][lk]

                # transfer function
                if not transfer:
                    fb2 *= np.sqrt(tf[m0][lk] * tf[m1][lk])

                # kernels
                if spec == "tt":
                    k = self.kern[xname][:, lk]
                elif spec in ["ee", "bb"]:
                    k = self.pkern[xname][:, lk]
                    mk = self.mkern[xname][:, lk]
                elif spec in ["te", "tb"]:
                    k = self.xkern[xname][:, lk]
                elif spec == "eb":
                    k = self.pkern[xname][:, lk] - self.mkern[xname][:, lk]

                # store final product
                mll[stag][xname] = k * fb2
                if spec in ["ee", "bb"]:
                    mll[mstag][xname] = mk * fb2

        return mll

    def bin_cl_template(
        self,
        cls_shape=None,
        map_tag=None,
        transfer=False,
        beam_error=False,
        use_precalc=True,
    ):
        """
        Compute the Cbl matrix from the input shape spectrum.

        This method requires beam windows, kernels and transfer functions
        (if ``transfer`` is False) to have been precomputed.

        Arguments
        ---------
        cls_shape : array_like
            The shape spectrum to use.  This can be computed using
            ``get_signal_shape`` or otherwise.
        map_tag : str
            If supplied, the Cbl is computed only for the given map tag
            (or cross if map_tag is map_tag1:map_tag2).
            Otherwise, it is computed for all maps and crosses.
        transfer : bool
            If True, this assumes a unity transfer function for all bins, and
            the output Cbl is used to compute the transfer functions that are
            then loaded when this method is called with ``transfer = False``.
        beam_error : bool
            If True, use beam error envelope instead of beam to get cbls that
            are 1 sigma beam error envelope offset of signal terms.
        use_precalc : bool
            If True, load pre-calculated terms stored from a previous iteration,
            and store for a future iteration.  Otherwise, all calculations are
            repeated.

        Returns
        -------
        cbl : dict of arrays (num_bins, 2, lmax + 1)
            The Cbl matrix, indexed by component and spectrum, then by map
            cross, e.g. ``cbl['cmb_tt']['map1:map2']``.  E/B mixing terms are
            stored in elements ``cbl['cmb_ee_mix']`` and ``cbl['cmb_bb_mix']``,
            and unmixed terms are stored in elements ``cbl['cmb_ee']`` and
            ``cbl['cmb_bb']``.
        """
        if cls_shape is None:
            cls_shape = self.cls_shape

        map_pairs = None
        if map_tag is not None:
            if map_tag in self.map_pairs:
                map_pairs = {map_tag: self.map_pairs[map_tag]}
                map_tags = list(set(self.map_pairs[map_tag]))
            else:
                map_tags = [map_tag]
        else:
            map_tags = self.map_tags

        if map_pairs is None:
            map_pairs = pt.tag_pairs(map_tags)

        specs = list(self.specs)
        if transfer:
            if "eb" in specs:
                specs.remove("eb")
            if "tb" in specs:
                specs.remove("tb")

        lmax = self.lmax
        lmax_kern = lmax  # 2 * self.lmax

        if getattr(self, "mll", None) is None or not use_precalc:
            mll = self.kernel_precalc(map_tag=map_tag, transfer=transfer)
            if use_precalc:
                self.mll = mll
        else:
            mll = self.mll

        if beam_error:
            beam_error = self.get_beam_errors()
            beam_keys = ["b1", "b2", "b3"]

        ls = slice(2, lmax + 1)
        lk = slice(0, lmax_kern + 1)
        cbl = OrderedDict()

        comps = []
        for comp in self.signal_components:
            if any([k.startswith(comp + "_") for k in cls_shape]):
                comps += [comp]
        if "fg" in comps and transfer:
            comps.remove("fg")
        if "res" in self.components and not transfer:
            comps += ["res"]
            cls_noise = self.cls_noise_null if self.null_run else self.cls_noise
            cls_res = self.cls_res_null if self.null_run else self.cls_res

        ell = np.arange(lmax_kern + 1)

        def binup(d, left, right, weights):
            return (d[..., left:right] * weights).sum(axis=-1)

        def binup2(d, bd, bw):
            d = np.asarray([binup(d, l, r, w) for (l, r), w in zip(bd, bw)])
            return d.transpose(1, 0, 2)

        # convert to matrices to speed up multiplication and binning
        nxmap = len(map_pairs)
        dshape = [nxmap, lmax + 1, lmax_kern + 1]
        d_arr = np.zeros(dshape)
        if "ee" in specs and "bb" in specs:
            md_arr = np.zeros(dshape)

        cshape = [nxmap, 1, lmax_kern + 1]
        s_arr = np.zeros(cshape)
        if beam_error:
            b_arr = {k: np.zeros(cshape) for k in beam_keys}

        for comp in comps:
            for spec in specs:
                if "res" not in comp:
                    # clear arrays
                    d_arr[:] = 0
                    if spec in ["ee", "bb"]:
                        md_arr[:] = 0
                    s_arr[:] = 0
                    if beam_error:
                        for k, v in b_arr.items():
                            v[:] = 0

                for xi, (xname, (tag1, tag2)) in enumerate(map_pairs.items()):
                    if "res" in comp:
                        s0, s1 = spec
                        bd = [[0, lmax + 1]]
                        # if any component of XY spec is in residual bin def,
                        # use that bin def
                        for v in [
                            "res_{}_{}".format(s0 * 2, tag1),
                            "res_{}_{}".format(s0 * 2, tag2),
                            "res_{}_{}".format(s1 * 2, tag1),
                            "res_{}_{}".format(s1 * 2, tag2),
                        ]:
                            if v in self.bin_def:
                                bd = self.bin_def[v]
                                break
                            spec0 = v.split("_")[1]
                            if spec0 in ["ee", "bb"]:
                                v = v.replace(spec0, "eebb")
                                if v in self.bin_def:
                                    bd = self.bin_def[v]
                                    break

                        comp_list = [("res", cls_noise)] + [
                            ("res_{}".format(k), cls_res[k]) for k in cls_res
                        ]

                        for res_comp, cls in comp_list:
                            stag = "{}_{}".format(res_comp, spec)
                            cbl.setdefault(stag, OrderedDict())
                            cbl[stag][xname] = np.zeros((len(bd), lmax + 1))
                            cl1 = cls[spec][xname]
                            for idx, (left, right) in enumerate(bd):
                                lls = slice(left, right)
                                cbl[stag][xname][idx, lls] = np.copy(cl1[lls])

                        continue

                    # use correct shape spectrum
                    stag = "{}_{}".format(comp, spec)
                    if comp == "fg":
                        freq_scale = self.fg_scales[xname][0]
                        s_arr[xi] = freq_scale * cls_shape[stag][lk]
                    else:
                        s_arr[xi] = cls_shape[stag][lk]

                    # use correct beam error shape
                    if beam_error:
                        b_arr["b1"][xi] = beam_error[spec][tag1][lk]
                        b_arr["b2"][xi] = beam_error[spec][tag2][lk]

                    # apply cross spectrum kernel terms
                    d_arr[xi, ls] = mll[stag][xname][ls, lk]
                    if spec in ["ee", "bb"]:
                        mstag = stag + "_mix"
                        md_arr[xi, ls] = mll[mstag][xname][ls, lk]

                if "res" in comp:
                    continue

                stag = "{}_{}".format(comp, spec)
                bd = self.bin_def[stag]
                bw = self.bin_weights[stag]
                cbl.setdefault(stag, OrderedDict())

                # mixing terms
                mstag = None
                if spec in ["ee", "bb"]:
                    mstag = stag + "_mix"
                    cbl.setdefault(mstag, OrderedDict())

                # multiply all the things
                d_arr *= s_arr
                if mstag:
                    md_arr *= s_arr

                # bin all the things
                if beam_error:
                    b_arr["b3"][:] = b_arr["b1"] * b_arr["b2"]
                    for k in beam_keys:
                        d = binup2(d_arr * b_arr[k], bd, bw)
                        if mstag:
                            md = binup2(md_arr * b_arr[k], bd, bw)
                        for xi, xname in enumerate(map_pairs):
                            cbl[stag][xname][k] = d[xi]
                            if mstag:
                                cbl[mstag][xname][k] = md[xi]
                else:
                    d = binup2(d_arr, bd, bw)
                    if mstag:
                        md = binup2(md_arr, bd, bw)
                    for xi, xname in enumerate(map_pairs):
                        cbl[stag][xname] = d[xi]
                        if mstag:
                            cbl[mstag][xname] = md[xi]

        return cbl

    def get_model_spectra(self, qb, cbl, cls_noise=None, cond_noise=None):
        """
        Compute unbinned model spectra from qb amplitudes and a Cbl matrix.
        Requires pre-loaded bin definitions using ``get_bin_def`` or
        ``get_transfer``.

        This method is used internally by ``fisher_calc``.

        Arguments
        ---------
        qb : dict of arrays
            Array of bandpowers for every spectrum bin.
        cbl : dict
            Cbl dict as computed by ``bin_cl_template``.
        cls_noise : OrderedDict
            If supplied, the noise spectrum is applied to the model spectrum.
        cond_noise : float
            Conditioning noise amplitude to add to TT, EE and BB autospectra,
            to improve convergence of the fisher iterations.  The noise model
            is constant cond_noise for EE, BB and 10x that for TT.

        Returns
        -------
        cls : dict of arrays
            Model spectra.  Keyed by spectrum type, e.g. 'total_xx' for the
            total model spectrom, 'fg_xx' for the foreground terms, 'res_xx' for
            the residual (noise) terms, where 'xx' is one of the six power
            spectrum components (tt, ee, bb, te, eb, tb).  Each entry in the
            dictionary is itself a dictionary keyed by map cross, e.g.
            'map1:map1' for an autospectrum term, 'map1:map2' for a cross
            spectrum, etc, and the map names are the same as those in the
            ``map_tags`` attribute.  Each individual spectrum is an array of
            length ``lmax + 1``.
        """
        comps = []

        for comp in self.components:
            if any([k.startswith(comp + "_") for k in qb]):
                comps += [comp]

        delta_beta = 0.0
        if "delta_beta" in qb:
            # Evaluate fg at spectral index pivot for derivative
            # in Fisher matrix, unless delta is True
            delta_beta = qb["delta_beta"][0]

        if cls_noise is not None:
            comps += ["noise"]

        if cond_noise is not None:
            cls_cond = np.ones(self.lmax + 1) * cond_noise
            cls_cond[:2] = 0
            comps += ["cond"]

        if not len(comps):
            raise ValueError("Must specify at least one model component")

        cls = OrderedDict()

        specs = []
        for spec in self.specs:
            for comp in self.signal_components:
                if "{}_{}".format(comp, spec) in cbl:
                    # Don't add entries that won't be filled in later
                    cls["total_{}".format(spec)] = OrderedDict()
            specs.append(spec)

        for comp in comps:
            for spec in specs:
                stag = "{}_{}".format(comp, spec)
                if spec in ["ee", "bb"]:
                    mstag = "{}_{}".format(comp, "bb" if spec == "ee" else "ee")

                if comp == "noise":
                    pairs = list(cls_noise[spec])
                elif comp == "cond":
                    pairs = list(cls["total_{}".format(spec)])
                else:
                    if "res" not in comp and stag not in qb:
                        continue
                    if "res" not in comp and stag not in cbl:
                        continue
                    pairs = list(self.map_pairs)  # list(cbl[stag])

                for xname in pairs:
                    tag1, tag2 = self.map_pairs[xname]

                    # extract qb's for the component spectrum
                    if comp == "cmb" or (comp == "fg" and delta_beta == 0.0):
                        qbs = qb[stag]
                        if spec in ["ee", "bb"]:
                            qbm = qb[mstag]

                    elif comp == "fg" and delta_beta != 0.0:
                        # beta scaling for foreground model
                        # beta_scale = self.fg_scales[xname][1] ** delta_beta
                        beta_scale = 1 + self.fg_scales[xname][2] * delta_beta
                        qbs = beta_scale * qb[stag]
                        if spec in ["ee", "bb"]:
                            qbm = beta_scale * qb[mstag]

                    elif comp == "res":
                        # modify model by previously fit res, including
                        # off diagonals and SXN/NXS for nulls
                        s0, s1 = spec  # separate qbs for, eg, TE resTT and resEE
                        res_tags = {
                            "s0m0": "res_{}_{}".format(s0 * 2, tag1),
                            "s0m1": "res_{}_{}".format(s0 * 2, tag2),
                            "s1m0": "res_{}_{}".format(s1 * 2, tag1),
                            "s1m1": "res_{}_{}".format(s1 * 2, tag2),
                        }
                        qbr = {"s0m0": 1, "s0m1": 1, "s1m0": 1, "s1m1": 1}

                        for k, v in res_tags.items():
                            if v in qb:
                                qbr[k] = np.sqrt(1 + qb[v])[:, None]
                            else:
                                spec0 = v.split("_")[1]
                                if spec0 in ["ee", "bb"]:
                                    res_tags[k] = v.replace(spec0, "eebb")
                                    if res_tags[k] in qb:
                                        qbr[k] = np.sqrt(1 + qb[res_tags[k]])[:, None]

                            if np.any(np.isnan(qbr[k])):
                                self.warn(
                                    "Unphysical residuals fit, "
                                    "setting to zero {} bins {}".format(
                                        spec, np.where(np.isnan(qbr[k]))
                                    )
                                )
                                qbr[k][np.isnan(qbr[k])] = 1

                        res_list = [
                            ("s0m0", "s1m1", "nxn0"),  # N_s0m0 x N_s1m1
                            ("s1m0", "s0m1", "nxn1"),  # N_s1m0 x N_s0m1
                        ]
                        if self.null_run:
                            res_list += [
                                (None, "s1m1", "sxn0"),  # S_s0m0 x N_s1m1
                                (None, "s0m1", "sxn1"),  # S_s1m0 x N_s0m1
                                ("s0m0", None, "nxs0"),  # N_s0m0 x S_s1m1
                                ("s1m0", None, "nxs1"),  # N_s1m0 x S_s0m1
                            ]

                        cl1 = np.zeros_like(cl1)
                        for k1, k2, k3 in res_list:
                            q1 = qbr[k1] if k1 else 1.0
                            q2 = qbr[k2] if k2 else 1.0
                            rtag = "res_{}_{}".format(k3, spec)
                            cl1 += ((q1 * q2 - 1.0) * cbl[rtag][xname]).sum(axis=0)

                        # all of these were asymmetric specs, divide by 2 for mean
                        cl1 /= 2.0

                    # compute model spectra
                    if comp in self.signal_components:
                        if xname not in cbl[stag]:
                            continue
                        cbl1 = cbl[stag][xname]
                        if isinstance(cbl1, dict):
                            # has beam error terms. deal with them individually
                            cl1 = OrderedDict()
                            for k, v in cbl1.items():
                                cl1[k] = (qbs[:, None] * v).sum(axis=0)
                        else:
                            cl1 = (qbs[:, None] * cbl1).sum(axis=0)
                        if spec in ["ee", "bb"]:
                            # mixing terms, add in-place
                            if qbm is not None and mstag + "_mix" in cbl:
                                cbl1_mix = cbl[mstag + "_mix"][xname]
                                if isinstance(cbl1_mix, dict):
                                    for k, v in cbl1_mix.items():
                                        cl1[k] += (qbm[:, None] * v).sum(axis=0)
                                else:
                                    cl1 += (qbm[:, None] * cbl1_mix).sum(axis=0)

                    elif comp == "noise":
                        cl1 = cls_noise[spec][xname][: self.lmax + 1]

                    elif comp == "cond":
                        # add conditioner along diagonal
                        if tag1 != tag2:
                            continue
                        if spec == "tt":
                            cl1 = 10 * cls_cond
                        elif spec in ["ee", "bb"]:
                            cl1 = cls_cond
                        else:
                            continue

                    # store
                    cls.setdefault(stag, OrderedDict())[xname] = cl1

                    # add to total model
                    ttag = "total_{}".format(spec)
                    cls[ttag].setdefault(xname, np.zeros_like(cl1))
                    cls[ttag][xname] += cl1

        return cls

    def get_data_spectra(self, map_tag=None, transfer=False, do_noise=True):
        """
        Return data and noise spectra for the given map tag(s).  Data spectra
        and signal/noise sim spectra must have been precomputed or loaded from
        disk.

        Arguments
        ---------
        map_tag : str
            If None, all map-map cross-spectra are included in the outputs.
            Otherwise, only the autospectra of the given map are included.
        transfer : bool
            If True, the data cls are the average of the signal simulations, and
            noise cls are ignored.  If False, the data cls are either
            ``cls_data_null`` (for null tests) or ``cls_data``.  See
            ``get_masked_data`` for how these are computed.  The input noise is
            similarly either ``cls_noise_null`` or ``cls_noise``.
        do_noise : bool
            If True, return noise spectra and debiased spectra along with data.

        Returns
        -------
        obs : OrderedDict
            Dictionary of data cross spectra
        nell : OrderedDict
            Dictionary of noise cross spectra, or None if ``transfer`` is True.
        debias : OrderedDict
            Dictionary of debiased data cross spectra, or None if ``transfer``
            is True.
        """
        # select map pairs
        if map_tag is not None:
            map_tags = [map_tag]
        else:
            map_tags = self.map_tags
        map_pairs = pt.tag_pairs(map_tags)

        # select spectra
        tbeb = "tb" in self.specs
        if transfer or not tbeb:
            specs = self.specs[:4]
        else:
            specs = self.specs

        # obs depends on what you're computing
        if transfer:
            obs_quant = self.cls_signal
        elif self.null_run:
            if self.reference_type is not None:
                obs_quant = self.cls_data_sub_null
            else:
                obs_quant = self.cls_data_null
        elif self.template_type is not None:
            obs_quant = self.cls_data_clean
        else:
            obs_quant = self.cls_data

        # in case we're excluding some spectra or maps, repopulate obs dict
        obs = OrderedDict()
        for spec in specs:
            obs[spec] = OrderedDict()
            for xname in map_pairs:
                obs[spec][xname] = obs_quant[spec][xname]

        if not do_noise:
            return obs

        elif transfer:
            return obs, None, None

        nell = OrderedDict()
        debias = OrderedDict()

        # Nulls are debiased by average of S+N sims
        if self.null_run and self.cls_sim_null is not None:
            for spec in specs:
                nell[spec] = OrderedDict()
                debias[spec] = OrderedDict()
                for xname in map_pairs:
                    nell[spec][xname] = np.copy(self.cls_sim_null[spec][xname])
                    if self.reference_type is not None:
                        # signal term already subtracted with reference maps
                        debias[spec][xname] = np.copy(self.cls_noise_null[spec][xname])
                    else:
                        debias[spec][xname] = np.copy(self.cls_sim_null[spec][xname])

        # Non-nulls are debiased by average of N sims
        elif not self.null_run and self.cls_noise is not None:
            for spec in specs:
                nell[spec] = OrderedDict()
                debias[spec] = OrderedDict()
                for xname, (m0, m1) in map_pairs.items():
                    if m0 != m1:
                        # set non-auto noise to 0-- don't care to fit cross
                        # spectrum noise
                        nell[spec][xname] = np.zeros_like(self.cls_noise[spec][xname])
                    else:
                        nell[spec][xname] = np.copy(self.cls_noise[spec][xname])
                    debias[spec][xname] = np.copy(nell[spec][xname])

        else:
            nell = debias = None

        return obs, nell, debias

    def do_qb2cb(self, qb, inv_fish, wbl):
        """
        Compute binned output spectra and covariances by averaging
        the shape spectrum over each bin, and applying the appropriate
        ``qb`` bandpower amplitude.

        This method is used internally by ``fisher_calc``, and requires
        bin definitions to have been pre-loaded using ``get_bin_def``
        or ``get_transfer``.

        Arguments
        ---------
        qb : dict
            Bandpower amplitudes for each spectrum bin.
        inv_fish : array_like, (nbins, nbins)
            Inverse fisher matrix for computing the bin errors and covariance.
        wbl : dict
            Window functions for each qb

        Returns
        -------
        cb : dict of arrays
            Binned spectrum
        dcb : dict of arrays
            Binned spectrum error
        ellb : dict of arrays
            Average bin center
        cov : array_like, (nbins, nbins)
            Binned spectrum covariance
        qb2cb : dict
            The conversion matrix from ``qb`` to ``cb`` for each spectrum
            component, computed from the qb window functions
        wbl_cb : dict
            Window functions for each cb
        """
        qb2cb = OrderedDict()
        ellb = OrderedDict()
        cb = OrderedDict()
        wbl_cb = OrderedDict()

        # truncate to only bins for which there are window functions
        bin_index = pt.dict_to_index(self.bin_def)
        nbins = max([bin_index[stag][1] for stag in wbl])
        inv_fish = inv_fish[:nbins, :nbins]
        qb2cb_mat = np.zeros_like(inv_fish)

        # window function normalization
        ell = np.arange(self.lmax + 1)
        norm = (2.0 * ell + 1.0) / 4.0 / np.pi

        # normalization shape spectrum
        model = 1
        if not self.return_cls:
            model = np.ones_like(ell, dtype=float)
            model[1:] = 2.0 * np.pi / ell[1:] / (ell[1:] + 1)

        for stag, wbl1 in wbl.items():
            # compute conversion factors
            qb2cb[stag] = np.zeros((len(wbl1), len(wbl1)))
            cls_shape = self.cls_shape[stag][: len(ell)]
            v = norm * wbl1 * cls_shape
            bd = self.bin_def[stag]
            bw = self.bin_weights[stag]
            for idx, ((l, r), w) in enumerate(zip(bd, bw)):
                qb2cb[stag][:, idx] = np.sum(v[..., l:r] * w, axis=-1)

            # normalize for cb's or db's
            qb2cb[stag] /= np.sum(norm * wbl1 * model, axis=-1)

            # construct conversion matrix for covariance
            left, right = bin_index[stag]
            qb2cb_mat[left:right, left:right] = qb2cb[stag]

            # compute wbls for cb's
            wbl_cb[stag] = np.einsum("ij,jl->il", qb2cb[stag], wbl1)

            # check normalization
            self.log(
                "{} cb window function normalization: {}".format(
                    stag, np.sum(norm * wbl_cb[stag] * model, axis=-1)
                ),
                "debug",
            )

            # compute bin centers
            ellb[stag] = np.sum(norm * wbl_cb[stag] * model * ell, axis=-1)

            # compute cb's
            cb[stag] = np.einsum("ij,j->i", qb2cb[stag], qb[stag])

        # compute covariance and errors
        cov = np.einsum("ik,jl,kl->ij", qb2cb_mat, qb2cb_mat, inv_fish)
        dcb = pt.arr_to_dict(np.sqrt(np.diag(cov)), qb2cb)

        return cb, dcb, ellb, cov, qb2cb, wbl_cb

    def fisher_precalc(
        self,
        cbl,
        cls_input,
        cls_debias=None,
        likelihood=False,
        windows=False,
    ):
        """
        Pre-compute the D_ell and signal derivative matrices necessary for
        ``fisher_calc`` from the input data spectra.  This method requires bin
        definitions precomputed by ``get_bin_def`` or ``get_transfer``.

        Arguments
        ---------
        cbl : OrderedDict
            Cbl dict computed by ``bin_cl_template`` for a given
            shape spectrum.
        cls_input : OrderedDict
            Input spectra.  If computing a transfer function, this is the
            average ``cls_signal``.  If computing a null test, this is
            ``cls_data_null``, and otherwise it is ``cls_data``, for a single map or
            several input maps.
        cls_debias : OrderedDict
            If supplied, the debias spectrum is subtracted from the input.
        likelihood : bool
            If True, compute just Dmat_obs_b.  Otherwise, Dmat_obs and
            dSdqb_mat1 are also computed.
        windows : bool
            If True, compute dSdqb and Mll for constructing window functions.

        Returns
        -------
        Dmat_obs : OrderedDict
            De-biased D_ell matrix from ``cls_input``
        Dmat_obs_b : OrderedDict
            Biased D_ell matrix from ``cls_input`` (for likelihood)
        dSdqb_mat1 : OrderedDict
            Signal derivative matrix from Cbl
        Mmat : OrderedDict
            Mode mixing matrix (Kll' * Fl * Bl^2) for constructing
            window functions.

        .. note:: the output arrays are also stored as attributes of the
        parent object to avoid repeating the computation in ``fisher_calc``
        """
        num_maps = self.num_maps
        pol_dim = 3 if self.pol else 1
        dim1 = pol_dim * num_maps

        comps = []
        sig_comps = []
        for comp in self.components:
            if any([k.startswith(comp + "_") for k in cbl]):
                comps += [comp]
                if comp in self.signal_components:
                    sig_comps += [comp]

        specs = list(cls_input)

        if likelihood:
            Dmat_obs_b = OrderedDict()
            Dmat_obs = None
            dSdqb = None
            Mmat = None
        else:
            if windows:
                Dmat_obs = None
                Mmat = OrderedDict()
                for comp in sig_comps:
                    for spec in specs:
                        Mmat["{}_{}".format(comp, spec)] = OrderedDict()
                mll = getattr(self, "mll", None)
                if mll is None:
                    mll = self.kernel_precalc()
            else:
                Dmat_obs = OrderedDict()
                Mmat = None
            Dmat_obs_b = None
            dSdqb = OrderedDict()

        for xname, (m0, m1) in self.map_pairs.items():
            # transfer function doesn't have all the crosses
            if xname not in cls_input[specs[0]]:
                continue

            if likelihood:
                Dmat_obs_b[xname] = OrderedDict()
            elif windows:
                for comp in sig_comps:
                    for spec in specs:
                        Mmat["{}_{}".format(comp, spec)][xname] = OrderedDict()
            else:
                Dmat_obs[xname] = OrderedDict()

            for spec in specs:
                if likelihood:
                    # without bias subtraction for likelihood
                    Dmat_obs_b[xname][spec] = cls_input[spec][xname]
                elif windows:
                    for comp in sig_comps:
                        stag = "{}_{}".format(comp, spec)
                        Mmat[stag][xname][spec] = mll[stag][xname]
                        if spec in ["ee", "bb"]:
                            mspec = "bb" if spec == "ee" else "ee"
                            Mmat[stag][xname][mspec] = mll["{}_mix".format(stag)][xname]
                else:
                    if cls_debias is not None:
                        Dmat_obs[xname][spec] = (
                            cls_input[spec][xname] - cls_debias[spec][xname]
                        )
                    else:
                        Dmat_obs[xname][spec] = cls_input[spec][xname]

            if likelihood:
                continue

            for comp in comps:
                for spec in specs:
                    stag = "{}_{}".format(comp, spec)
                    if stag not in cbl:
                        continue
                    if xname not in cbl[stag]:
                        continue

                    dSdqb.setdefault(comp, OrderedDict()).setdefault(
                        xname, OrderedDict()
                    ).setdefault(spec, OrderedDict())
                    dSdqb[comp][xname][spec][spec] = cbl[stag][xname]

                    if spec in ["ee", "bb"]:
                        if stag + "_mix" not in cbl:
                            continue
                        mspec = "bb" if spec == "ee" else "ee"
                        mix_cbl = cbl[stag + "_mix"][xname]
                        dSdqb[comp][xname][spec][mspec] = mix_cbl

        return Dmat_obs, Dmat_obs_b, dSdqb, Mmat

    def clear_precalc(self):
        """
        Clear variables pre-computed with ``fisher_precalc``.
        """
        for k in ["Dmat_obs", "Dmat_obs_b", "dSdqb_mat1", "Mmat", "mll"]:
            if hasattr(self, k):
                delattr(self, k)
            setattr(self, k, None)

    def fisher_calc(
        self,
        qb,
        cbl,
        cls_input,
        cls_noise=None,
        cls_debias=None,
        cls_model=None,
        cond_noise=None,
        cond_criteria=None,
        likelihood=False,
        like_lmin=2,
        like_lmax=None,
        delta_beta_prior=None,
        null_first_cmb=False,
        use_precalc=True,
        windows=False,
        inv_fish=None,
    ):
        """
        Re-compute the Fisher matrix and qb amplitudes based on
        input data spectra.  This method is called iteratively
        by ``fisher_iterate``, and requires bin definitions precomputed
        by ``get_bin_def`` or ``get_transfer``.

        Arguments
        ---------
        qb : OrderedDict
            Bandpower amplitudes, typically computed in a previous call
            to this method.
        cbl : OrderedDict
            Cbl matrix computed by ``bin_cl_template`` for a given
            shape spectrum.
        cls_input : OrderedDict
            Input spectra.  If computing a transfer function,
            this is the average ``cls_signal``.  If computing a null
            test, this is ``cls_data_null``, and otherwise it is
            ``cls_data``, for a single map or several input maps.
        cls_noise : OrderedDict
            If supplied, the noise spectrum is applied to the model spectrum.
        cls_debias : OrderedDict
            If supplied, the debias spectrum is subtracted from the input.
        cls_model : OrderedDict
            Unbinned model spectrum computed from cbl
        cond_noise : float
            The level of regularizing noise to add to EE and BB diagonals.
        cond_criteria : float
            The maximum condition number allowed for Dmat1 to be acceptable
            for taking its inverse.
        likelihood : bool
            If True, return the likelihood for the given input bandpowers, shapes
            and data spectra.  Otherwise, computes output bandpowers and the fisher
            covariance for a NR iteration.
        like_lmin : int
            The minimum ell value to be included in the likelihood calculation
        like_lmax : int
            The maximum ell value to be included in the likelihood calculation
        delta_beta_prior : float
            The width of the prior on the additive change from beta_ref. If you
            don't want the code to fit for a spectral index different
            from beta_ref, set this to be a very small value (O(1e-10)).
        null_first_cmb : bool
            Keep first CMB bandpowers fixed to input shape (qb=1).
        use_precalc : bool
            If True, load pre-calculated terms stored from a previous iteration,
            and store for a future iteration.  Otherwise, all calculations are
            repeated.
        windows : bool
            If True, return W_bl window functions for each CMB qb.
        inv_fish : array_like
            Inverse Fisher matrix. If provided, don't need to recompute. Useful
            if just getting bandpower window functions.

        Returns
        -------
        qb : OrderedDict
            New bandpower amplitudes
        inv_fish : array_like
            Inverse Fisher correlation matrix over all bins
        -- or --
        likelihood : scalar
            Likelihood of the given input parameters.
        -- or --
        windows : OrderedDict
            qb window functions
        """
        if cond_criteria is None:
            cond_criteria = np.inf
        well_cond = False

        pol_dim = 3 if self.pol else 1

        dkey = "Dmat_obs_b" if likelihood else "Mmat" if windows else "Dmat_obs"

        if getattr(self, dkey, None) is None or not use_precalc:
            if use_precalc:
                self.clear_precalc()
            Dmat_obs, Dmat_obs_b, dSdqb_mat1, Mmat = self.fisher_precalc(
                cbl,
                cls_input,
                cls_debias=cls_debias,
                likelihood=likelihood,
                windows=windows,
            )
            if use_precalc:
                self.Dmat_obs = Dmat_obs
                self.Dmat_obs_b = Dmat_obs_b
                self.dSdqb_mat1 = dSdqb_mat1
                self.Mmat = Mmat
        else:
            if likelihood:
                Dmat_obs_b = self.Dmat_obs_b
            else:
                if windows:
                    Mmat = self.Mmat
                else:
                    Dmat_obs = self.Dmat_obs
                dSdqb_mat1 = self.dSdqb_mat1

        if windows:
            self.clear_precalc()

        delta_beta = 0.0
        if "delta_beta" in qb:
            delta_beta = qb["delta_beta"][0]

        if likelihood or windows or not cond_noise:
            well_cond = True
            cond_noise = None

        gmat_ell = OrderedDict()
        Dmat1 = OrderedDict()

        if cls_model is None:
            cls_model = self.get_model_spectra(
                qb, cbl, cls_noise=cls_noise, cond_noise=cond_noise
            )

        mkeys = list(cls_model)
        for xname, (m0, m1) in self.map_pairs.items():
            # transfer function does not have crosses
            if xname not in cls_model[mkeys[0]]:
                continue
            gmat_ell[xname] = self.gmat_ell[xname]

            if well_cond:
                for spec in self.specs:
                    # transfer function doesn't have eb/tb
                    if "total_{}".format(spec) not in cls_model:
                        continue
                    Dmat1.setdefault(xname, OrderedDict())
                    Dmat1[xname][spec] = cls_model["total_{}".format(spec)][xname]

        if well_cond:
            Dmat1_mat = pt.dict_to_dmat(Dmat1)

        # Set up dSdqb spectral index dependence
        if "delta_beta" in qb and not likelihood:
            # don't make changes to cached matrix
            dSdqb_mat1 = copy.deepcopy(dSdqb_mat1)
            dSdqb_mat1["delta_beta"] = OrderedDict()

            for xname in self.map_pairs:
                # transfer function does not have crosses
                if xname not in cls_model[mkeys[0]]:
                    continue

                dSdqb_mat1["delta_beta"][xname] = OrderedDict()

                # Add spectral index dependence
                # dSdqb now depends on qb (spec index) because
                # model is non-linear so cannot be precomputed.

                # get foreground at pivot point spectral index
                _, beta_scale, log_beta_scale = self.fg_scales[xname]
                # separable beta correction
                # beta_scale = beta_scale ** delta_beta  # non-linear
                beta_scale = 1 + log_beta_scale * delta_beta  # linear
                beta_scale0 = log_beta_scale / beta_scale

                # scale foreground model by frequency scaling adjusted for beta
                for s1, sdat in dSdqb_mat1["fg"][xname].items():
                    for s2, sdat2 in sdat.items():
                        sdat2 *= beta_scale

                    # with linearized derivative term, evaluated at input beta
                    dSdqb_mat1["delta_beta"][xname][s1] = OrderedDict(
                        {s1: cls_model["fg_{}".format(s1)][xname] * beta_scale0}
                    )

        # Set up Dmat -- if it's not well conditioned, add noise to the
        # diagonal until it is.
        cond_iter = 0
        while not well_cond:
            cls_model = self.get_model_spectra(
                qb, cbl, cls_noise=cls_noise, cond_noise=cond_noise
            )

            for xname, (m0, m1) in self.map_pairs.items():
                # transfer function does not have crosses
                if xname not in cls_model[mkeys[0]]:
                    continue

                for spec in self.specs:
                    # transfer function doesn't have eb/tb
                    if "total_{}".format(spec) not in cls_model:
                        continue
                    Dmat1.setdefault(xname, OrderedDict())
                    Dmat1[xname][spec] = cls_model["total_{}".format(spec)][xname]

            Dmat1_mat = pt.dict_to_dmat(Dmat1)

            cond = np.abs(
                np.linalg.cond(Dmat1_mat[:, :, self.lmin :].swapaxes(0, -1))
            ).max()

            if cond > cond_criteria and cond_noise:
                cond_iter += 1
                # cond_noise iteration factor found through trial and error
                cond_noise *= 10 ** (cond_iter / 100.0)
                if cond_iter == 1:
                    self.log(
                        "Condition criteria not met. "
                        "Max Cond={:.0f}, Thresh={:.0f}".format(cond, cond_criteria),
                        "debug",
                    )
            else:
                well_cond = True
                self.log(
                    "Condition criteria met. "
                    "Max Cond={:.0f}, Thresh={:.0f}, Iter={:d}".format(
                        cond, cond_criteria, cond_iter
                    ),
                    "debug",
                )
                if cond_noise is not None:
                    self.log("Cond_noise = {:.3e}".format(cond_noise), "debug")

        # construct arrays from dictionaries
        Dmat1 = Dmat1_mat
        gmat = pt.dict_to_dmat(gmat_ell)
        if likelihood:
            Dmat_obs_b = pt.dict_to_dmat(Dmat_obs_b)
        else:
            if not windows:
                Dmat_obs = pt.dict_to_dmat(Dmat_obs)
            # select only derivative terms for bins that are iterated
            bin_def = OrderedDict([(k, v) for k, v in self.bin_def.items() if k in qb])
            dSdqb_mat1 = pt.dict_to_dsdqb_mat(dSdqb_mat1, bin_def)

        # apply ell limits
        if likelihood:
            ell = slice(
                self.lmin if like_lmin is None else like_lmin,
                self.lmax + 1 if like_lmax is None else like_lmax + 1,
            )
        else:
            ell = slice(self.lmin, self.lmax + 1)
        Dmat1 = Dmat1[..., ell]
        if likelihood:
            Dmat_obs_b = Dmat_obs_b[..., ell]
        else:
            if not windows:
                Dmat_obs = Dmat_obs[..., ell]
            dSdqb_mat1 = dSdqb_mat1[..., ell]
        gmat = gmat[..., ell]

        lam, R = np.linalg.eigh(Dmat1.swapaxes(0, -1))
        del Dmat1
        bad = (lam <= 0).sum(axis=-1).astype(bool)
        if bad.sum():
            # exclude any ell's with ill-conditioned D matrix
            # this should happen only far from max like
            bad_idx = np.unique(np.where(bad)[0])
            bad_ells = np.arange(ell.start, ell.stop)[bad_idx]
            bads = (lam <= 0).sum(axis=0).astype(bool)
            bad_specs = np.array(
                [
                    "{}_{}".format(m, s)
                    for m in self.map_tags
                    for s in ["tt", "ee", "bb"]
                ]
            )[bads]
            self.log(
                "Found negative eigenvalues in specs {} at ells {}".format(
                    bad_specs, bad_ells
                ),
                "warning",
            )
            if likelihood:
                gmat[..., bad_idx] = 0
        inv_lam = 1.0 / lam
        Dinv = np.einsum("...ij,...j,...kj->...ik", R, inv_lam, R).swapaxes(0, -1)
        del inv_lam

        if likelihood:
            # log(det(D)) = tr(log(D)), latter is numerically stable
            # compute log(D) by eigenvalue decomposition per ell
            log_lam = np.log(lam)
            del lam
            Dlog = np.einsum("...ij,...j,...kj->...ik", R, log_lam, R).swapaxes(0, -1)
            del R, log_lam

        else:
            # optimized matrix multiplication
            # there is something super weird about this whole matrix operation
            # that causes the computation of mats to take four times as long
            # if mat1 is not computed.
            del lam, R
            eye = np.eye(len(gmat))
            mat1 = np.einsum("ij...,jk...->ik...", eye, Dinv)
            mat2 = np.einsum("klm...,ln...->knm...", dSdqb_mat1, Dinv)
            del Dinv
            mat = np.einsum("ik...,knm...->inm...", mat1, mat2)
            del mat1, mat2

        if likelihood:
            # compute log likelihood as tr(g * (D^-1 * Dobs + log(D)))
            arg = np.einsum("ij...,jk...->ik...", Dinv, Dmat_obs_b) + Dlog
            del Dinv, Dmat_obs_b, Dlog
            like = -np.einsum("iij,iij->", gmat, arg) / 2.0

            # include priors in likelihood
            if "delta_beta" in qb and delta_beta_prior is not None:
                chi = (delta_beta - self.delta_beta_fix) / delta_beta_prior
                like -= chi ** 2 / 2.0

            if null_first_cmb:
                for spec in self.specs:
                    stag = "cmb_{}".format(spec)
                    if stag not in qb:
                        continue
                    chi = (qb[stag][0] - 1) / np.sqrt(1e-10)
                    like -= chi ** 2 / 2.0

            return like

        # construct matrices for the qb and fisher terms,
        # and take the trace and sum over ell simultaneously
        if not windows or (windows and inv_fish is None):
            fisher = np.einsum("iil,ijkl,jiml->km", gmat, mat, dSdqb_mat1) / 2
            del dSdqb_mat1
        if not windows:
            qb_vec = np.einsum("iil,ijkl,jil->k", gmat, mat, Dmat_obs) / 2.0
            del gmat, mat, Dmat_obs

        if windows:

            if inv_fish is None:
                inv_fish = np.linalg.solve(fisher, np.eye(len(fisher)))

            # compute prefactors
            ells = np.arange(0, self.lmax + 1)
            norm = (2.0 * ells + 1.0) / 4.0 / np.pi

            # compute binning term
            arg = np.einsum("ij,kljm->klim", inv_fish, mat)
            del mat
            wbl = OrderedDict()
            bin_index = pt.dict_to_index(qb)

            # compute foreground scaling per frequency
            if "fg" in self.signal_components:
                fmat = OrderedDict()
                for xname, (a, b, c) in self.fg_scales.items():
                    fmat[xname] = OrderedDict()
                    amp = a
                    if "delta_beta" in qb:
                        amp *= 1.0 + delta_beta * c
                        # amp *= b ** delta_beta
                    fmat[xname] = OrderedDict([(s, amp) for s in self.specs])
                fmat = pt.dict_to_dmat(fmat, pol=self.pol)

            # compute window functions for each spectrum
            for k, (left, right) in bin_index.items():
                if k not in Mmat:
                    continue
                comp, spec = k.split("_", 1)

                # select bins for corresponding spectrum
                sarg = arg[:, :, left:right]

                # construct Mll' matrix
                marg = pt.dict_to_dmat(Mmat[k], pol=self.pol)[..., ell, :]

                # apply frequency scaling
                if comp == "fg":
                    marg = np.einsum("ij,ijlm->ijlm", fmat, marg)

                # qb window function
                wbl1 = np.einsum("iil,ijkl,jilm->km", gmat, sarg, marg) / 2.0 / norm
                del marg

                # bin weighting, allowing for overlapping bin edges
                chi_bl = np.zeros_like(norm)
                for (l, r), w in zip(self.bin_def[k], self.bin_weights[k]):
                    chi_bl[l:r] += w

                # check normalization
                cls_shape = self.cls_shape[k][: len(norm)]
                self.log(
                    "{} qb window function normalization: {}".format(
                        k, np.sum(wbl1 * norm * chi_bl * cls_shape, axis=-1)
                    ),
                    "debug",
                )

                wbl[k] = wbl1

            return wbl

        bin_index = pt.dict_to_index(qb)

        if "delta_beta" in qb and delta_beta_prior is not None:
            # XXX need documentation for what happens here
            # for imposing the delta_beta prior
            sl = slice(*(bin_index["delta_beta"]))
            d = 1.0 / delta_beta_prior ** 2
            qb_vec[sl] += d * self.delta_beta_fix
            fisher[sl, sl] += d

        if null_first_cmb:
            # blow up the fisher matrix for first bin
            # by implementing a tight prior
            for spec in self.specs:
                stag = "cmb_{}".format(spec)
                if stag in bin_index:  # check for transfer function
                    b0 = bin_index[stag][0]
                    fisher[b0, b0] += 1e10
                    qb_vec[b0] += 1e10

        # invert
        qb_vec = np.linalg.solve(fisher, qb_vec)
        inv_fish = np.linalg.solve(fisher, np.eye(len(qb_vec)))
        qb_vec = pt.arr_to_dict(qb_vec, qb)

        return qb_vec, inv_fish

    def fisher_iterate(
        self,
        cbl,
        map_tag=None,
        iter_max=200,
        converge_criteria=0.005,
        qb_start=None,
        transfer=False,
        save_iters=False,
        null_first_cmb=False,
        delta_beta_prior=None,
        cond_noise=None,
        cond_criteria=None,
        like_profiles=False,
        like_profile_sigma=3.0,
        like_profile_points=100,
        file_tag=None,
    ):
        """
        Iterate over the Fisher calculation to compute bandpower estimates
        assuming an input shape spectrum.

        Arguments
        ---------
        cbl : OrderedDict
            Cbl matrix computed from an input shape spectrum
        map_tag : str
            If not None, then iteration is performed over the spectra
            corresponding to the given map, rather over all possible
            combinations of map-map cross-spectra. In this case, the first
            dimension of the input cbl must be of size 1 (this is done
            automatically by calling ``bin_cl_template(..., map_tag=<map_tag>)``.
        iter_max : int
            Maximum number of iterations to perform.  if this limit is
            reached, a warning is issued.
        converge_criteria : float
            Maximum fractional change in qb that indicates convergence and
            stops iteration.
        qb_start : OrderedDict
            Initial guess at ``qb`` bandpower amplitudes.  If None, unity is
            assumed for all bins.
        transfer : bool
            If True, the input Cls passed to ``fisher_calc`` are the average
            of the signal simulations, and noise cls are ignored.
            If False, the input Cls are either ``cls_data_null``
            (for null tests) or ``cls_data``.  See ``get_masked_data`` for
            how these are computed.  The input noise is similarly either
            ``cls_noise_null`` or ``cls_noise``.
        save_iters : bool
            If True, the output data from each Fisher iteration are stored
            in an individual npz file.
        null_first_cmb : bool
            Keep first CMB bandpowers fixed to input shape (qb=1).
        delta_beta_prior : float
            The width of the prior on the additive change from beta_ref. If you
            don't want the code to fit for a spectral index different
            from beta_ref, set this to be a very small value (O(1e-10)).
        cond_noise : float
            The level of regularizing noise to add to EE and BB diagonals.
        cond_criteria : float
            The maximum condition number allowed for Dmat1 to be acceptable
            for taking its inverse.
        like_profiles : bool
            If True, compute profile likelihoods for each qb, leaving all
            others fixed at their maximum likelihood values.  Profiles are
            computed over a range +/--sigma as estimated from the diagonals
            of the inverse Fisher matrix.
        like_profile_sigma : float
            Range in units of 1sigma over which to compute profile likelihoods
        like_profile_points : int
            Number of points to sample along the likelihood profile
        file_tag : string
            If supplied, appended to the bandpowers filename.

        Returns
        -------
        data : dict
            The results of the Fisher iteration process.  This dictionary
            contains the fields::

                qb : converged bandpower amplitudes
                cb : output binned spectrum
                dcb : binned spectrum errors
                ellb : bin centers
                cov : spectrum covariance
                inv_fish : inverse fisher matrix
                fqb : fractional change in qb for the last iteration
                qb2cb : conversion factor
                cbl : Cbl matrix
                cls_model : unbinned model spectrum computed from Cbl
                bin_def : bin definition array
                cls_obs : observed input spectra
                cls_noise : noise spectra
                cls_shape : shape spectrum
                iters : number of iterations completed

            If ``transfer`` is False, this dictionary also contains::

                qb_transfer : transfer function amplitudes
                transfer : ell-by-ell transfer function
                nbins_res : number of residual bins


        Notes
        -----
        This method stores outputs to files with name 'transfer' for transfer
        function runs (if ``transfer = True``), otherwise with name
        'bandpowers'.  Outputs from each individual iteration, containing
        only the quantities that change with each step, are stored in
        separate files with the same name (and different index).
        """

        if transfer:
            null_first_cmb = False
            save_name = "transfer_wbins" if self.weighted_bins else "transfer"
        else:
            save_name = "bandpowers"

        # previous fqb iterations to monitor convergence and adjust conditioning
        prev_fqb = []
        cond_adjusted = False

        if qb_start is None:
            qb = OrderedDict()
            for k, v in self.bin_def.items():
                if transfer:
                    if "cmb" not in k or "eb" in k or "tb" in k:
                        continue
                if k == "delta_beta":
                    # qb_delta beta is a coefficient on the change from beta,
                    # so expect that it should be small if beta_ref is close
                    # (zeroes cause singular matrix problems)
                    qb[k] = [self.delta_beta_fix]
                elif k.startswith("res_"):
                    # res qb=0 means noise model is 100% accurate.
                    qb[k] = 1e-5 * np.ones(len(v))
                else:
                    # start by assuming model is 100% accurate
                    qb[k] = np.ones(len(v))
        else:
            qb = qb_start

        obs, nell, debias = self.get_data_spectra(map_tag=map_tag, transfer=transfer)

        bin_index = pt.dict_to_index(self.bin_def)

        success = False
        for iter_idx in range(iter_max):
            self.log(
                "Doing Fisher step {}/{}...".format(iter_idx + 1, iter_max), "info"
            )

            qb_new, inv_fish = self.fisher_calc(
                qb,
                cbl,
                obs,
                cls_noise=nell,
                cls_debias=debias,
                cond_noise=cond_noise,
                delta_beta_prior=delta_beta_prior,
                cond_criteria=cond_criteria,
                null_first_cmb=null_first_cmb,
            )

            qb_arr = pt.dict_to_arr(qb, flatten=True)
            qb_new_arr = pt.dict_to_arr(qb_new, flatten=True)
            dqb = qb_new_arr - qb_arr
            fqb = dqb / qb_arr
            max_fqb = np.nanmax(np.abs(fqb))

            prev_fqb.append(max_fqb)

            fnan = np.isnan(fqb)
            if fnan.any():
                (nanidx,) = np.where(fnan)
                self.log(
                    "Iter {}: Ignoring {} bins with fqb=nan: bins={}, qb_new={}, "
                    "qb={}".format(
                        iter_idx,
                        len(nanidx),
                        nanidx,
                        qb_new_arr[nanidx],
                        qb_arr[nanidx],
                    ),
                    "warning",
                )

            self.log("Max fractional change in qb: {}".format(max_fqb), "info")

            # put qb_new in original dict
            qb = copy.deepcopy(qb_new)
            cls_model = self.get_model_spectra(qb, cbl, cls_noise=nell, cond_noise=None)

            if "delta_beta" in qb:
                # get beta fit and beta error
                beta_fit = qb["delta_beta"][0] + self.beta_ref
                db_idx = slice(*bin_index["delta_beta"])
                beta_err = np.sqrt(np.diag(inv_fish[db_idx, db_idx]))[0]
            else:
                beta_fit = None
                beta_err = None

            if save_iters:
                # save only the quantities that change with each iteration
                out = dict(
                    map_tag=map_tag,
                    map_tags=self.map_tags,
                    iter_index=iter_idx,
                    bin_def=self.bin_def,
                    bin_weights=self.bin_weights,
                    cls_shape=self.cls_shape,
                    cls_obs=obs,
                    qb=qb,
                    fqb=fqb,
                    inv_fish=inv_fish,
                    cls_model=cls_model,
                    cbl=cbl,
                    cls_signal=self.cls_signal,
                    cls_noise=self.cls_noise,
                    Dmat_obs=self.Dmat_obs,
                    gmat_ell=self.gmat_ell,
                    extra_tag=file_tag,
                )

                if "fg" in self.components and not transfer:
                    out.update(
                        map_freqs=self.map_freqs,
                        freq_ref=self.freq_ref,
                        beta_ref=self.beta_ref,
                    )
                    if "delta_beta" in qb:
                        out.update(
                            beta_fit=beta_fit,
                            beta_err=beta_err,
                        )

                self.save_data(save_name, bp_opts=not transfer, **out)

            (nans,) = np.where(np.isnan(qb_new_arr))
            if len(nans):
                msg = "Found NaN values in qb bins {} at iter {}".format(nans, iter_idx)
                break

            if fnan.all():
                msg = (
                    "All bins have fqb=NaN at iter {}, "
                    "something has gone horribly wrong.".format(iter_idx)
                )
                break

            negs = np.where(np.diag(inv_fish) < 0)[0]
            if len(negs):
                self.log(
                    "Iter {}: Found negatives in inv_fish diagonal at locations "
                    "{}".format(iter_idx, negs),
                    "warning",
                )

            if np.nanmax(np.abs(fqb)) < converge_criteria:
                if not transfer:
                    # Calculate final fisher matrix without conditioning
                    self.log("Calculating final Fisher matrix.", "info")
                    _, inv_fish = self.fisher_calc(
                        qb,
                        cbl,
                        obs,
                        cls_noise=nell,
                        cls_debias=debias,
                        cond_noise=None,
                        delta_beta_prior=delta_beta_prior,
                        null_first_cmb=null_first_cmb,
                    )

                # If any diagonals of inv_fisher are negative, something went wrong
                negs = np.where(np.diag(inv_fish) < 0)[0]
                if len(negs):
                    self.log(
                        "Found negatives in inv_fish diagonal at locations "
                        "{}".format(negs),
                        "warning",
                    )

                success = True
                break

            else:
                msg = "{} {} did not converge in {} iterations".format(
                    "Multi-map" if map_tag is None else "Map {}".format(map_tag),
                    "transfer function" if transfer else "spectrum",
                    iter_max,
                )
                # Check the slope of the last ten fqb_maxpoints.
                # If there's not a downward trend, adjust conditioning
                # criteria to help convergence.
                if len(prev_fqb) <= 10 or transfer:
                    continue
                m, b = np.polyfit(np.arange(10), prev_fqb[-10:], 1)
                if m > 0:  # Not converging
                    # First, start from very little conditioning
                    if not cond_adjusted:
                        cond_criteria = 5e3
                        cond_adjusted = True
                        self.log(
                            "Iter {}: Not converging. Setting cond_criteria={}".format(
                                iter_idx, cond_criteria
                            ),
                            "warning",
                        )

                    elif cond_criteria > 100:
                        cond_criteria /= 2.0
                        self.log(
                            "Iter {}: Tightening condition criteria to help convergence. "
                            "cond_criteria={}".format(iter_idx, cond_criteria),
                            "warning",
                        )
                    else:
                        self.log(
                            "Iter {}: Can't reduce cond_criteria any more.".format(
                                iter_idx
                            ),
                            "warning",
                        )
                    # give it ten tries to start converging
                    prev_fqb = []

        # save and return
        out = dict(
            qb=qb,
            inv_fish=inv_fish,
            fqb=fqb,
            bin_def=self.bin_def,
            bin_weights=self.bin_weights,
            iters=iter_idx,
            success=success,
            map_tags=self.map_tags,
            converge_criteria=converge_criteria,
            cond_noise=cond_noise,
            cond_criteria=cond_criteria,
            null_first_cmb=null_first_cmb,
            apply_gcorr=self.apply_gcorr,
            weighted_bins=self.weighted_bins,
        )

        if "fg" in self.components and not transfer:
            out.update(
                map_freqs=self.map_freqs,
                freq_ref=self.freq_ref,
                beta_ref=self.beta_ref,
            )
            if "delta_beta" in qb:
                out.update(
                    delta_beta_prior=delta_beta_prior,
                    beta_fit=beta_fit,
                    beta_err=beta_err,
                )

        if self.debug:
            out.update(
                cbl=cbl,
                cls_obs=obs,
                cls_signal=self.cls_signal,
                cls_noise=self.cls_noise,
                cls_model=cls_model,
                cls_shape=self.cls_shape,
                cond_noise=cond_noise,
                Dmat_obs=self.Dmat_obs,
            )

        if not transfer:
            out.update(
                qb_transfer=self.qb_transfer,
                transfer=self.transfer,
            )
            if self.template_type is not None:
                out.update(template_alpha=self.template_alpha)

        if success and not transfer:
            # do one more fisher calc that doesn't include sample variance
            # set qb=very close to 0. 0 causes singular matrix problems.
            # don't do this for noise residual bins
            self.log("Calculating final Fisher matrix without sample variance.", "info")
            qb_zeroed = copy.deepcopy(qb)
            qb_new_ns = copy.deepcopy(qb)
            for comp in self.signal_components:
                for spec in self.specs:
                    stag = "{}_{}".format(comp, spec)
                    if stag not in qb_zeroed:
                        continue
                    qb_zeroed[stag][:] = 1e-20
                    qb_new_ns[stag][:] = 1.0
            if "delta_beta" in qb:
                qb_zeroed["delta_beta"][:] = 1e-20
                qb_new_ns["delta_beta"][:] = 0

            _, inv_fish_ns = self.fisher_calc(
                qb_zeroed,
                cbl,
                obs,
                cls_noise=nell,
                cls_debias=debias,
                cond_noise=None,
                delta_beta_prior=None,
                null_first_cmb=null_first_cmb,
            )

            out.update(
                invfish_nosampvar=inv_fish_ns,
            )

            # compute window functions for signal bins
            self.log("Calculating signal window functions", "info")
            wbl_qb = self.fisher_calc(
                qb,
                cbl,
                obs,
                cls_noise=nell,
                cls_debias=None,
                cond_noise=None,
                delta_beta_prior=delta_beta_prior,
                null_first_cmb=null_first_cmb,
                windows=True,
                inv_fish=inv_fish,
            )
            out.update(wbl_qb=wbl_qb)

            # compute bandpowers and covariances
            cb, dcb, ellb, cov, qb2cb, wbl_cb = self.do_qb2cb(qb, inv_fish, wbl_qb)
            _, dcb_ns, _, cov_ns, _, _ = self.do_qb2cb(qb, inv_fish_ns, wbl_qb)

            out.update(
                cb=cb,
                dcb=dcb,
                ellb=ellb,
                cov=cov,
                qb2cb=qb2cb,
                wbl_cb=wbl_cb,
                dcb_nosampvar=dcb_ns,
                cov_nosampvar=cov_ns,
            )

            if like_profiles:
                # compute bandpower likelihoods
                self.log("Calculating bandpower profile likelihoods", "info")
                max_like = self.fisher_calc(
                    qb,
                    cbl,
                    obs,
                    cls_noise=nell,
                    cond_noise=None,
                    delta_beta_prior=delta_beta_prior,
                    null_first_cmb=null_first_cmb,
                    likelihood=True,
                )

                dqb = pt.arr_to_dict(np.sqrt(np.abs(np.diag(inv_fish))), qb)
                qb_like = OrderedDict()
                cb_like = OrderedDict()

                for stag, qbs in qb.items():
                    qb_like[stag] = np.zeros(
                        (len(qbs), 2, like_profile_points), dtype=float
                    )
                    if "res" not in stag:
                        cb_like[stag] = np.zeros(
                            (len(qbs), 2, like_profile_points), dtype=float
                        )

                    for ibin, q in enumerate(qbs):
                        qb1 = copy.deepcopy(qb)
                        dq = dqb[stag][ibin] * like_profile_sigma
                        q_arr = np.linspace(q - dq, q + dq, like_profile_points)
                        like_arr = np.zeros_like(q_arr)
                        cb_arr = np.zeros_like(q_arr)

                        for iq, q1 in enumerate(q_arr):
                            qb1[stag][ibin] = q1
                            # use max likelihood qb's qb2cb to convert qb->cb
                            if "res" not in stag:
                                cb_arr[iq] = np.einsum(
                                    "ij,j->i", qb2cb[stag], qb1[stag]
                                )[ibin]
                            try:
                                like = self.fisher_calc(
                                    qb1,
                                    cbl,
                                    obs,
                                    cls_noise=nell,
                                    cond_noise=None,
                                    delta_beta_prior=delta_beta_prior,
                                    null_first_cmb=null_first_cmb,
                                    likelihood=True,
                                )
                            except np.linalg.LinAlgError:
                                like = np.nan

                            like_arr[iq] = like

                            self.log(
                                "{} bin {} delta qb {} delta like: {}".format(
                                    stag, ibin, q1 - q, like - max_like
                                ),
                                "debug",
                            )

                        qb_like[stag][ibin] = np.vstack([q_arr, like_arr])
                        if "res" not in stag:
                            cb_like[stag][ibin] = np.vstack([cb_arr, like_arr])

                out.update(max_like=max_like, qb_like=qb_like, cb_like=cb_like)

        if not success:
            save_name = "ERROR_{}".format(save_name)
            self.log(msg, "error")
            self.warn(msg)

        return self.save_data(
            save_name, map_tag=map_tag, bp_opts=not transfer, extra_tag=file_tag, **out
        )

    def get_transfer(
        self,
        converge_criteria=0.005,
        iter_max=200,
        save_iters=False,
        fix_bb_transfer=False,
    ):
        """
        Compute the transfer function from signal simulations created using the
        same spectrum as the input shape.

        This raises a ValueError if a negative transfer function amplitude is
        found.

        Arguments
        ---------
        converge_criteria : float
            Maximum fractional change in qb that indicates convergence and stops
            iteration.
        iter_max : int
            Maximum number of iterations to perform.  if this limit is reached,
            a warning is issued.
        save_iters : bool
            If True, the output data from each Fisher iteration are stored in an
            individual npz file.
        fix_bb_transfer : bool
            If True, after transfer functions have been calculated, impose the
            BB xfer is exactly equal to the EE transfer.

        Returns
        -------
        transfer : OrderedDict
            Ell-by-ell transfer function for each map

        Notes
        -----
        This method is called at the 'transfer' checkpoint, and loads or saves a
        data dictionary named 'transfer_all' with the following entries:

            nbins:
                number of bins
            bin_def:
                bin definition dictionary (see ``get_bin_def``)
            qb_transfer:
                binned transfer function for each map and spectrum component
            transfer:
                ell-by-ell transfer function for each map and spectrum component

        Additionally the final output of ``fisher_iterate`` is stored in a
        dictionary called ``transfer_map<idx>`` for each map.
        """
        comps = list(self.signal_components)

        opts = dict(
            converge_criteria=converge_criteria,
            fix_bb_transfer=fix_bb_transfer,
            apply_gcorr=self.apply_gcorr,
            weighted_bins=self.weighted_bins,
        )

        save_name = "transfer_all"
        if self.weighted_bins:
            save_name = "{}_wbins".format(save_name)

        ret = self.load_data(
            save_name,
            "transfer",
            to_attrs=False,
            value_ref=opts,
        )

        # function for converting qb's to ell-by-ell transfer function
        def expand_transfer(qb_transfer, bin_def, check_only=False):
            if not check_only:
                transfer = OrderedDict()
            if not check_only and "fg" in comps:
                for stag in [k for k in qb_transfer if k.startswith("cmb_")]:
                    ftag = stag.replace("cmb_", "fg_")
                    qb_transfer[ftag] = OrderedDict()
                    for m0, v in qb_transfer[stag].items():
                        cb = np.mean(bin_def[stag], axis=1)
                        fb = np.mean(bin_def[ftag], axis=1)
                        vint = np.interp(fb, np.append([0], cb), np.append([0], v))
                        qb_transfer[ftag][m0] = vint
            for comp in comps:
                for spec in self.specs:
                    stag = "{}_{}".format(comp, spec)
                    if stag not in bin_def or stag not in qb_transfer:
                        raise KeyError(stag)
                    if not check_only:
                        transfer[stag] = OrderedDict()
                    bd = bin_def[stag]
                    if len(self.bin_def[stag]) != len(bd):
                        raise ValueError(
                            "Found {} transfer bins for component {}, expected {}".format(
                                len(self.bin_def[stag]), len(bd)
                            )
                        )
                    for m0 in self.map_tags:
                        if m0 not in qb_transfer[stag]:
                            raise KeyError(m0)
                        lq = len(qb_transfer[stag][m0])
                        lb = len(self.bin_def[stag])
                        if lq != lb:
                            raise ValueError(
                                "Found {} transfer bins for component {} map {}, expected {}".format(
                                    lq, stag, m0, lb
                                )
                            )
                        if check_only:
                            continue
                        if spec == "bb" and fix_bb_transfer:
                            v = transfer["{}_ee".format(comp)][m0]
                        elif spec in ["tb", "eb"] and stag not in qb_transfer:
                            staga, stagb = ["{}_{}{}".format(comp, s, s) for s in spec]
                            v = np.sqrt(
                                np.abs(transfer[staga][m0] * transfer[stagb][m0])
                            )
                        else:
                            v = pt.expand_qb(qb_transfer[stag][m0], bd, self.lmax + 1)
                        transfer[stag][m0] = v
            if not check_only:
                return transfer

        if ret is not None:
            try:
                check_only = ret.get("transfer", None) is not None
                if "fg" in comps:
                    check_only &= any(k.startswith("fg_") for k in ret["qb_transfer"])
                xfer = expand_transfer(ret["qb_transfer"], ret["bin_def"], check_only)
            except:
                ret = None
            else:
                self.qb_transfer = ret["qb_transfer"]
                self.transfer = xfer or ret["transfer"]
                return self.transfer

        self.qb_transfer = OrderedDict()
        for spec in self.specs:
            self.qb_transfer["cmb_" + spec] = OrderedDict()

        success = True
        msg = ""

        for im0, m0 in enumerate(self.map_tags):
            if not self.fit_transfer[self.map_tags_orig[im0]]:
                for spec in self.specs:
                    self.qb_transfer["cmb_{}".format(spec)][m0] = np.ones(
                        self.nbins_cmb // len(self.specs)
                    )
                self.log("Setting map {} transfer to unity".format(m0), "info")
                continue

            self.log(
                "Computing transfer function for map {}/{}".format(
                    im0 + 1, self.num_maps
                ),
                "info",
            )
            self.clear_precalc()
            cbl = self.bin_cl_template(map_tag=m0, transfer=True)
            ret = self.fisher_iterate(
                cbl,
                m0,
                transfer=True,
                iter_max=iter_max,
                converge_criteria=converge_criteria,
                save_iters=save_iters,
            )
            qb = ret["qb"]

            success = success and ret["success"]
            if not ret["success"]:
                msg = "Error in fisher_iterate for map {}".format(m0)

            # fix negative amplitude bins
            for k, v in qb.items():
                if np.any(v < 0):
                    (negbin,) = np.where(v < 0)
                    self.warn(
                        "Transfer function amplitude {} < 0"
                        "for {} bin {} of map {}".format(v, k, negbin, m0)
                    )
                    # XXX cludge
                    # This happens in first bin
                    # try linear interp between zero and next value
                    try:
                        qb[k][negbin] = qb[k][negbin + 1] / 2.0
                        self.warn(
                            "Setting Transfer function in negative bin to small "
                            "positive. This is probably due to choice of bins or "
                            "insufficient number of signal sims"
                        )
                    except Exception as e:
                        msg = "Unable to adjust negative bins for map {}: {}".format(
                            m0, str(e)
                        )
                        success = False

            # Set EB/TB qb transfers to geometric means of components
            if len(self.specs) > 4:
                qb["cmb_eb"] = np.sqrt(np.abs(qb["cmb_ee"] * qb["cmb_bb"]))
                qb["cmb_tb"] = np.sqrt(np.abs(qb["cmb_tt"] * qb["cmb_bb"]))

            for stag, qbdat in qb.items():
                self.qb_transfer[stag][m0] = qbdat

        if success:
            self.transfer = expand_transfer(self.qb_transfer, self.bin_def)
        else:
            self.transfer = None

        self.save_data(
            "{}{}".format("" if success else "ERROR_", save_name),
            from_attrs=["nbins", "bin_def", "qb_transfer", "map_tags", "transfer"],
            cls_shape=self.cls_shape,
            success=success,
            **opts
        )

        if not success:
            raise RuntimeError("Error computing transfer function: {}".format(msg))

        return self.transfer

    def get_bandpowers(
        self,
        map_tag=None,
        converge_criteria=0.005,
        iter_max=200,
        return_qb=False,
        save_iters=False,
        delta_beta_prior=None,
        cond_noise=None,
        cond_criteria=None,
        null_first_cmb=False,
        return_cls=False,
        like_profiles=False,
        like_profile_sigma=3.0,
        like_profile_points=100,
        file_tag=None,
    ):
        """
        Compute the maximum likelihood bandpowers of the data, assuming
        a given input spectrum shape.  Requires the transfer function to
        have been computed and loaded using ``get_transfer``.

        Arguments
        ---------
        map_tag : string
            If not None, then iteration is performed over the spectra
            corresponding to the given map, rather over all possible
            combinations of map-map cross-spectra. In this case, the first
            dimension of the input cbl must be of size 1 (this is done
            automatically by calling ``bin_cl_template(..., map_tag=<map_tag>)``.
        converge_criteria : float
            Maximum fractional change in qb that indicates convergence and
            stops iteration.
        iter_max : int
            Maximum number of iterations to perform.  if this limit is
            reached, a warning is issued.
        return_qb : bool
            If True, only the maximum likelihood ``qb`` values are returned.
            Otherwise, the complete output dictionary is returned.
        save_iters : bool
            If True, the output data from each Fisher iteration are stored
            in an individual npz file.
        delta_beta_prior : float
            The width of the prior on the additive change from beta_ref. If you
            don't want the code to fit for a spectral index different
            from beta_ref, set this to be a very small value (O(1e-10)).
        cond_noise : float
            The level of regularizing noise to add to EE and BB diagonals.
        cond_criteria : float
            The maximum condition number allowed for Dmat1 to be acceptable
            for taking its inverse.
        null_first_cmb : bool
            Keep first CMB bandpowers fixed to input shape (qb=1).
        return_cls : bool
            If True, return C_ls rather than D_ls
        like_profiles : bool
            If True, compute profile likelihoods for each qb, leaving all
            others fixed at their maximum likelihood values.  Profiles are
            computed over a range +/--sigma as estimated from the diagonals
            of the inverse Fisher matrix.
        like_profile_sigma : float
            Range in units of 1sigma over which to compute profile likelihoods
        like_profile_points : int
            Number of points to sample along the likelihood profile
        file_tag : string
            If supplied, appended to the bandpowers filename.

        Returns
        -------
        data : dict
            Dictionary of maximum likelihood quantities, as output by
            ``fisher_iterate``.
        -- or --
        qb, inv_fish : array_like
            Maximum likelihood bandpower amplitudes and fisher covariance.

        Notes
        -----
        This method is called at the 'bandpowers' checkpoint, and loads or
        saves a data dictionary named 'bandpowers' with the quantities
        returned by ``fisher_iterate``.
        """

        save_name = "bandpowers"

        fish_shape = (len(pt.dict_to_arr(self.bin_def)),) * 2

        # check all options that require rerunning fisher iterations
        opts = dict(
            converge_criteria=converge_criteria,
            cond_noise=cond_noise,
            cond_criteria=cond_criteria,
            null_first_cmb=null_first_cmb,
            apply_gcorr=self.apply_gcorr,
            weighted_bins=self.weighted_bins,
        )

        # foreground fitting
        if "fg" in self.components:
            opts.update(
                freq_ref=self.freq_ref,
                beta_ref=self.beta_ref,
            )
            if "delta_beta" in self.bin_def:
                # priors on frequency spectral index
                self.delta_beta_fix = 1.0e-8
                opts.update(delta_beta_prior=delta_beta_prior)

        if self.template_type is not None:
            opts.update(template_alpha=self.template_alpha)
        self.return_cls = return_cls

        ret = self.load_data(
            save_name,
            "bandpowers",
            bp_opts=True,
            to_attrs=False,
            shape=fish_shape,
            shape_ref="inv_fish",
            map_tag=map_tag,
            value_ref=opts,
            extra_tag=file_tag,
        )
        if ret is not None:
            if return_qb:
                return ret["qb"], ret["inv_fish"]
            return ret

        self.clear_precalc()

        cbl = self.bin_cl_template(map_tag=map_tag)

        ret = self.fisher_iterate(
            cbl,
            map_tag,
            iter_max=iter_max,
            converge_criteria=converge_criteria,
            save_iters=save_iters,
            cond_noise=cond_noise,
            cond_criteria=cond_criteria,
            null_first_cmb=null_first_cmb,
            delta_beta_prior=delta_beta_prior,
            like_profiles=like_profiles,
            like_profile_sigma=like_profile_sigma,
            like_profile_points=like_profile_points,
            file_tag=file_tag,
        )

        if not ret["success"]:
            raise RuntimeError("Error computing bandpowers")

        # return
        if return_qb:
            return ret["qb"], ret["inv_fish"]
        return ret

    def get_likelihood(
        self,
        qb,
        inv_fish,
        map_tag=None,
        null_first_cmb=False,
        lmin=33,
        lmax=250,
        mcmc=True,
        alpha_tags="all",
        beam_tags="all",
        r_prior=[0, np.inf],
        alpha_prior=[0, np.inf],
        res_prior=None,
        beam_prior=[0, 1],
        num_walkers=50,
        num_steps=20000,
        converge_criteria=0.01,
        reset_backend=None,
        file_tag=None,
        r_specs=["ee", "bb"],
        template_specs=["ee", "bb", "eb"],
    ):
        """
        Explore the likelihood, optionally with an MCMC sampler.

        Arguments
        ---------
        qb : OrderedDict
            Bandpower parameters previously computed by Fisher iteration.
        inv_fish : array_like
            Inverse Fisher matrix computed with the input qb's.
        map_tag : string
            If not None, then the likelihood is sampled using the spectra
            corresponding to the given map, rather over all possible
            combinations of map-map cross-spectra.  The input qb's and inv_fish
            must have been computed with the same option.
        null_first_cmb : bool
            Keep first CMB bandpowers fixed to input shape (qb=1).
        lmin : int
            The minimum ell value to be included in the likelihood calculation
        lmax : int
            The maximum ell value to be included in the likelihood calculation
        mcmc : bool
            If True, sample the likelihood using an MCMC sampler.  Remaining
            options determine parameter space and sampler configuration.
        alpha_tags : list of strings
            List of map tags from which foreground template maps should be
            subtracted.  These should be the original map tags, not
            those generated for chunk sets.  If "all", then all map tags
            used in the template subtraction are included, as determined by
            the keys in the `template_alpha` attribute.  If None, then the alpha
            parameters are not included in the likelihood.
        beam_tags : list of strings
            List of map tags from which beam error envelopes should be
            marginalized over. These should be the original map tags, not
            those generated for chunk sets.  If "all", then all available map
            tags in the dataset are included.  If None, then the beam error
            parameters are not included in the likelihood.
        r_prior : 2-list or None
            Prior upper and lower bound on tensor to scalar ratio.  If None, the
            fiducial shape spectrum is assumed, and the r parameter space is not
            varied.
        alpha_prior : 2-list or None
            Prior upper and lower bound on template coefficients.  If None, the
            alpha parameter space is not varied.
        res_prior : 2-list or none
            Prior upper and lower bound on residual qbs.  If None, the
            res parameter space is not varied.
        beam_prior : 2-list or none
            Prior mean and width of gaussian width on beam error (when
            multiplied by beam error envelope).  If None, the
            beam parameter space is not varied.
        num_walkers : int
            Number of unique walkers with which to sample the parameter space.
        num_steps : int
            Maximum number of steps each walker can take in sampling the
            parameter space.
        converge_criteria : float
            Convergence criteria for likelihood MCMC chains
        reset_backend : bool
            If True, clear the backend buffer before sampling.  If False,
            samples are appended to the existing buffer.  If not supplied,
            set to True if the checkpoint has been forced to be rerun.
        file_tag : string
            If supplied, appended to the likelihood filename.
        r_specs : list
            Which spectra to use in the r likelihood.
        template_specs : list
            Which spectra to use for alpha in the likelihood.
        """

        # Check if OMP threads > 1. It actually slows the code way down to have
        # more threads. Break instead.
        assert os.getenv("OMP_NUM_THREADS") in [
            None,
            "1",
        ], "OMP threads must be set to 1 for likelihood"

        for x in [
            r_prior,
            alpha_prior,
            res_prior,
            beam_prior,
        ]:
            if x is not None:
                x[:] = [float(x[0]), float(x[1])]

        r_specs = [x.lower() for x in r_specs]
        template_specs = [x.lower() for x in template_specs]

        save_name = "like_mcmc"
        if not mcmc:
            alpha_prior = None
            res_prior = None
            beam_prior = None

        # no template cleaning if there aren't any templates specified
        if getattr(self, "template_type", None) is None:
            alpha_prior = None

        # null out unused priors
        self.template_alpha = getattr(self, "template_alpha", None)
        if self.template_alpha is None or all(
            [x is None for x in self.template_alpha.values()]
        ):
            alpha_prior = None

        # count alpha parameters to fit
        if alpha_tags is None:
            alpha_tags = []
        elif alpha_tags == "all":
            alpha_tags = list(self.template_alpha) if self.template_alpha else []
        alpha_tags = [x for x in alpha_tags if x in self.map_tags_orig]
        if not len(alpha_tags):
            alpha_prior = None

        num_alpha = 0
        if alpha_prior is not None:
            num_alpha = len(alpha_tags)

        # count beam parameters to fit
        if beam_tags is None:
            beam_tags = []
        elif beam_tags == "all":
            beam_tags = list(self.map_tags_orig)
        beam_tags = [x for x in beam_tags if x in self.map_tags_orig]
        if not len(beam_tags):
            beam_prior = None

        num_beam = 0
        if beam_prior is not None:
            num_beam = len(beam_tags)

        if not any([k.startswith("res_") for k in qb]):
            res_prior = None

        # bookkeeping: ordered priors
        priors = {
            "r_prior": r_prior,
            "alpha_prior": alpha_prior,
            "res_prior": res_prior,
            "beam_prior": beam_prior,
        }
        # priors on quantities that affect Dmat_obs or gmat (precalculated)
        obs_priors = [alpha_prior]

        # check parameter space
        if all([x is None for x in priors.values()]):
            raise RuntimeError("Empty parameter space")

        out = dict(
            r_prior=r_prior,
            alpha_prior=alpha_prior,
            res_prior=res_prior,
            beam_prior=beam_prior,
            alpha_tags=alpha_tags,
            num_walkers=num_walkers,
            null_first_cmb=null_first_cmb,
            apply_gcorr=self.apply_gcorr,
            weighted_bins=self.weighted_bins,
            lmin=lmin,
            lmax=lmax,
        )

        if mcmc and reset_backend is None:
            ret = self.load_data(
                save_name,
                "likelihood",
                bp_opts=True,
                to_attrs=False,
                map_tag=map_tag,
                value_ref=out,
                extra_tag=file_tag,
            )
            if ret is not None and ret.get("converged", False):
                if converge_criteria >= ret.get("converge_criteria", 0.01):
                    return ret
            if ret is not None:
                for pname, pval in priors.items():
                    if np.all(pval != ret.get(pname, None)):
                        ret = None
            # clear chain cache if rerunning, otherwise append to chain by default
            reset_backend = ret is None

        out.update(converge_criteria=converge_criteria)

        # save state
        if mcmc and reset_backend:
            self.save_data(
                save_name, map_tag=map_tag, extra_tag=file_tag, bp_opts=True, **out
            )

        # clear pre-computed quantities
        self.clear_precalc()
        use_precalc = all([x is None for x in obs_priors])

        cls_input, cls_noise, cls_debias = self.get_data_spectra()

        # extract residual bins, ignoring bins outside of lmin/lmax
        if res_prior is not None:
            bin_def_orig = copy.deepcopy(self.bin_def)
            nbins_res_orig = self.nbins_res
            qb_res = OrderedDict()
            num_res = 0
            for k in list(qb):
                if k.startswith("res_"):
                    bd = self.bin_def[k]
                    good = np.where((bd[:, 1] > lmin) & (bd[:, 0] < lmax))[0]
                    # use all qb res in range lmin, lmax
                    self.bin_def[k] = bd[good]
                    v = qb.pop(k)[good]
                    num_res += len(v)
                    qb_res[k] = v
            self.nbins_res = num_res

        # set CMB model bandpowers to unity, since we are computing
        # the likelihood of this model given the data
        if r_prior is None:
            self.log("Computing model spectrum", "debug")
            self.warn("Beam variation not implemented for case of no r fit")
            cbl = self.bin_cl_template(map_tag=map_tag)
            cls_model = self.get_model_spectra(qb, cbl, cls_noise=cls_noise)
        else:
            qb = copy.deepcopy(qb)
            for spec in self.specs:
                stags = ["{}_{}".format(c, spec) for c in self.signal_components]
                for stag in stags:
                    if stag not in qb:
                        continue
                    qb[stag] = np.ones_like(qb[stag])
            qb_cmb = OrderedDict(
                [(k, v) for k, v in qb.items() if k.startswith("cmb_")]
            )

            self.log("Computing r model spectrum", "debug")
            cls_shape_scalar = self.get_signal_shape(
                r=1.0, save=False, component="scalar"
            )
            if "fg" in self.signal_components:
                cls_shape_scalar.update(
                    {k: cls for k, cls in self.cls_shape.items() if k.startswith("fg_")}
                )

            cls_shape_tensor = self.get_signal_shape(
                r=1.0, save=False, component="tensor"
            )

            # load tensor and scalar terms separately
            cbl_scalar = self.bin_cl_template(cls_shape_scalar, map_tag)
            cls_model_scalar = self.get_model_spectra(
                qb, cbl_scalar, cls_noise=cls_noise
            )
            cbl_tensor = self.bin_cl_template(cls_shape_tensor, map_tag)
            cls_model_tensor = self.get_model_spectra(qb_cmb, cbl_tensor)
            if beam_prior is not None:
                # load beam error term for tensor and scalar
                cbl_scalar_beam = self.bin_cl_template(
                    cls_shape_scalar, map_tag, beam_error=True
                )
                cls_mod_scal_beam = self.get_model_spectra(qb, cbl_scalar_beam)
                cbl_tensor_beam = self.bin_cl_template(
                    cls_shape_tensor, map_tag, beam_error=True
                )
                cls_mod_tens_beam = self.get_model_spectra(qb_cmb, cbl_tensor_beam)

            cbl = copy.deepcopy(cbl_scalar)
            cls_model = copy.deepcopy(cls_model_scalar)

        # XXX TODO
        # how to marginalize over the garbage bin?

        def parse_params(theta):
            """
            Parse array of parameters into a dict
            """
            params = {}
            if r_prior is not None:
                params["r"] = theta[0]
                theta = theta[1:]
            if alpha_prior is not None:
                params["alpha"] = theta[:num_alpha]
                theta = theta[num_alpha:]
            if res_prior is not None:
                params["res"] = theta[:num_res]
                theta = theta[num_res:]
            if beam_prior is not None:
                params["beam"] = theta[:num_beam]
                theta = theta[num_beam:]
            if len(theta):
                raise ValueError("Too many parameters to parse")
            return params

        def log_prior(
            r=None,
            alpha=None,
            res=None,
            beam=None,
        ):
            """
            Log prior function constructed from input options
            """
            values = {
                "r_prior": r,
                "alpha_prior": alpha,
                "res_prior": res,
            }
            for v, pval in values.items():
                prior = priors[v]
                if pval is not None and prior is not None:
                    if np.any(pval < prior[0]) or np.any(pval > prior[1]):
                        return -np.inf

            values_gauss = {
                "beam_prior": beam,
            }
            # for beam, use gaussian prior
            log_prob = 0.0
            for v, pval in values_gauss.items():
                prior = priors[v]
                if pval is not None and prior is not None:
                    pval = np.atleast_1d(pval)
                    norm = np.log(1.0 / (prior[1] * np.sqrt(2 * np.pi)))
                    chi = (pval - prior[0]) / prior[1]
                    log_prob += np.sum(norm - chi ** 2 / 2.0)

            return log_prob

        def log_like(
            r=None,
            alpha=None,
            res=None,
            beam=None,
        ):
            """
            Log likelihood function constructed from input options
            """
            cls_model0 = copy.deepcopy(cls_model)

            # compute new template subtracted data spectra
            if alpha is None:
                clsi = cls_input
            else:
                self.get_masked_data(
                    update_template=True,
                    template_alpha=OrderedDict(zip(alpha_tags, alpha)),
                    template_specs=template_specs,
                )
                clsi = self.get_data_spectra(do_noise=False)

            if beam is not None:
                beam = dict(zip(beam_tags, beam))
                beam_coeffs = dict()
                for xname, (m0, m1) in self.map_pairs_orig.items():
                    d = {}
                    b0, b1 = [beam.get(m, None) for m in (m0, m1)]
                    if b0 is not None:
                        d["b1"] = b0
                    if b1 is not None:
                        d["b2"] = b1
                        if b0 is not None:
                            d["b3"] = b0 * b1
                    beam_coeffs[xname] = d

            # compute new signal shape by scaling tensor component by r
            # XXX handle beam fg terms here too
            if r is not None:
                for stag, d in cls_model0.items():
                    comp, spec = stag.split("_", 1)
                    if spec not in r_specs or comp not in ["cmb", "total"]:
                        continue
                    ctag = "cmb_{}".format(spec)
                    for xname, dd in d.items():
                        dd[:] = (
                            cls_model_scalar[stag][xname]
                            + r * cls_model_tensor[ctag][xname]
                        )

                        if beam is None:
                            continue
                        beam_term = 0
                        for bn, bc in beam_coeffs[xname].items():
                            beam_term += bc * (
                                cls_mod_scal_beam[ctag][xname][bn]
                                + r * cls_mod_tens_beam[ctag][xname][bn]
                            )
                        dd[:] += beam_term

            elif beam is not None:
                for stag, d in cls_model0.items():
                    comp, spec = stag.split("_", 1)
                    if spec not in r_specs or comp not in ["cmb", "total"]:
                        continue
                    ctag = "cmb_{}".format(spec)
                    for xname, dd in d.items():
                        beam_term = 0
                        for bn, bc in beam_coeffs[xname].items():
                            beam_term += bc * cls_mod_scal_beam[ctag][xname][bn]
                        dd[:] = cls_model_scalar[stag][xname] + beam_term

            # compute noise model terms
            if res is None:
                clsm = cls_model0
            else:
                res = pt.arr_to_dict(res, qb_res)
                clsm = copy.deepcopy(cls_model0)
                cls_res = self.get_model_spectra(res, cbl)
                for stag, d in cls_res.items():
                    if stag not in clsm:
                        clsm[stag] = OrderedDict()
                    for xname, dd in d.items():
                        if xname not in clsm[stag]:
                            clsm[stag][xname] = dd
                        else:
                            clsm[stag][xname] += dd

            # compute likelihood
            like = self.fisher_calc(
                qb,
                cbl,
                clsi,
                cls_noise=cls_noise,
                cls_debias=cls_debias,
                cls_model=clsm,
                null_first_cmb=null_first_cmb,
                likelihood=True,
                use_precalc=use_precalc,
                like_lmin=lmin,
                like_lmax=lmax,
            )
            return like

        def log_prob(theta):
            """
            Log posterior probability from prior and likelihood

            Returns log_prior with each step
            """
            params = parse_params(theta)
            prior = log_prior(**params)
            if not np.isfinite(prior):
                return -np.inf, -np.inf
            like = log_like(**params)
            if not np.isfinite(like):
                return -np.inf, prior
            return prior + like, prior

        # initial values
        x0 = []
        brute_force = True if not mcmc else False  # only vary r
        if r_prior is not None:
            x0 += [0.01]
        if alpha_prior is not None:
            alphas = [self.template_alpha[tag] for tag in alpha_tags]
            x0 += [0.01 if a == 0 else a for a in alphas]
            brute_force = False
        if res_prior is not None:
            x0 += list(pt.dict_to_arr(qb_res, flatten=True))
            brute_force = False
        if beam_prior is not None:
            # add a beam term for each frequency
            x0 += [0.01] * len(beam_tags)
            brute_force = False

        ndim = len(x0)
        if ndim * 2 > num_walkers:
            num_walkers = int(np.round(ndim / float(num_walkers)) * num_walkers * 2)
            self.warn(
                "Found {} parameters, increasing number of MCMC walkers to {}".format(
                    ndim, num_walkers
                )
            )
        x0 = np.array(x0)[None, :] * (1 + 1e-4 * np.random.randn(num_walkers, len(x0)))

        if brute_force or (r_prior is not None and ndim == 1):
            self.log("Computing brute-force r profile likelihood", "info")
            likefile = self.get_filename(
                save_name, ext=".txt", map_tag=map_tag, extra_tag=file_tag, bp_opts=True
            )
            rs = np.linspace(0, 3, 500)
            likes = np.zeros_like(rs)
            for idx, r in enumerate(rs):
                like = log_like(r=r)
                if idx % 20 == 0:
                    self.log("r = {:.3f}, loglike = {:.2f}".format(r, like), "debug")
                likes[idx] = like
            header = "{} r likelihood\nColumns: r, loglike".format(
                "Multi-map" if map_tag is None else "Map {}".format(map_tag)
            )
            np.savetxt(likefile, np.column_stack((rs, likes)), header=header)

        if not mcmc:
            return [rs, likes]

        # run chains!
        import emcee

        # setup sampler output file
        filename = self.get_filename(
            save_name, ext=".h5", map_tag=map_tag, extra_tag=file_tag, bp_opts=True
        )
        backend_exists = os.path.exists(filename)
        backend = emcee.backends.HDFBackend(filename)
        if backend_exists and backend.shape != (num_walkers, ndim):
            self.warn(
                "Expected backend of shape ({}, {}), found {}. Resetting".format(
                    num_walkers, ndim, backend.shape
                )
            )
            reset_backend = True
        if reset_backend:
            backend.reset(num_walkers, ndim)

        # initialize sampler
        self.log("Initializing sampler", "info")
        sampler = emcee.EnsembleSampler(num_walkers, ndim, log_prob, backend=backend)
        if not reset_backend and backend_exists:
            # grab the last sample if appending to an existing run
            x0 = sampler.run_mcmc(None, 1)

        # track autocorrelation time
        old_tau = np.inf
        converged = False

        self.log(
            "Starting {} iterations with {} parameters".format(num_steps, ndim), "info"
        )
        for sample in sampler.sample(x0, iterations=num_steps):
            if not sampler.iteration % 10:
                self.log("MCMC iteration {}".format(sampler.iteration), "debug")
            # check convergence every 100 steps
            if sampler.iteration % 100:
                continue

            # compute autocorrelation time
            tau = sampler.get_autocorr_time(tol=0)
            if np.isnan(tau).any():
                self.log('Found nan autocorr time! Stopping.', "error")
                break

            # check convergence
            converged = np.all(tau / converge_criteria < sampler.iteration)
            converged &= np.all(np.abs(old_tau - tau) / tau < converge_criteria)
            self.log(
                "MCMC iteration {} autocorr time: mean {:.1f} min {:.1f} max {:.1f}".format(
                    sampler.iteration, np.mean(tau), np.min(tau), np.max(tau)
                ),
                "info",
            )
            if converged:
                break
            old_tau = tau

        out.update(converged=converged, num_steps=sampler.iteration)

        # converged posterior distribution
        if converged:
            self.log(
                "MCMC converged in {} iterations".format(sampler.iteration), "info"
            )
            tau = sampler.get_autocorr_time()
            burnin = int(2 * np.max(tau))
            thin = int(0.5 * np.min(tau))
            samples = sampler.get_chain(discard=burnin, thin=thin, flat=True)
            out.update(tau=tau, burnin=burnin, thin=thin, samples=samples)
        else:
            self.warn("MCMC not converged in {} iterations".format(num_steps))

        if res_prior is not None:
            self.bin_def = bin_def_orig
            self.nbins_res = nbins_res_orig

        # save and return
        return self.save_data(
            save_name, map_tag=map_tag, extra_tag=file_tag, bp_opts=True, **out
        )
