"""Calc module.

For taking the output from :class:`lasertram.tram.LaserTRAM` along with
user input to calculate concentrations for a series of spot analyses,
together with statistics on calibration standards.
"""

from __future__ import annotations

import re
import warnings

import numpy as np
import numpy.typing as npt
import pandas as pd
import statsmodels.api as sm
from scipy import stats
from statsmodels.tools.eval_measures import rmse

from ..helpers import conversions, formatting


class LaserCalc:
    """Calculate concentrations from LA-ICP-MS normalised ratios.

    Implements the methodology of Longerich et al. (1996) and
    Kent & Ungerer (2006) for converting internally normalised ratios
    produced by :class:`~lasertram.tram.LaserTRAM` into absolute
    concentrations (ppm). The basic workflow is:

    1. Upload SRM compositions via :meth:`get_SRM_comps`.
    2. Upload LaserTRAM output via :meth:`get_data`.
    3. Set the calibration standard via :meth:`set_calibration_standard`.
    4. Check for drift via :meth:`drift_check`.
    5. Set internal standard concentrations via
       :meth:`set_int_std_concentrations`.
    6. Calculate concentrations via :meth:`calculate_concentrations`.

    Parameters
    ----------
    name : str
        Name for the experiment / processing run.

    References
    ----------
    .. [1] Longerich, H. P., Jackson, S. E., & Günther, D. (1996).
       Inter-laboratory note. Laser ablation inductively coupled plasma
       mass spectrometric transient signal data acquisition and analyte
       concentration calculation. *J. Anal. At. Spectrom.*, 11(9), 899–904.
    .. [2] Kent, A. J., & Ungerer, C. A. (2006). Analysis of light
       lithophile elements (Li, Be, B) by laser ablation ICP-MS:
       comparison between magnetic sector and quadrupole ICP-MS.
       *Am. Mineral.*, 91(8–9), 1401–1411.

    Examples
    --------
    >>> from lasertram import LaserCalc, helpers
    >>> concentrations = LaserCalc(name="tutorial")
    >>> concentrations.get_SRM_comps(srm_data)
    >>> concentrations.get_data(processed_df)
    >>> concentrations.set_calibration_standard("GSD-1G")
    >>> concentrations.drift_check()
    >>> concentrations.get_calibration_std_ratios()
    >>> concentrations.set_int_std_concentrations(spots, values, uncertainties)
    >>> concentrations.calculate_concentrations()
    """

    def __init__(self, name: str) -> None:
        """Initialise a LaserCalc object.

        Parameters
        ----------
        name : str
            Name for the experiment / processing run.
        """
        # all attributes in relative chronological order that they are created in
        # if everything is done correctly. These all will get rewritten throughout the
        # data processing pipeline but this allows us to see what all the potential attributes
        # are going to be from the beginning (PEP convention)

        # for the math involved please see:

        # name for the lasercalc object
        # for notekeeping
        self.name = name

        # 2D pandas dataframe of standards reference material preferred compositions
        # from georem
        self.standards_data = None

        # List of standard reference materials in self.standards_data
        self.database_standards = None

        # list of standard reference material elements/oxides in self.standards_data
        self.standard_elements = None

        # list of standard reference material element/oxide 1 sigma uncertainties in self.standards_data
        self.standard_element_uncertainties = None

        # list of spot analyses for which concentrations are being calculated
        # this is the equivalent of self.data['Spot']
        self.spots = None

        # list of analytes for which concentrations are being calculated
        # these are column headers in self.data
        self.analytes = None

        # 1 sigma standard deviation of the calibration standard values
        # in self.data. Is len(analytes) in shape
        self.calibration_std_stdevs = None

        # 2D pandas dataframe that represents the metadata and data for numerous
        # spot analyses. Each row is the equivalent of a LaserTRAM.output_report
        # and has the following columns:
        # |timestamp|Spot|despiked|omitted_region|bkgd_start|bkgd_stop|int_start|int_stop|norm|norm_cps|analyte vals and uncertainties -->|
        # |---------|----|--------|--------------|----------|---------|---------|--------|----|--------|----------------------------------|
        self.data = None

        # element used as internal standard. NOT to be confused with analyte
        # e.g. self.int_std_element == 'Si' NOT '29Si'
        self.int_std_element = None
        self.int_std_units = None

        # list of standard reference materials in found in self.data that are
        # also found in self.database_standards. This lets you know which standard reference
        # materials you can use as potential calibration standards
        self.potential_calibration_standards = None

        # list of samples in self.data with the self.potential_calibration_standards
        # removed
        self.samples_nostandards = None

        # list of elements for which concentrations are being calculated
        # this is the equivalent to self.analytes with the atomic masses
        # removed
        self.elements = None

        # string representing the standard reference material used
        # as the calibration standard for calculating concentrations
        self.calibration_std = None

        # 2D pandas dataframe which is a subset of self.data for only the
        # calibration standard data. This is essentially self.data.loc[self.calibration_std,:]
        self.calibration_std_data = None

        # mean calibration standard values for all analytes
        # equivalent of self.calibration_std_data.mean(axis = 0)
        self.calibration_std_means = None

        # calibration standard standard error of the mean for all analytes
        self.calibration_std_ses = None

        # 2D dataframe that is contains statistics for each analyte in self.calibration_std_data
        # columns are:
        # drift_correct | f_pval | f_value | f_crit_value | rmse | slope | intercept | mean | std_dev | percent_std_err
        # These stats are based on the following regression:
        # for each analyte
        # x = self.calibration_std_data.loc[:,'timestamp']
        # y = self.calibration_std_data.loc[:, analyte]

        # X = sm.add_constant(x)
        # Note the difference in argument order
        # model = sm.OLS(y, X).fit()
        # now generate predictions
        # ypred = model.predict(X)

        # calc rmse
        # RMSE = rmse(y, ypred)

        self.calibration_std_stats = None

        # the ratio of concentrations between an analyte and the internal standard
        # in the georem calibration standard values
        self.calibration_std_conc_ratios = None

        # list of standard reference materials that are not used as calibration standard
        # this is effectively self.potential_calibration_standards with self.calibration_std
        # removed
        self.secondary_standards = None

        # 2D pandas dataframe of calculated concentrations for all spots in self.secondary_standards and all
        # analytes in self.analytes. This is self.data.loc[self.secondary_standards,self.analytes].shape in shape
        self.SRM_concentrations = None

        # 2D pandas dataframe of calculated concentrations for all spots in self.spots and all
        # analytes in self.analytes. This is self.data.loc[self.spots,self.analytes].shape in shape
        self.unknown_concentrations = None

        # 2D pandas dataframe of calculated accuracies for all spots in self.secondary_standards and all
        # analytes in self.analytes. This is self.data.loc[self.secondary_standards,self.analytes].shape in shape
        # here accuracy is just 100*measured_concentration / georem_concentration
        self.SRM_accuracies = None

    def get_SRM_comps(self, df: pd.DataFrame) -> None:
        """Load a database of standard reference material compositions.

        Must be called **before** :meth:`get_data` so that potential
        calibration standards can be identified.

        Parameters
        ----------
        df : pandas.DataFrame
            DataFrame of standard reference materials where each row
            represents data for one SRM. The first column must be named
            ``"Standard"``. All other columns are elemental
            concentrations. Standard names must match those found in
            `GEOREM <http://georem.mpch-mainz.gwdg.de/>`_.

        Examples
        --------
        >>> import pandas as pd
        >>> srm_data = pd.read_excel("laicpms_stds_tidy.xlsx")
        >>> concentrations.get_SRM_comps(srm_data)
        """
        # TODO: get_SRM_comps
        # - [x] add: check to make sure data are in right format else throw an error. do this by having a list of required columns and checking for them.

        print(
            "checking uploaded standards database format for correct column headers and data types..."
        )
        col_check, type_check = formatting.check_srm_format(df)

        if col_check is not None:
            raise ValueError(
                f"It looks like your input data are missing the following required columns: {col_check}. Please fix before continuing with processing."
            )
        if type_check is not None:
            raise TypeError(
                f"It looks like your input data have incorrect types in the following column indices: {type_check}. Please fix before continuing with processing."
            )
        self.standards_data = df.set_index("Standard")

        print("check complete...standards database format looks good!")

        self.database_standards = self.standards_data.index.unique().to_list()
        # Get a list of all of the elements supported in the published standard datasheet
        # Get a second list for the same elements but their corresponding uncertainty columns
        self.standard_elements = [
            analyte
            for analyte in self.standards_data.columns.tolist()
            if "_std" not in analyte
        ]
        self.standard_element_uncertainties = [
            analyte + "_std" for analyte in self.standard_elements
        ]

    def get_data(self, df: pd.DataFrame, verbose: bool = True) -> None:
        """Load LaserTRAM output for concentration calculations.

        Automatically identifies potential calibration standards by
        comparing spot names to the SRM database loaded via
        :meth:`get_SRM_comps`.

        Parameters
        ----------
        df : pandas.DataFrame
            Concatenated :attr:`LaserTRAM.output_report` rows for
            many spots.
        verbose : bool, optional
            Whether to print status messages, by default ``True``.

        Examples
        --------
        >>> concentrations.get_data(processed_df)
        """
        # check if first row is nan (output from GUI does this).
        # If so, remove it
        df = df[df.iloc[:, 0].isna() == False]

        # TODO: get_data
        # - [x] add: check to make sure data are in right format else throw an error. do this by having a list of required columns and checking for them. this can't include analytes though - just the metadata.
        # - [x] add: in some checking for analytes to make sure that the measured analytes exist within the standards database. compare self.elements to self.standard_elements by going through each standard and checking for nan for that element
        # - [ ] add: duplicate name check.
        if verbose:
            print(
                "checking LaserCalc input data format for correct column headers and data types..."
            )

        col_check, type_check = formatting.check_lt_complete_format(df)
        if col_check is not None:
            raise ValueError(
                f"It looks like your input data are missing the following required columns: {col_check}. Please fix before continuing with processing."
            )
        if type_check is not None:
            raise TypeError(
                f"It looks like your input data have incorrect types in the following column indices: {type_check}. Please fix before continuing with processing."
            )

        if verbose:
            print("check complete...input data format looks good!")

        data = df.set_index("Spot")
        data.insert(loc=1, column="index", value=np.arange(1, len(data) + 1))

        self.spots = data.index.unique().dropna().tolist()

        # Check for potential calibration standards. This will let us know what our options
        # are for choosing calibration standards by looking for spots that have the same string
        # as the standard spreadsheet

        stds_column = [
            [std for std in self.database_standards if std in spot]
            for spot in self.spots
        ]

        stds_column = [["unknown"] if not l else l for l in stds_column]

        stds_column = [std for sublist in stds_column for std in sublist]

        # standards that can be used as calibrations standards (must have more than 1 analysis)
        # potential_standards = list(np.unique(stds_column))
        potential_standards = [
            std for std in np.unique(stds_column) if stds_column.count(std) > 1
        ]
        potential_standards.remove("unknown")

        # all of the samples in your input sheet that are NOT potential standards
        all_standards = list(np.unique(stds_column))
        all_standards.remove("unknown")

        data["sample"] = stds_column

        data.reset_index(inplace=True)
        data.set_index("sample", inplace=True)

        self.data = data
        self.potential_calibration_standards = potential_standards
        self.samples_nostandards = list(np.setdiff1d(stds_column, all_standards))

        self.analytes = [
            analyte
            for analyte in data.columns.tolist()
            if not (
                "_se" in analyte
                or "norm" in analyte
                or "index" in analyte
                or "Spot" in analyte
                or "wt%" in analyte
                or "1stdev%" in analyte
                or "start" in analyte
                or "stop" in analyte
                or "long" in analyte
                or "timestamp" in analyte
                or "despiked" in analyte
                or "omitted_region" in analyte
            )
        ]
        # elements without isotopes in the front
        self.elements = [re.split(r"(\d+)", analyte)[2] for analyte in self.analytes]

        # first check to make sure that the element exists within the standards database
        for el in self.elements:
            if el not in self.standard_elements:
                raise ValueError(
                    f"{el} is not in the standards database. Please remove it from your data before proceeding with processing."
                )
        # now for each potential calibration standard check to see if any of the analytes measured
        # don't have published values
        nan_dict = {}
        for standard in self.potential_calibration_standards:

            # isolate just the elements measured in the experiment
            to_check = self.standards_data.loc[standard, self.elements]

            # now check to see if any are nan by creating a mask
            nan_mask = to_check.isna()
            nan_dict[standard] = nan_mask

        # create dataframe of masks
        nan_df = pd.DataFrame(nan_dict)

        # for each column check to see if there are any nans
        # if so this eliminates them as potential calibration standards
        # so they get removed
        for col in nan_df.columns:
            if nan_df[col].any():
                self.potential_calibration_standards.remove(col)

        # if no potential calibration standards exist throw some messages and then
        # an error
        if len(self.potential_calibration_standards) == 0:
            print(
                f"your list of measured analytes is: {self.analytes}.\nThe following measured standard reference materials have no published value for the selected analytes:"
            )

            for col in nan_df.columns:
                print(f"{col}: {nan_df[col][nan_dict[col]].index.values}")

            raise ValueError(
                "cannot process data. There are no potential calibration standards in your dataset that contain accepted values for all analytes in the uploaded standards database. The easiest way to proceed is to remove the problematic analytes from your dataset."
            )
        # else tell them their potential calibration standards
        else:
            print(
                f"Your potential calibration standards are: {[str(s) for s in self.potential_calibration_standards]}"
            )

        # internal standard analyte from lasertram
        self.int_std_element = re.split(r"(\d+)", self.data["norm"].unique()[0])[2]

    def set_calibration_standard(self, std: str) -> None:
        """Designate the calibration standard for concentration calculations.

        Calculates mean values, standard deviations, and relative
        standard errors for the chosen SRM across all analytes.

        Parameters
        ----------
        std : str
            Name of the standard reference material, e.g.
            ``"GSD-1G"``, ``"NIST-612"``, ``"BCR-2G"``.

        Examples
        --------
        >>> concentrations.set_calibration_standard("GSD-1G")
        """
        self.calibration_std = std

        self.calibration_std_data = self.data.loc[std, :]
        # Calibration standard information
        # mean
        self.calibration_std_means = self.calibration_std_data.loc[
            :, self.analytes
        ].mean()
        # std deviation
        self.calibration_std_stdevs = self.calibration_std_data.loc[
            :, self.analytes
        ].std()
        # relative standard error
        self.calibration_std_ses = 100 * (
            (self.calibration_std_stdevs / self.calibration_std_means)
            / np.sqrt(self.calibration_std_data.shape[0])
        )

    def drift_check(self, pval: float = 0.01) -> None:
        """Assess instrument drift for each analyte.

        Performs a linear regression of each analyte's normalised ratio
        through time for the calibration standard. If the regression is
        statistically significant (F-test *p*-value < ``pval`` **and**
        F > F_crit), the analyte is flagged for drift correction
        in :meth:`calculate_concentrations`.

        Results are stored in ``self.calibration_std_stats``.

        Parameters
        ----------
        pval : float, optional
            Significance threshold to reject the null hypothesis of no
            drift, by default 0.01.

        Examples
        --------
        >>> concentrations.drift_check()
        >>> concentrations.calibration_std_stats.head()
        """
        calib_std_rmses = []
        calib_std_slopes = []
        calib_std_intercepts = []
        drift_check = []

        f_pvals = []
        f_vals = []
        f_crits = []
        for analyte in self.analytes:
            # Getting regression statistics on analyte normalized ratios through time
            # for the calibration standard. This is what we use to check to see if it needs
            # to be drift corrected
            if "timestamp" in self.calibration_std_data.columns.tolist():
                # get an array in time units based on timestamp column. This is
                # is in seconds
                x = np.array(
                    [
                        np.datetime64(d, "m")
                        for d in self.calibration_std_data["timestamp"]
                    ]
                ).astype(np.float64)
                # x = np.cumsum(np.diff(x))
                # x = np.insert(x, 0, 0).astype(np.float64)

            else:
                x = self.calibration_std_data["index"].to_numpy()

            y = self.calibration_std_data.loc[:, analyte].astype("float64")

            X = sm.add_constant(x)
            # Note the difference in argument order
            model = sm.OLS(y, X).fit()
            # now generate predictions
            ypred = model.predict(X)

            # calc rmse
            RMSE = rmse(y, ypred)

            calib_std_rmses.append(RMSE)

            if model.params.shape[0] < 2:
                calib_std_slopes.append(model.params.loc["x1"])
                calib_std_intercepts.append(0)

            else:
                calib_std_slopes.append(model.params.loc["x1"])
                calib_std_intercepts.append(model.params.loc["const"])

            # new stuff
            # confidence limit 99%

            # f value stuff

            fvalue = model.fvalue
            f_vals.append(fvalue)
            f_pvalue = model.f_pvalue
            f_pvals.append(f_pvalue)
            fcrit = stats.f.ppf(q=1 - pval, dfn=len(x) - 1, dfd=len(y) - 1)
            f_crits.append(fcrit)
            if (f_pvalue < pval) and (fvalue > fcrit):
                drift = "True"
                drift_check.append(drift)
            else:
                drift = "False"
                drift_check.append(drift)

        self.calibration_std_stats = pd.DataFrame(
            {
                "drift_correct": drift_check,
                "f_pval": f_pvals,
                "f_value": f_vals,
                "f_crit_value": f_crits,
                "rmse": calib_std_rmses,
                "slope": calib_std_slopes,
                "intercept": calib_std_intercepts,
                "mean": self.calibration_std_means[self.analytes].to_numpy(),
                "std_dev": self.calibration_std_stdevs[self.analytes].to_numpy(),
                "percent_std_err": self.calibration_std_ses[self.analytes].to_numpy(),
            },
            index=self.analytes,
        )

    def get_calibration_std_ratios(self) -> None:
        """Calculate concentration ratios for the calibration standard.

        For the chosen calibration standard, compute the ratio of each
        analyte's concentration to the internal standard element's
        concentration. Results are stored in
        ``self.calibration_std_conc_ratios``.

        Examples
        --------
        >>> concentrations.get_calibration_std_ratios()
        """

        # For our calibration standard, calculate the concentration ratio
        # of each analyte to the element used as the internal standard
        std_conc_ratios = []

        for element in self.elements:
            if element in self.standard_elements:
                std_conc_ratios.append(
                    self.standards_data.loc[self.calibration_std, element]
                    / self.standards_data.loc[
                        self.calibration_std, self.int_std_element
                    ]
                )

        # make our list an array for easier math going forward
        self.calibration_std_conc_ratios = np.array(std_conc_ratios)

    def set_int_std_concentrations(
        self,
        spots: pd.Series | None = None,
        concentrations: npt.ArrayLike | None = None,
        uncertainties: npt.ArrayLike | None = None,
        units: str = "wt_per_ox",
    ) -> None:
        """Assign internal standard concentrations to spots.

        A linear change in the concentration value produces a
        proportional linear change in the calculated concentration.

        Parameters
        ----------
        spots : pandas.Series or None, optional
            Spot names to assign. Corresponds to the ``Spot`` column
            from the LaserTRAM output.
        concentrations : array-like or None, optional
            Internal standard concentration values. Must be the
            same length as ``spots``.
        uncertainties : array-like or None, optional
            Relative uncertainty in percent for the internal standard.
            Must be the same length as ``spots``.
        units : str, optional
            Units for the concentration values. One of ``"wt_per_ox"``
            (weight percent oxide, default), ``"wt_per_el"`` (weight
            percent element), or ``"ppm_el"`` (ppm element).

        Examples
        --------
        >>> concentrations.set_int_std_concentrations(
        ...     internal_std_comps["Spot"],
        ...     internal_std_comps["SiO2"],
        ...     internal_std_comps["SiO2_std%"],
        ... )
        """

        assert units in [
            "wt_per_ox",
            "wt_per_el",
            "ppm_el",
        ], f"{units} is not a supported unit for concentrations. accepted units are: ['wt_per_ox','wt_per_el', 'ppm_el']"

        if spots is None:
            spots = (self.data["Spot"],)
            concentrations = (np.full(self.data["Spot"].shape[0], 10),)
            uncertainties = (np.full(self.data["Spot"].shape[0], 1),)

        self.data["int_std_comp"] = 10.0
        self.data["int_std_rel_unc"] = 1.0
        df = self.data.reset_index().set_index("Spot")

        for spot, concentration, uncertainty in zip(
            spots, concentrations, uncertainties
        ):
            df.loc[spot, "int_std_comp"] = concentration
            df.loc[spot, "int_std_rel_unc"] = uncertainty

        self.data["int_std_comp"] = df["int_std_comp"].to_numpy()
        self.data["int_std_rel_unc"] = df["int_std_rel_unc"].to_numpy()

        self.int_std_units = units

    def calculate_concentrations(self) -> None:
        """Calculate concentrations and uncertainties for all spots.

        Uses the calibration standard, internal standard concentrations,
        and drift correction information to compute absolute
        concentrations. Stores results in
        ``self.unknown_concentrations`` and ``self.SRM_concentrations``.

        Values below the detection limit are replaced with ``"b.d.l."``.

        Examples
        --------
        >>> concentrations.calculate_concentrations()
        >>> concentrations.unknown_concentrations.head()
        """

        secondary_standards = self.potential_calibration_standards.copy()
        secondary_standards.remove(self.calibration_std)
        self.secondary_standards = secondary_standards
        secondary_standards_concentrations_list = []
        unknown_concentrations_list = []

        for sample in secondary_standards:
            Cn_u = self.standards_data.loc[
                sample,
                re.split(
                    r"(\d+)",
                    self.calibration_std_data["norm"].unique()[0],
                )[2],
            ]
            Cin_std = self.calibration_std_conc_ratios
            Ni_std = self.calibration_std_stats["mean"][self.analytes]
            Ni_u = self.data.loc[sample, self.analytes]

            concentrations = Cn_u * (Cin_std / Ni_std) * Ni_u

            drift_concentrations_list = []

            for j, analyte, slope, intercept, drift in zip(
                range(len(self.analytes)),
                self.analytes,
                self.calibration_std_stats["slope"],
                self.calibration_std_stats["intercept"],
                self.calibration_std_stats["drift_correct"],
            ):
                if "True" in drift:
                    if "timestamp" in self.data.columns.tolist():
                        frac = (
                            slope
                            * np.array(
                                [
                                    np.datetime64(d, "m")
                                    for d in self.data.loc[sample, "timestamp"]
                                ]
                            ).astype(np.float64)
                            + intercept
                        )
                    else:
                        frac = slope * self.data.loc[sample, "index"] + intercept

                    Ni_std = frac

                    drift_concentrations = Cn_u * (Cin_std[j] / Ni_std) * Ni_u[analyte]

                    if isinstance(drift_concentrations, np.float64):
                        df = pd.DataFrame(
                            np.array([drift_concentrations]), columns=[analyte]
                        )

                    else:
                        df = pd.DataFrame(drift_concentrations, columns=[analyte])

                    drift_concentrations_list.append(df)

            if len(drift_concentrations_list) > 0:
                drift_df = pd.concat(drift_concentrations_list, axis="columns")

                if drift_df.shape[0] == 1:
                    drift_df["sample"] = sample
                    drift_df.set_index("sample", inplace=True)
            else:
                drift_df = pd.DataFrame()

            for column in drift_df.columns.tolist():
                if isinstance(concentrations, pd.Series):
                    concentrations.loc[column] = drift_df[column].to_numpy()[0]

                else:
                    concentrations[column] = drift_df[column].to_numpy()

            if isinstance(concentrations, pd.Series):
                concentrations = pd.DataFrame(concentrations).T
                concentrations["sample"] = sample
                concentrations.set_index("sample", inplace=True)

            secondary_standards_concentrations_list.append(concentrations)

        ###############################

        for sample in self.samples_nostandards:
            # Cn_u = conversions.oxide_to_ppm(
            #     self.data.loc[sample, "int_std_comp"],
            #     self.data.loc[sample, "norm"].unique()[0],
            # ).to_numpy()
            int_std_element = "".join(
                [
                    i
                    for i in self.data.loc[sample, "norm"].unique()[0]
                    if not i.isdigit()
                ]
            )

            # handle conversions from various units to all end up at ppm element
            if self.int_std_units == "wt_per_ox":
                Cn_u = conversions.oxide_to_ppm(
                    self.data.loc[sample, "int_std_comp"], int_std_element
                )
            elif self.int_std_units == "wt_per_el":
                Cn_u = self.data.loc[sample, "int_std_comp"].values * 1e4
            elif self.int_std_units == "ppm_el":
                Cn_u = self.data.loc[sample, "int_std_comp"].values

            Cin_std = self.calibration_std_conc_ratios
            Ni_std = self.calibration_std_stats["mean"][self.analytes].to_numpy()
            Ni_u = self.data.loc[sample, self.analytes].to_numpy()

            concentrations = pd.DataFrame(
                Cn_u[:, np.newaxis] * (Cin_std / Ni_std) * Ni_u, columns=self.analytes
            )

            drift_concentrations_list = []

            for j, analyte, slope, intercept, drift in zip(
                range(len(self.analytes)),
                self.analytes,
                self.calibration_std_stats["slope"],
                self.calibration_std_stats["intercept"],
                self.calibration_std_stats["drift_correct"],
            ):
                if "True" in drift:
                    if "timestamp" in self.data.columns.tolist():
                        frac = (
                            slope
                            * np.array(
                                [
                                    np.datetime64(d, "m")
                                    for d in self.data.loc[sample, "timestamp"]
                                ]
                            ).astype(np.float64)
                            + intercept
                        )
                    else:
                        frac = slope * self.data.loc[sample, "index"] + intercept
                    frac = np.array(frac)
                    drift_concentrations = (
                        Cn_u[:, np.newaxis]
                        * (Cin_std[j] / frac)[:, np.newaxis]
                        * Ni_u[:, j][:, np.newaxis]
                    )

                    if isinstance(drift_concentrations, np.float64):
                        df = pd.DataFrame(
                            np.array([drift_concentrations]), columns=[analyte]
                        )

                    else:
                        df = pd.DataFrame(drift_concentrations, columns=[analyte])

                    drift_concentrations_list.append(df)

            if len(drift_concentrations_list) > 0:
                drift_df = pd.concat(drift_concentrations_list, axis="columns")

                if drift_df.shape[0] == 1:
                    drift_df["sample"] = sample
                    drift_df.set_index("sample", inplace=True)

            for column in drift_df.columns.tolist():
                if isinstance(concentrations, pd.Series):
                    concentrations.loc[column] = drift_df[column].to_numpy()[0]

                else:
                    concentrations[column] = drift_df[column].to_numpy()

            if isinstance(concentrations, pd.Series):
                concentrations = pd.DataFrame(concentrations).T
                concentrations["sample"] = sample
                concentrations.set_index("sample", inplace=True)

            unknown_concentrations_list.append(concentrations)

        self.SRM_concentrations = pd.concat(secondary_standards_concentrations_list)
        self.unknown_concentrations = pd.concat(unknown_concentrations_list)

        self.calculate_uncertainties()

        # INSERT IN SPOT METADATA NOW
        # OLD WAY OF REPLACING NEGATIVES. WILL THROW ERROR IN FUTURE FOR MIXING
        # STRINGS WITH FLOATS
        # self.unknown_concentrations[self.unknown_concentrations < 0] = "b.d.l."
        # self.SRM_concentrations[self.SRM_concentrations < 0] = "b.d.l."

        # THE NEW WAY OF DOING IT IS TO GO THROUGH COLUMN BY COLUMN AND CHECK FOR BELOW
        # 0 VALUES, CHANGE THE DTYPE TO OBJECT, AND THEN REPLACE THE NEGATIVE VALUES WITH BDL STRING
        # THEN CHANGE THE UNCERTAINTY VALUES TO BDL STRINGS BASED ON THE ROW WE DID FOR THE ACTUAL CONCENTRATION VALUE
        for analyte in self.analytes:
            if any(self.unknown_concentrations[analyte] < 0):
                self.unknown_concentrations[analyte] = self.unknown_concentrations[
                    analyte
                ].astype("object")
                self.unknown_concentrations[f"{analyte}_interr"] = (
                    self.unknown_concentrations[f"{analyte}_interr"].astype("object")
                )
                self.unknown_concentrations[f"{analyte}_exterr"] = (
                    self.unknown_concentrations[f"{analyte}_exterr"].astype("object")
                )

                self.unknown_concentrations.loc[
                    self.unknown_concentrations[analyte] < 0, analyte
                ] = "b.d.l."
                self.unknown_concentrations.loc[
                    self.unknown_concentrations[analyte] == "b.d.l.",
                    f"{analyte}_interr",
                ] = "b.d.l."
                self.unknown_concentrations.loc[
                    self.unknown_concentrations[analyte] == "b.d.l", f"{analyte}_interr"
                ] = "b.d.l."

            if any(self.SRM_concentrations[analyte] < 0):
                self.SRM_concentrations[analyte] = self.SRM_concentrations[
                    analyte
                ].astype("object")
                self.SRM_concentrations[f"{analyte}_interr"] = self.SRM_concentrations[
                    f"{analyte}_interr"
                ].astype("object")
                self.SRM_concentrations[f"{analyte}_exterr"] = self.SRM_concentrations[
                    f"{analyte}_exterr"
                ].astype("object")

                self.SRM_concentrations.loc[
                    self.SRM_concentrations[analyte] < 0, analyte
                ] = "b.d.l."
                self.SRM_concentrations.loc[
                    self.SRM_concentrations[analyte] == "b.d.l.", f"{analyte}_interr"
                ] = "b.d.l."
                self.SRM_concentrations.loc[
                    self.SRM_concentrations[analyte] == "b.d.l", f"{analyte}_interr"
                ] = "b.d.l."

        self.SRM_concentrations.insert(
            0, "Spot", list(self.data.loc[self.secondary_standards, "Spot"])
        )

        if "timestamp" in self.data.columns.tolist():
            self.SRM_concentrations.insert(
                0,
                "timestamp",
                list(self.data.loc[self.secondary_standards, "timestamp"]),
            )
        else:
            self.SRM_concentrations.insert(
                0, "index", list(self.data.loc[self.secondary_standards, "index"])
            )
        self.unknown_concentrations.insert(
            0, "Spot", list(self.data.loc[self.samples_nostandards, "Spot"])
        )
        if "timestamp" in self.data.columns.tolist():
            self.unknown_concentrations.insert(
                0,
                "timestamp",
                list(self.data.loc[self.samples_nostandards, "timestamp"]),
            )
        else:
            self.unknown_concentrations.insert(
                0, "index", list(self.data.loc[self.samples_nostandards, "index"])
            )

        self.unknown_concentrations.index = [
            "unknown"
        ] * self.unknown_concentrations.shape[0]
        self.unknown_concentrations.index.name = "sample"

    def calculate_uncertainties(self) -> None:
        """Calculate internal and external uncertainties for each analysis.

        Called automatically by :meth:`calculate_concentrations`. The
        results are appended as ``<analyte>_interr`` and
        ``<analyte>_exterr`` columns to ``self.unknown_concentrations``
        and ``self.SRM_concentrations``.
        """

        myuncertainties = [analyte + "_se" for analyte in self.analytes]
        srm_rel_ext_uncertainties_list = []
        unk_rel_ext_uncertainties_list = []
        srm_rel_int_uncertainties_list = []
        unk_rel_int_uncertainties_list = []
        # use RMSE of regression for elements where drift correction is applied rather than the standard error
        # of the mean of all the calibration standard normalized ratios
        rse_i_std = []
        for analyte in self.analytes:
            if "True" in self.calibration_std_stats.loc[analyte, "drift_correct"]:
                rse_i_std.append(
                    100
                    * self.calibration_std_stats.loc[analyte, "rmse"]
                    / self.calibration_std_stats.loc[analyte, "mean"]
                )
            else:
                rse_i_std.append(
                    self.calibration_std_stats.loc[analyte, "percent_std_err"]
                )

        rse_i_std = np.array(rse_i_std)

        for sample in self.secondary_standards:
            t1 = (
                self.standards_data.loc[sample, f"{self.int_std_element}_std"]
                / self.standards_data.loc[sample, f"{self.int_std_element}"]
            ) ** 2

            # concentration of internal standard in calibration standard uncertainties
            t2 = (
                self.standards_data.loc[
                    self.calibration_std, f"{self.int_std_element}_std"
                ]
                / self.standards_data.loc[
                    self.calibration_std, f"{self.int_std_element}"
                ]
            ) ** 2

            # concentration of each analyte in calibration standard uncertainties
            std_conc_stds = []
            for element in self.elements:
                # if our element is in the list of standard elements take the ratio
                if element in self.standard_elements:
                    std_conc_stds.append(
                        (
                            self.standards_data.loc[
                                self.calibration_std, f"{element}_std"
                            ]
                            / self.standards_data.loc[self.calibration_std, element]
                        )
                        ** 2
                    )

            std_conc_stds = np.array(std_conc_stds)

            # Overall uncertainties
            # Need to loop through each row?

            rel_ext_uncertainty = pd.DataFrame(
                np.sqrt(
                    np.array(
                        t1
                        + t2
                        + std_conc_stds
                        + (rse_i_std[np.newaxis, :] / 100) ** 2
                        + (self.data.loc[sample, myuncertainties].to_numpy() / 100) ** 2
                    ).astype(np.float64)
                )
            )
            rel_int_uncertainty = pd.DataFrame(
                np.sqrt(
                    np.array(
                        t1
                        # +t2
                        # + std_conc_stds
                        + (rse_i_std[np.newaxis, :] / 100) ** 2
                        + (self.data.loc[sample, myuncertainties].to_numpy() / 100) ** 2
                    ).astype(np.float64)
                )
            )
            rel_ext_uncertainty.columns = [f"{a}_exterr" for a in self.analytes]
            srm_rel_ext_uncertainties_list.append(rel_ext_uncertainty)
            rel_int_uncertainty.columns = [f"{a}_interr" for a in self.analytes]
            srm_rel_int_uncertainties_list.append(rel_int_uncertainty)

        srm_rel_ext_uncertainties = pd.concat(srm_rel_ext_uncertainties_list)
        srm_rel_int_uncertainties = pd.concat(srm_rel_int_uncertainties_list)

        srm_ext_uncertainties = pd.DataFrame(
            srm_rel_ext_uncertainties.values
            * self.SRM_concentrations.loc[:, self.analytes].values,
            columns=[f"{a}_exterr" for a in self.analytes],
            index=self.SRM_concentrations.index,
        )
        srm_int_uncertainties = pd.DataFrame(
            srm_rel_int_uncertainties.values
            * self.SRM_concentrations.loc[:, self.analytes].values,
            columns=[f"{a}_interr" for a in self.analytes],
            index=self.SRM_concentrations.index,
        )

        self.SRM_concentrations = pd.concat(
            [self.SRM_concentrations, srm_ext_uncertainties, srm_int_uncertainties],
            axis="columns",
        )

        ######################################

        for sample in self.samples_nostandards:
            # concentration of internal standard in unknown uncertainties
            int_std_element = re.split(
                r"(\d+)", self.calibration_std_data["norm"].unique()[0]
            )[2]
            # concentration of internal standard in unknown uncertainties
            t1 = (self.data.loc[sample, "int_std_rel_unc"] / 100) ** 2
            t1 = np.array(t1)
            t1 = t1[:, np.newaxis]

            # concentration of internal standard in calibration standard uncertainties
            t2 = (
                self.standards_data.loc[self.calibration_std, f"{int_std_element}_std"]
                / self.standards_data.loc[self.calibration_std, f"{int_std_element}"]
            ) ** 2

            # concentration of each analyte in calibration standard uncertainties
            std_conc_stds = []
            for element in self.elements:
                # # if our element is in the list of standard elements take the ratio
                if element in self.standard_elements:
                    std_conc_stds.append(
                        (
                            self.standards_data.loc[
                                self.calibration_std, f"{element}_std"
                            ]
                            / self.standards_data.loc[self.calibration_std, element]
                        )
                        ** 2
                    )

            std_conc_stds = np.array(std_conc_stds)

            # Overall uncertainties
            # Need to loop through each row?

            rel_ext_uncertainty = pd.DataFrame(
                np.sqrt(
                    np.array(
                        t1
                        + t2
                        + std_conc_stds
                        + (rse_i_std[np.newaxis, :] / 100) ** 2
                        + (self.data.loc[sample, myuncertainties].to_numpy() / 100) ** 2
                    ).astype(np.float64)
                )
            )
            rel_int_uncertainty = pd.DataFrame(
                np.sqrt(
                    np.array(
                        t1
                        # +t2
                        # + std_conc_stds
                        + (rse_i_std[np.newaxis, :] / 100) ** 2
                        + (self.data.loc[sample, myuncertainties].to_numpy() / 100) ** 2
                    ).astype(np.float64)
                )
            )
            rel_ext_uncertainty.columns = [f"{a}_exterr" for a in self.analytes]
            unk_rel_ext_uncertainties_list.append(rel_ext_uncertainty)
            rel_int_uncertainty.columns = [f"{a}_interr" for a in self.analytes]
            unk_rel_int_uncertainties_list.append(rel_int_uncertainty)

        unk_rel_ext_uncertainties = pd.concat(unk_rel_ext_uncertainties_list)
        unk_rel_int_uncertainties = pd.concat(unk_rel_int_uncertainties_list)

        unknown_ext_uncertainties = pd.DataFrame(
            unk_rel_ext_uncertainties.values
            * self.unknown_concentrations.loc[:, self.analytes].values,
            columns=[f"{a}_exterr" for a in self.analytes],
            index=self.unknown_concentrations.index,
        )

        unknown_int_uncertainties = pd.DataFrame(
            unk_rel_int_uncertainties.values
            * self.unknown_concentrations.loc[:, self.analytes].values,
            columns=[f"{a}_interr" for a in self.analytes],
            index=self.unknown_concentrations.index,
        )

        self.unknown_concentrations = pd.concat(
            [
                self.unknown_concentrations,
                unknown_ext_uncertainties,
                unknown_int_uncertainties,
            ],
            axis="columns",
        )

    # make an accuracy checking function
    # need to use analytes no mass to check SRM vals
    def get_secondary_standard_accuracies(self) -> None:
        """Calculate accuracy of secondary standard measurements.

        Accuracy is defined as ``100 * measured / accepted`` where
        *accepted* is the GEOREM preferred value for that SRM–analyte
        pair. Results are stored in ``self.SRM_accuracies``.

        Examples
        --------
        >>> concentrations.get_secondary_standard_accuracies()
        >>> concentrations.SRM_accuracies.head()
        """
        df_list = []

        for standard in self.secondary_standards:
            # need to go through column by column and check for bdl and then
            # replace with nan for numeric calculation. This explicit type declaration
            # is now required by pandas.

            for analyte in self.analytes:
                if self.SRM_concentrations[analyte].dtype == "object":
                    if self.SRM_concentrations[analyte].str.contains("b.d.l.").any():
                        ser = pd.to_numeric(
                            self.SRM_concentrations[analyte], errors="coerce"
                        )
                        ser.replace("b.d.l.", np.nan, inplace=True)

                        self.SRM_concentrations[analyte] = ser

            df = pd.DataFrame(
                100
                * self.SRM_concentrations.loc[standard, self.analytes].values
                / self.standards_data.loc[standard, self.elements].values[
                    np.newaxis, :
                ],
                columns=self.analytes,
                index=self.SRM_concentrations.loc[standard, :].index,
            ).fillna("b.d.l.")
            df.insert(0, "Spot", self.SRM_concentrations.loc[standard, "Spot"])
            if "timestamp" in self.data.columns:
                df.insert(
                    0, "timestamp", self.SRM_concentrations.loc[standard, "timestamp"]
                )
            else:
                df.insert(0, "index", self.SRM_concentrations.loc[standard, "index"])

            df_list.append(df)

        self.SRM_accuracies = pd.concat(df_list)
