#!/usr/bin/env python
# encoding: utf-8
"""
*detect arc-lines on a pinhole frame to generate a dispersion solution*

Author
: Marco Landoni & David Young

Date Created
: September  1, 2020
"""
################# GLOBAL IMPORTS ####################
from fundamentals import fmultiprocess
from soxspipe.commonutils.toolkit import unpack_order_table, read_spectral_format, twoD_disp_map_image_to_dataframe
from soxspipe.commonutils.dispersion_map_to_pixel_arrays import dispersion_map_to_pixel_arrays
from soxspipe.commonutils.filenamer import filenamer
from soxspipe.commonutils.polynomials import chebyshev_order_wavelength_polynomials

from soxspipe.commonutils.toolkit import get_calibrations_path
from os.path import expanduser
from soxspipe.commonutils import detector_lookup
from soxspipe.commonutils import keyword_lookup
from fundamentals import tools
from builtins import object
import sys
import os
from datetime import datetime

os.environ['TERM'] = 'vt100'


class create_dispersion_map(object):
    """
    *detect arc-lines on a pinhole frame to generate a dispersion solution*

    **Key Arguments:**

    - ``log`` -- logger
    - ``settings`` -- the settings dictionary
    - ``recipeSettings`` -- the recipe specific settings
    - ``pinholeFrame`` -- the calibrated pinhole frame (single or multi)
    - ``firstGuessMap`` -- the first guess dispersion map from the `soxs_disp_solution` recipe (needed in `soxs_spat_solution` recipe). Default *False*.
    - ``orderTable`` -- the order geometry table
    - ``qcTable`` -- the data frame to collect measured QC metrics
    - ``productsTable`` -- the data frame to collect output products
    - ``sofName`` -- name of the originating SOF file
    - ``create2DMap`` -- create the 2D image map of wavelength, slit-position and order from disp solution.
    - ``lineDetectionTable`` -- the list of arc-lines detected on the pinhole frame (used only for pipeline tuning)
    - ``startNightDate`` -- YYYY-MM-DD date of the observation night. Default ""

    **Usage:**

    ```python
    from soxspipe.commonutils import create_dispersion_map
    mapPath, mapImagePath, res_plots, qcTable, productsTable = create_dispersion_map(
        log=log,
        settings=settings,
        pinholeFrame=frame,
        firstGuessMap=False,
        qcTable=self.qc,
        productsTable=self.products
    ).get()
    ```
    """

    def __init__(
            self,
            log,
            settings,
            recipeSettings,
            pinholeFrame,
            firstGuessMap=False,
            orderTable=False,
            qcTable=False,
            productsTable=False,
            sofName=False,
            create2DMap=True,
            lineDetectionTable=False,
            startNightDate="",
            arcFrame=None
    ):
        self.log = log
        log.debug("instantiating a new 'create_dispersion_map' object")

        import warnings
        import copy

        self.settings = settings
        self.pinholeFrame = pinholeFrame
        self.firstGuessMap = firstGuessMap
        self.orderTable = orderTable
        self.qc = qcTable
        self.products = productsTable
        self.sofName = sofName
        self.create2DMap = create2DMap
        self.recipeSettings = copy.deepcopy(recipeSettings)
        self.lineDetectionTable = lineDetectionTable
        self.startNightDate = startNightDate
        self.arcFrame = arcFrame

        # KEYWORD LOOKUP OBJECT - LOOKUP KEYWORD FROM DICTIONARY IN RESOURCES
        # FOLDER
        kw = keyword_lookup(
            log=self.log,
            settings=self.settings
        ).get
        self.kw = kw
        self.arm = pinholeFrame.header[kw("SEQ_ARM")]
        self.dateObs = pinholeFrame.header[kw("DATE_OBS")]
        self.inst = pinholeFrame.header[kw("INSTRUME")]
        self.exptime = pinholeFrame.header[kw("EXPTIME")]

        from soxspipe.commonutils.toolkit import get_calibration_lamp
        self.lamp = get_calibration_lamp(log=self.log, frame=pinholeFrame, kw=kw)

        if self.arm.upper() == "NIR" and not ("ar" in self.lamp.lower() and "hg" in self.lamp.lower()):
            raise Exception("wrong lamp")

        # WHICH RECIPE ARE WE WORKING WITH?
        if self.firstGuessMap:
            self.recipeName = "soxs-spatial-solution"
        else:
            self.recipeName = "soxs-disp-solution"

        # DETECTOR PARAMETERS LOOKUP OBJECToff
        self.detectorParams = detector_lookup(
            log=log,
            settings=settings
        ).get(self.arm)

        from photutils.utils import NoDetectionsWarning
        warnings.simplefilter('ignore', NoDetectionsWarning)
        # ASTROPY HAS RESET LOGGING LEVEL -- FIX
        import logging
        logging.getLogger().setLevel(logging.INFO + 5)

        # SET IMAGE ORIENTATION
        if self.detectorParams["dispersion-axis"] == "x":
            self.axisA = "x"
            self.axisB = "y"
        else:
            self.axisA = "y"
            self.axisB = "x"

        home = expanduser("~")
        self.qcDir = self.settings["workspace-root-dir"].replace("~", home) + f"/qc/{self.startNightDate}/{self.recipeName}/"
        self.qcDir = self.qcDir.replace("//", "/")
        # RECURSIVELY CREATE MISSING DIRECTORIES
        if not os.path.exists(self.qcDir):
            os.makedirs(self.qcDir)

        return None

    def get(self):
        """
        *generate the dispersion map*

        **Return:**

        - ``mapPath`` -- path to the file containing the coefficients of the x,y polynomials of the global dispersion map fit
        """
        self.log.debug('starting the ``get`` method')

        import pandas as pd
        from astropy.table import Table
        import numpy as np
        from astropy.stats import sigma_clipped_stats
        from astropy.stats import sigma_clip

        bootstrap_dispersion_solution = self.settings["bootstrap_dispersion_solution"]
        tightFit = self.settings["bootstrap_dispersion_solution"]

        orderDeg = self.recipeSettings["order-deg"]
        wavelengthDeg = self.recipeSettings["wavelength-deg"]

        # WHICH RECIPE ARE WE WORKING WITH?
        if self.firstGuessMap:
            slitDeg = self.recipeSettings["slit-deg"]
        else:
            if isinstance(orderDeg, list):
                slitDeg = [0, 0]
            else:
                slitDeg = 0

        # READ PREDICTED LINE POSITIONS FROM FILE - RETURNED AS DATAFRAME
        orderPixelTable = self.get_predicted_line_list()
        totalLines = len(orderPixelTable.index)
        self.uniqueSlitPos = orderPixelTable['slit_position'].unique()

        # GET THE WINDOW SIZE FOR ATTEMPTING TO DETECT LINES ON FRAME
        windowSize = self.recipeSettings["pixel-window-size"]

        # MASK THE PINHOLE FRAME
        pinholeFrame = self.pinholeFrame
        pinholeFrameMasked = np.ma.array(pinholeFrame.data, mask=pinholeFrame.mask)
        mean, median, std = sigma_clipped_stats(pinholeFrameMasked, sigma=5.0, stdfunc="mad_std", cenfunc="median", maxiters=3)

        # pinholeFrameMasked = pinholeFrameMasked - median
        self.pinholeFrameMasked = pinholeFrameMasked

        self.mean = mean
        self.std = std

        from soxspipe.commonutils.toolkit import quicklook_image
        quicklook_image(
            log=self.log, CCDObject=pinholeFrame, show=False, ext='data', stdWindow=3, title=False, surfacePlot=True)

        if False:
            import matplotlib.pyplot as plt
            from matplotlib.patches import Rectangle
            fig, ax = plt.subplots()
            ax.imshow(pinholeFrameMasked, cmap='gray', origin='lower', vmin=self.mean, vmax=self.mean + 5 * self.std)
            for index, row in orderPixelTable.iterrows():
                x = row['detector_x']
                y = row['detector_y']
                ax.add_patch(Rectangle((x - windowSize / 2, y - windowSize / 2), windowSize, windowSize, fill=None, edgecolor='red'))
            plt.show()

        boost = True
        while boost:

            # SORT BY COLUMN NAME
            orderPixelTable.sort_values(['wavelength'], inplace=True)

            boost = False

            if not isinstance(self.lineDetectionTable, bool):
                orderPixelTable = self.lineDetectionTable.copy()
            else:
                # DETECT THE LINES ON THE PINHOLE FRAME AND
                # ADD OBSERVED LINES TO DATAFRAME
                iteration = 0
                self.log.print(f"\n# FINDING PINHOLE ARC-LINES ON IMAGE")
                iraf = False
                tmpDF = orderPixelTable.copy()
                while iteration < 3:

                    # self.windowHalf = round(windowSize / 2)
                    # sigmaLimit = self.recipeSettings['pinhole-detection-thres-sigma']

                    if tightFit:
                        self.windowHalf = round(windowSize / 2)
                        sigmaLimit = self.recipeSettings['pinhole-detection-thres-sigma']
                    elif iteration == 0:
                        sigmaLimit = 50
                        self.windowHalf = round(windowSize * 2)
                    elif iteration == 1:
                        sigmaLimit = 25
                        self.windowHalf = windowSize
                    else:
                        self.windowHalf = round(windowSize / 2)
                        sigmaLimit = self.recipeSettings['pinhole-detection-thres-sigma']

                    # print(self.windowHalf, sigmaLimit)

                    orderPixelTable = orderPixelTable.apply(self.detect_pinhole_arc_line, axis=1, iraf=iraf, sigmaLimit=sigmaLimit, iteration=iteration)

                    iteration += 1

                    # AFTER FIRST ITERATION WE CAN NARROW DOWN THE STAMP WINDOW TO REDUCE THE RISK OF IDENTIFYING THE WRONG PINHOLE
                    # self.windowHalf = 5
                    iraf = True

                    if 'detector_x_shifted' not in orderPixelTable.columns:
                        orderPixelTable['detector_x_shifted'] = orderPixelTable['detector_x']
                        orderPixelTable['detector_y_shifted'] = orderPixelTable['detector_y']

                    orderPixelTable['x_diff'] = orderPixelTable['detector_x_shifted'] - orderPixelTable['observed_x']
                    orderPixelTable['y_diff'] = orderPixelTable['detector_y_shifted'] - orderPixelTable['observed_y']

                    # FIND THE GLOBAL SHIFT SEEN BETWEEN PREDICTED AND OBSERVED PIXEL POSITIONS IN EACH ORDER ... THE MOVE TO A NEW SET OF PREDICTED LINE POSITIONS USING THAT SHIFT
                    uniqueorders = orderPixelTable['order'].unique()
                    for o in uniqueorders:
                        mask = (orderPixelTable['order'] == o)

                        meanx, medianx, stdx = sigma_clipped_stats(orderPixelTable.loc[mask]['x_diff'], sigma=3.0, stdfunc="mad_std", cenfunc="median")
                        meany, mediany, stdy = sigma_clipped_stats(orderPixelTable.loc[mask]['y_diff'], sigma=3.0, stdfunc="mad_std", cenfunc="median")
                        if np.isnan(medianx) or np.isnan(mediany):
                            self.log.warning(f"Could not find any arc lines in order {o}.")
                            continue

                        orderPixelTable.loc[mask, 'detector_x_shifted'] = orderPixelTable.loc[mask]['detector_x_shifted'] - medianx
                        orderPixelTable.loc[mask, 'detector_y_shifted'] = orderPixelTable.loc[mask]['detector_y_shifted'] - mediany
                        orderPixelTable.loc[mask, 'xy_diff'] = np.sqrt(np.square(orderPixelTable.loc[mask]["x_diff"]) + np.square(orderPixelTable.loc[mask]["y_diff"]))
                        sys.stdout.flush()
                        sys.stdout.write("\x1b[1A\x1b[2K")
                        self.log.print(f"\t ITERATION {iteration}: Median X Y difference between predicted and measured positions: {medianx:0.5f},{mediany:0.5f} (order {o})")

            # MAKE A CLEAN COPY OF THE DETECTION TABLE ... USED FOR PIPELINE TUNING ONLY
            lineDetectionTable = orderPixelTable.copy()

            # COLLECT MISSING LINES
            mask = (orderPixelTable['observed_x'].isnull())
            missingLines = orderPixelTable.loc[mask]
            # GROUP RESULTS BY WAVELENGTH
            lineGroups = missingLines.groupby(['wavelength', 'order'])
            lineGroups = lineGroups.size().to_frame(name='count').reset_index()

            # CREATE THE LIST OF INCOMPLETE MULTIPINHOLE WAVELENGTHS & ORDER SETS TO DROP
            orderPixelTable["dropped"] = False
            if "SPAT" in self.recipeName.upper():
                missingLineThreshold = 9 - self.recipeSettings['mph_line_set_min']
                mask = (lineGroups['count'] > missingLineThreshold)
                lineGroups = lineGroups.loc[mask]
                setsToDrop = lineGroups[['wavelength', 'order']]
                s = orderPixelTable[['wavelength', 'order']].merge(setsToDrop, indicator=True, how='left')
                s["dropped"] = False
                s.loc[(s["_merge"] == "both"), "dropped"] = True
                orderPixelTable["droppedOnMissing"] = s["dropped"].values
                orderPixelTable.loc[(orderPixelTable["droppedOnMissing"] == True), "dropped"] = True

            # DROP MISSING VALUES
            orderPixelTable.dropna(axis='index', how='any', subset=[
                'observed_x'], inplace=True)

            detectedLines = len(orderPixelTable.index)
            percentageDetectedLines = (float(detectedLines) / float(totalLines))
            percentageDetectedLines = float("{:.6f}".format(percentageDetectedLines))

            utcnow = datetime.utcnow()
            utcnow = utcnow.strftime("%Y-%m-%dT%H:%M:%S")

            if "DISP" in self.recipeName.upper():
                tag = "single"
            else:
                tag = "multi"

            self.qc = pd.concat([self.qc, pd.Series({
                "soxspipe_recipe": self.recipeName,
                "qc_name": "TLINE",
                "qc_value": totalLines,
                "qc_comment": f"Total number of line in {tag} line-list",
                "qc_unit": "lines",
                "obs_date_utc": self.dateObs,
                "reduction_date_utc": utcnow,
                "to_header": True
            }).to_frame().T], ignore_index=True)

            self.qc = pd.concat([self.qc, pd.Series({
                "soxspipe_recipe": self.recipeName,
                "qc_name": "NLINE",
                "qc_value": detectedLines,
                "qc_comment": f"Number of lines detected in {tag} pinhole frame",
                "qc_unit": "lines",
                "obs_date_utc": self.dateObs,
                "reduction_date_utc": utcnow,
                "to_header": True
            }).to_frame().T], ignore_index=True)

            self.qc = pd.concat([self.qc, pd.Series({
                "soxspipe_recipe": self.recipeName,
                "qc_name": "PLINE",
                "qc_value": percentageDetectedLines,
                "qc_comment": f"Proportion of input line-list lines detected on {tag} pinhole frame",
                "qc_unit": None,
                "obs_date_utc": self.dateObs,
                "reduction_date_utc": utcnow,
                "to_header": True
            }).to_frame().T], ignore_index=True)

            # GROUP FOUND LINES INTO SETS AND CLIP ON MEAN XY SHIFT RESIDUALS
            if True:
                orderPixelTable = self._clip_on_measured_line_metrics(orderPixelTable)

            # ITERATIVELY FIT THE POLYNOMIAL SOLUTIONS TO THE DATA
            fitFound = False
            tryCount = 0
            while not fitFound and tryCount < 5:

                popt_x, popt_y, goodLinesTable, clippedLinesTable = self.fit_polynomials(
                    orderPixelTable=orderPixelTable,
                    wavelengthDeg=wavelengthDeg,
                    orderDeg=orderDeg,
                    slitDeg=slitDeg,
                    missingLines=missingLines
                )

                if isinstance(popt_x, np.ndarray) and isinstance(popt_y, np.ndarray):
                    fitFound = True
                else:
                    if self.settings["tune-pipeline"]:
                        raise ArithmeticError("Could not converge on a good fit to the dispersion solution. Please check the quality of your data or adjust your fitting parameters.")

                    if not isinstance(wavelengthDeg, list):
                        degList = [wavelengthDeg, orderDeg, slitDeg]
                        degList[degList.index(max(degList))] -= 1
                        wavelengthDeg, orderDeg, slitDeg = degList
                    elif popt_x == "xerror":
                        degList = [wavelengthDeg[0], orderDeg[0], slitDeg[0]]
                        degList[degList.index(max(degList))] -= 1
                        wavelengthDeg[0], orderDeg[0], slitDeg[0] = degList
                    elif popt_y == "yerror":
                        degList = [wavelengthDeg[1], orderDeg[1], slitDeg[1]]
                        degList[degList.index(max(degList))] -= 1
                        wavelengthDeg[1], orderDeg[1], slitDeg[1] = degList

                    self.recipeSettings["order-deg"] = orderDeg
                    self.recipeSettings["wavelength-deg"] = wavelengthDeg

                    # WHICH RECIPE ARE WE WORKING WITH?
                    if self.firstGuessMap:
                        self.recipeSettings["slit-deg"] = slitDeg

                    self.log.print(f"Wavelength, Order and Slit fitting orders reduced to {wavelengthDeg}, {orderDeg}, {slitDeg} to try and achieve a dispersion solution.")
                    tryCount += 1
                    if tryCount == 5:
                        self.log.error(f"Could not converge on a good fit to the dispersion solution. Please check the quality of your data or adjust your fitting parameters.")
                        raise ArithmeticError("Could not converge on a good fit to the dispersion solution. Please check the quality of your data or adjust your fitting parameters.")

            if bootstrap_dispersion_solution:
                # WRITE THE MAP TO FILE
                mapPath = self.write_map_to_file(
                    popt_x, popt_y, orderDeg, wavelengthDeg, slitDeg)
                orderPixelTable = self.create_new_static_line_list(dispersionMapPath=mapPath)

                boost = True
                bootstrap_dispersion_solution = False
                from tabulate import tabulate
                print(tabulate(orderPixelTable, headers='keys', tablefmt='psql'))

        # GET FILENAME FOR THE LINE LISTS
        if not self.sofName:
            filename = filenamer(
                log=self.log,
                frame=self.pinholeFrame,
                settings=self.settings
            )
            goodLinesFN = filename.replace(".fits", "_FITTED_LINES.fits")
            missingLinesFN = filename.replace(".fits", "_MISSED_LINES.fits")
        else:
            goodLinesFN = self.sofName + "_FITTED_LINES.fits"
            missingLinesFN = self.sofName + "_MISSED_LINES.fits"

        # WRITE CLIPPED LINE LIST TO FILE
        keepColumns = ['wavelength', 'order', 'slit_index', 'slit_position', 'detector_x', 'detector_y', 'observed_x', 'observed_y', 'x_diff', 'y_diff', 'fit_x', 'fit_y', 'residuals_x', 'residuals_y', 'residuals_xy', 'sigma_clipped', "sharpness", "roundness1", "roundness2", "npix", "sky", "peak", "flux", 'fwhm_pin_px', 'R_pin', 'pixelScaleNm', 'detector_x_shifted', 'detector_y_shifted', 'R_slit', 'fwhm_slit_px']
        if "ion" in goodLinesTable.columns:
            keepColumns.insert(0, 'ion')
        clippedLinesTable['sigma_clipped'] = True
        clippedLinesTable['R'] = np.nan
        clippedLinesTable['pixelScaleNm'] = np.nan

        self.CLINE = len(clippedLinesTable.index)
        self.qc = pd.concat([self.qc, pd.Series({
            "soxspipe_recipe": self.recipeName,
            "qc_name": "CLINE",
            "qc_value": self.CLINE,
            "qc_comment": f"Total number of detected lines clipped during solution fitting",
            "qc_unit": "lines",
            "obs_date_utc": self.dateObs,
            "reduction_date_utc": utcnow,
            "to_header": True
        }).to_frame().T], ignore_index=True)

        if False:
            # SAVE OUT NEW STATIC LINE LIST
            from astropy.table import Table
            freshTable = goodLinesTable[["ion", 'wavelength', 'order', 'slit_index', 'slit_position', 'observed_x', 'observed_y']]
            freshTable.rename(columns={"observed_x": "detector_x", "observed_y": "detector_y"}, inplace=True)
            t = Table.from_pandas(freshTable)
            t.write("Hg_clean.fits", overwrite=True)

        if len(clippedLinesTable.index):
            goodAndClippedLines = pd.concat([clippedLinesTable[keepColumns], goodLinesTable[keepColumns]], ignore_index=True)
        else:
            goodAndClippedLines = goodLinesTable[keepColumns]
        goodLinesTable = goodLinesTable[keepColumns]

        # SORT BY COLUMN NAME
        goodAndClippedLines.sort_values(['order', 'wavelength', 'slit_index'], inplace=True)
        goodLinesTable.sort_values(['order', 'wavelength', 'slit_index'], inplace=True)

        # WRITE GOOD LINE LIST TO FILE
        t = Table.from_pandas(goodAndClippedLines)
        filePath = f"{self.qcDir}/{goodLinesFN}"
        if not self.settings["tune-pipeline"]:
            t.write(filePath, overwrite=True)
        self.products = pd.concat([self.products, pd.Series({
            "soxspipe_recipe": self.recipeName,
            "product_label": "DISP_MAP_LINES",
            "file_name": goodLinesFN,
            "file_type": "FITS",
            "obs_date_utc": self.dateObs,
            "reduction_date_utc": utcnow,
            "product_desc": f"{self.arm} dispersion solution fitted lines",
            "file_path": filePath,
            "label": "QC"
        }).to_frame().T], ignore_index=True)

        # WRITE MISSING LINE LIST TO FILE
        keepColumns = ['wavelength', 'order', 'slit_index', 'slit_position', 'detector_x', 'detector_y']
        # SORT BY COLUMN NAME
        missingLines.sort_values(['order', 'wavelength', 'slit_index'], inplace=True)
        t = Table.from_pandas(missingLines[keepColumns])
        filePath = f"{self.qcDir}/{missingLinesFN}"
        if not self.settings["tune-pipeline"]:
            t.write(filePath, overwrite=True)
        self.products = pd.concat([self.products, pd.Series({
            "soxspipe_recipe": self.recipeName,
            "product_label": "DISP_MAP_LINES_MISSING",
            "file_name": missingLinesFN,
            "file_type": "FITS",
            "obs_date_utc": self.dateObs,
            "reduction_date_utc": utcnow,
            "product_desc": f"{self.arm} undetected arc lines",
            "file_path": filePath,
            "label": "QC"
        }).to_frame().T], ignore_index=True)

        # WRITE THE MAP TO FILE
        if not self.settings["tune-pipeline"]:
            mapPath = self.write_map_to_file(
                popt_x, popt_y, orderDeg, wavelengthDeg, slitDeg)
        else:
            mapPath = None
            self.create2DMap = False

        if self.firstGuessMap and self.orderTable and self.create2DMap:
            mapImagePath = self.map_to_image(dispersionMapPath=mapPath, orders=list(goodLinesTable['order'].unique()))
            res_plots = self._create_dispersion_map_qc_plot(
                xcoeff=popt_x,
                ycoeff=popt_y,
                orderDeg=orderDeg,
                wavelengthDeg=wavelengthDeg,
                slitDeg=slitDeg,
                orderPixelTable=goodLinesTable,
                missingLines=missingLines,
                allClippedLines=clippedLinesTable,
                dispMap=mapPath,
                dispMapImage=mapImagePath
            )
            return mapPath, mapImagePath, res_plots, self.qc, self.products, lineDetectionTable

        res_plots = self._create_dispersion_map_qc_plot(
            xcoeff=popt_x,
            ycoeff=popt_y,
            orderDeg=orderDeg,
            wavelengthDeg=wavelengthDeg,
            slitDeg=slitDeg,
            orderPixelTable=goodLinesTable,
            missingLines=missingLines,
            allClippedLines=clippedLinesTable
        )

        self.log.debug('completed the ``get`` method')
        return mapPath, None, res_plots, self.qc, self.products, lineDetectionTable

    def get_predicted_line_list(
            self):
        """*lift the predicted line list from the static calibrations*

        **Return:**

        - ``orderPixelTable`` -- a panda's data-frame containing wavelength,order,slit_index,slit_position,detector_x,detector_y
        """
        self.log.debug('starting the ``get_predicted_line_list`` method')

        from astropy.table import Table

        kw = self.kw
        pinholeFrame = self.pinholeFrame
        dp = self.detectorParams

        # WHICH TYPE OF PINHOLE FRAME DO WE HAVE - SINGLE OR MULTI
        if self.pinholeFrame.header[kw("DPR_TECH")] == "ECHELLE,PINHOLE":
            frameTech = "single"
        elif self.pinholeFrame.header[kw("DPR_TECH")] == "ECHELLE,MULTI-PINHOLE":
            frameTech = "multi"
        else:
            raise TypeError(
                "The input frame needs to be a calibrated single- or multi-pinhole arc lamp frame")

        # FIND THE APPROPRIATE PREDICTED LINE-LIST
        arm = self.arm
        if arm != "NIR" and kw('WIN_BINX') in pinholeFrame.header:
            binx = int(self.pinholeFrame.header[kw('WIN_BINX')])
            biny = int(self.pinholeFrame.header[kw('WIN_BINY')])
        else:
            binx = 1
            biny = 1

        # READ THE FILE
        home = expanduser("~")

        calibrationRootPath = get_calibrations_path(log=self.log, settings=self.settings)

        predictedLinesFile = calibrationRootPath + "/" + dp["predicted pinhole lines"][frameTech][f"{binx}x{biny}"]

        # LINE LIST TO PANDAS DATAFRAME
        dat = Table.read(predictedLinesFile, format='fits')
        orderPixelTable = dat.to_pandas()

        # RENAME ALL COLUMNS FOR CONSISTENCY
        listName = []
        listName[:] = [l if l else l for l in listName]
        orderPixelTable.columns = [d.lower() if d.lower() in [
            "order", "wavelength"] else d for d in orderPixelTable.columns]

        floats = ["wavelength", 'slit_position', 'detector_x', 'detector_y']
        ints = ['order', 'slit_index']
        strings = ['ion']
        for f in floats:
            try:
                orderPixelTable[f] = orderPixelTable[f].astype(float)
            except:
                pass
        for i in ints:
            try:
                orderPixelTable[i] = orderPixelTable[i].astype(int)
            except:
                pass
        for s in strings:
            try:
                orderPixelTable[s] = orderPixelTable[s].astype(str)
            except:
                pass

        if "delete" in orderPixelTable.columns:
            # REMOVE FILTERED ROWS FROM DATA FRAME
            mask = (orderPixelTable['delete'] == 1)
            orderPixelTable.drop(index=orderPixelTable[mask].index, inplace=True)

        # THERE ARE DUPLICATES IN XSHOOTER LINE LIST
        orderPixelTable.drop_duplicates(inplace=True)

        # FITS TO PYTHON INDEXING
        # PHOTUTILS CENTRE OF BOTTOM LEFT PIXEL IS (0,0) BUT FOR WCS IT IS (1,1)
        # AND FOR PYTHON IT IS ALSO (0,0)
        if self.inst != "SOXS":
            orderPixelTable["detector_x"] -= 1.0
            orderPixelTable["detector_y"] -= 1.0
        elif True:

            if arm == "NIR" or arm == "VIS":
                dp = self.detectorParams
                science_pixels = dp["science-pixels"]
                xlen = science_pixels["columns"]["end"] - \
                    science_pixels["columns"]["start"]
                ylen = science_pixels["rows"]["end"] - science_pixels["rows"]["start"]
            else:
                orderPixelTable["detector_x"] -= science_pixels["columns"]["start"]
                orderPixelTable["detector_y"] -= science_pixels["rows"]["start"]
            if arm == "NIR":
                pass
                # THIS IS TEMPORARALLY NEEDED TO ADJUST OLD LINE LIST
                # orderPixelTable["detector_x"] += 7
                # orderPixelTable["detector_y"] -= 9
                # mask = (orderPixelTable['order'] == 24)
                # orderPixelTable.loc[mask, "detector_x"] += 1
                # orderPixelTable.loc[mask, "detector_y"] += 1
                # mask = (orderPixelTable['order'] == 23)
                # orderPixelTable.loc[mask, "detector_x"] += 1
                # orderPixelTable.loc[mask, "detector_y"] += 1
                # mask = (orderPixelTable['order'] == 22)
                # orderPixelTable.loc[mask, "detector_x"] += 1
                # orderPixelTable.loc[mask, "detector_y"] += 1
                # mask = (orderPixelTable['order'] == 21)
                # orderPixelTable.loc[mask, "detector_x"] += 1
                # orderPixelTable.loc[mask, "detector_y"] += 2
                # mask = (orderPixelTable['order'] == 20)
                # orderPixelTable.loc[mask, "detector_x"] += 2
                # orderPixelTable.loc[mask, "detector_y"] += 2
                # mask = (orderPixelTable['order'] == 19)
                # orderPixelTable.loc[mask, "detector_x"] += 2
                # orderPixelTable.loc[mask, "detector_y"] += 2
                # mask = (orderPixelTable['order'] == 18)
                # orderPixelTable.loc[mask, "detector_x"] += 2
                # orderPixelTable.loc[mask, "detector_y"] += 2
                # mask = (orderPixelTable['order'] == 17)
                # orderPixelTable.loc[mask, "detector_x"] += 2
                # orderPixelTable.loc[mask, "detector_y"] += 3
                # mask = (orderPixelTable['order'] == 16)
                # orderPixelTable.loc[mask, "detector_x"] += 2
                # orderPixelTable.loc[mask, "detector_y"] += 3
                # mask = (orderPixelTable['order'] == 15)
                # orderPixelTable.loc[mask, "detector_x"] += 2
                # orderPixelTable.loc[mask, "detector_y"] += 3
                # mask = (orderPixelTable['order'] == 14)
                # orderPixelTable.loc[mask, "detector_x"] += 2
                # orderPixelTable.loc[mask, "detector_y"] += 3
                # mask = (orderPixelTable['order'] == 13)
                # orderPixelTable.loc[mask, "detector_x"] += 2
                # orderPixelTable.loc[mask, "detector_y"] += 3
                # mask = (orderPixelTable['order'] == 12)
                # orderPixelTable.loc[mask, "detector_x"] += 2
                # orderPixelTable.loc[mask, "detector_y"] += 3
                # mask = (orderPixelTable['order'] == 11)
                # orderPixelTable.loc[mask, "detector_x"] += 2
                # orderPixelTable.loc[mask, "detector_y"] += 3
                # mask = (orderPixelTable['order'] == 10)
                # orderPixelTable.loc[mask, "detector_x"] += 2
                # orderPixelTable.loc[mask, "detector_y"] += 3
                # mask = (orderPixelTable['order'] == 9)
                # orderPixelTable.loc[mask, "detector_x"] += 2
                # orderPixelTable.loc[mask, "detector_y"] += 3
                # # mask = (orderPixelTable['order'] == 2)
                # # orderPixelTable.loc[mask, "detector_x"] -= 23
                # # orderPixelTable.loc[mask, "detector_y"] -= 0
                # # mask = (orderPixelTable['order'] == 3)
                # # orderPixelTable.loc[mask, "detector_x"] -= 25
                # # orderPixelTable.loc[mask, "detector_y"] -= 0
                # # mask = (orderPixelTable['order'] == 4)
                # # orderPixelTable.loc[mask, "detector_x"] -= 10
                # # orderPixelTable.loc[mask, "detector_y"] -= 2
                # from astropy.table import Table
                # t = Table.from_pandas(orderPixelTable)
                # # t.write("Xe.fits", overwrite=True)
                # t.write("ArHgNeXe_clean_within_2.0pixel.fits", overwrite=True)

        if not self.firstGuessMap:
            slitIndex = int(dp["mid_slit_index"])
            # REMOVE FILTERED ROWS FROM DATA FRAME
            mask = (orderPixelTable['slit_index'] == slitIndex)
            orderPixelTable = orderPixelTable.loc[mask]

        # WANT TO DETERMINE SYSTEMATIC SHIFT IF FIRST GUESS SOLUTION PRESENT
        if self.firstGuessMap:
            # ADD SOME EXTRA COLUMNS TO DATAFRAME

            # FILTER THE PREDICTED LINES TO ONLY SLIT POSITION INCLUDED IN
            # SINGLE PINHOLE FRAMES
            slitIndex = int(dp["mid_slit_index"])

            # GET THE OBSERVED PIXELS VALUES
            orderPixelTable = dispersion_map_to_pixel_arrays(
                log=self.log,
                dispersionMapPath=self.firstGuessMap,
                orderPixelTable=orderPixelTable
            )

            # CREATE A COPY OF THE DATA-FRAME TO DETERMINE SHIFTS
            tmpList = orderPixelTable.copy()

            mask = (tmpList['slit_index'] == slitIndex)
            tmpList.loc[mask, 'shift_x'] = tmpList.loc[
                mask, 'detector_x'].values - tmpList.loc[mask, 'fit_x'].values
            tmpList.loc[mask, 'shift_y'] = tmpList.loc[
                mask, 'detector_y'].values - tmpList.loc[mask, 'fit_y'].values

            # MERGING SHIFTS INTO MAIN DATAFRAME
            tmpList = tmpList.loc[tmpList['shift_x'].notnull(
            ), ['wavelength', 'order', 'shift_x', 'shift_y']]
            orderPixelTable = orderPixelTable.merge(tmpList, on=[
                'wavelength', 'order'], how='outer')

            # DROP ROWS WITH MISSING SHIFTS
            orderPixelTable.dropna(axis='index', how='any', subset=[
                'shift_x'], inplace=True)

            # SHIFT DETECTOR LINE PIXEL POSITIONS BY SHIFTS
            # UPDATE FILTERED VALUES
            orderPixelTable.loc[
                :, 'detector_x'] -= orderPixelTable.loc[:, 'shift_x']
            orderPixelTable.loc[
                :, 'detector_y'] -= orderPixelTable.loc[:, 'shift_y']

            # DROP HELPER COLUMNS
            orderPixelTable.drop(columns=['fit_x', 'fit_y',
                                          'shift_x', 'shift_y'], inplace=True)

            # DROP MPH SETS THAT DON'T CONTAIN AlL SLIT-POSITIONS
            lineGroups = orderPixelTable.groupby(['wavelength', 'order'])
            lineGroups = lineGroups.size().to_frame(name='count').reset_index()
            fullSet = lineGroups['count'].max()
            mask = (lineGroups['count'] < fullSet)
            lineGroups = lineGroups.loc[mask]
            setsToDrop = lineGroups[['wavelength', 'order']]
            s = orderPixelTable[['wavelength', 'order']].merge(setsToDrop, indicator=True, how='left')
            s["dropped"] = False
            s.loc[(s["_merge"] == "both"), "dropped"] = True
            orderPixelTable["droppedOnMissing"] = s["dropped"].values
            orderPixelTable = orderPixelTable.loc[(orderPixelTable["droppedOnMissing"] == False)]
            orderPixelTable.drop(columns=['droppedOnMissing'], inplace=True)

        self.log.debug('completed the ``get_predicted_line_list`` method')
        return orderPixelTable

    def detect_pinhole_arc_line(
            self,
            predictedLine,
            iraf=True,
            sigmaLimit=3,
            iteration=False):
        """*detect the observed position of an arc-line given the predicted pixel positions*

        **Key Arguments:**

        - ``predictedLine`` -- single predicted line coordinates from predicted line-list
        - ``iraf`` -- use IRAF star finder to generate a FWHM
        - ``sigmaLimit`` -- the lower sigma limit for arc line to be considered detected
        - ``iteration`` -- which detect and shift iteration are we on?

        **Return:**

        - ``predictedLine`` -- the line with the observed pixel coordinates appended (if detected, otherwise nan)
        """
        self.log.debug('starting the ``detect_pinhole_arc_line`` method')

        import numpy as np
        from photutils import DAOStarFinder, IRAFStarFinder
        from astropy.stats import sigma_clipped_stats
        # ASTROPY HAS RESET LOGGING LEVEL -- FIX
        import logging
        logging.getLogger().setLevel(logging.INFO + 5)

        if iteration is False:
            iterationText = ""
        else:
            iterationText = f", iter #{iteration}"

        pinholeFrame = self.pinholeFrameMasked
        windowHalf = self.windowHalf
        if 'detector_x_shifted' in predictedLine:
            x = predictedLine['detector_x_shifted']
            y = predictedLine['detector_y_shifted']
        else:
            x = predictedLine['detector_x']
            y = predictedLine['detector_y']

        order = predictedLine['order']
        wl = predictedLine['wavelength']

        # CLIP A STAMP FROM IMAGE AROUNDS PREDICTED POSITION
        xlow = round(np.max([x - windowHalf, 0]))
        xup = round(np.min([x + windowHalf, pinholeFrame.shape[1]]))
        ylow = round(np.max([y - windowHalf, 0]))
        yup = round(np.min([y + windowHalf, pinholeFrame.shape[0]]))
        stamp = pinholeFrame[ylow:yup, xlow:xup]

        # FORCE CONVERSION OF CCDData OBJECT TO NUMPY ARRAY
        stamp = np.ma.array(stamp.data, mask=stamp.mask)
        # USE DAOStarFinder TO FIND LINES WITH 2D GAUSSIAN FITTING
        mean, median, std = sigma_clipped_stats(stamp, sigma=3.0, stdfunc="mad_std", cenfunc="median")

        try:
            # LET AS MANY LINES BE DETECTED AS POSSIBLE ... WE WILL CLEAN UP LATER
            daofind = DAOStarFinder(
                fwhm=3., threshold=sigmaLimit * std, roundlo=-2.0, roundhi=2.0, sharplo=-0.5, sharphi=2.0, exclude_border=False)
            # SUBTRACTING MEDIAN MAKES LITTLE TO NO DIFFERENCE .. EXCEPT FOR LOW SIGNAL IMAGES
            # sources = daofind(stamp.data, mask=stamp.mask)
            sources = daofind(stamp.data - median, mask=stamp.mask)
        except Exception as e:
            sources = None

        old_resid = windowHalf * 4
        selectedSource = None
        observed_x = np.nan
        observed_y = np.nan
        fwhm = np.nan

        if sources:
            # FIND SOURCE CLOSEST TO CENTRE
            if len(sources) > 1:
                for source in sources:
                    tmp_x = source['xcentroid']
                    tmp_y = source['ycentroid']
                    new_resid = ((windowHalf - tmp_x)**2 +
                                 (windowHalf - tmp_y)**2)**0.5
                    if new_resid < old_resid:
                        observed_x = tmp_x + xlow
                        observed_y = tmp_y + ylow
                        stamp_x = tmp_x
                        stamp_y = tmp_y
                        old_resid = new_resid
                        selectedSource = source
                        detectionSigma = source['peak'] / std

            else:
                observed_x = sources[0]['xcentroid'] + xlow
                observed_y = sources[0]['ycentroid'] + ylow
                stamp_x = sources[0]['xcentroid']
                stamp_y = sources[0]['ycentroid']
                selectedSource = sources[0]
                detectionSigma = sources[0]['peak'] / std

            if iraf:
                try:
                    # Rerun detection with IRAFStarFinder
                    iraf_find = IRAFStarFinder(
                        fwhm=3., threshold=1 * std, roundlo=-2.0, roundhi=2.0, sharplo=-1, sharphi=3.0, exclude_border=True, xycoords=[(stamp_x, stamp_y)])
                    iraf_sources = iraf_find(stamp)
                    fwhm = iraf_sources['fwhm'][0]
                except Exception as e:
                    fwhm = np.nan
                    # print(np.shape(stamp))
                    # print(stamp_x,stamp_y)
                    # print(e)
                    pass

            if False and iteration == 2:
                import random
                ran = random.randint(1, 3)
                # if int(wl) == 7125:
                # if True or order == 2.:
                if True:
                    print(x, y)
                    # if ran == 1:
                    import matplotlib.pyplot as plt
                    plt.clf()
                    plt.imshow(stamp)
                    # plt.show()
                    plt.scatter(0, 0, marker='x', s=30)
                    plt.scatter(observed_x - xlow, observed_y -
                                ylow, s=30)
                    plt.text(windowHalf - 2, windowHalf - 2, f"{observed_x-xlow:0.2f},{observed_y -ylow:0.2f}", fontsize=16, c="black", verticalalignment='bottom')
                    plt.text(2, 2, f"{sigmaLimit}$\\sigma$, order {order}{iterationText}\n{wl:0.1f},{detectionSigma:0.1f}", fontsize=16, c="white", verticalalignment='bottom')
                    plt.show()

        # plt.show()

        keepValues = ["sharpness", "roundness1", "roundness2", "npix", "sky", "peak", "flux"]
        if not selectedSource:
            for k in keepValues:
                predictedLine[k] = np.nan
        else:
            for k in keepValues:
                predictedLine[k] = selectedSource[k]

        predictedLine['observed_x'] = observed_x
        predictedLine['observed_y'] = observed_y
        if iraf:
            predictedLine['fwhm_pin_px'] = fwhm

        self.log.debug('completed the ``detect_pinhole_arc_line`` method')
        return predictedLine

    def write_map_to_file(
            self,
            xcoeff,
            ycoeff,
            orderDeg,
            wavelengthDeg,
            slitDeg):
        """*write out the fitted polynomial solution coefficients to file*

        **Key Arguments:**

        - ``xcoeff`` -- the x-coefficients
        - ``ycoeff`` -- the y-coefficients
        - ``orderDeg`` -- degree of the order fitting
        - ``wavelengthDeg`` -- degree of wavelength fitting
        - ``slitDeg`` -- degree of the slit fitting (False for single pinhole)

        **Return:**

        - ``disp_map_path`` -- path to the saved file
        """
        self.log.debug('starting the ``write_map_to_file`` method')

        import pandas as pd
        from astropy.table import Table
        from astropy.io import fits
        from contextlib import suppress
        import copy
        import math
        arm = self.arm
        kw = self.kw

        # SORT X COEFFICIENT OUTPUT TO WRITE TO FILE
        if isinstance(orderDeg, list):
            coeff_dict_x = {}
            coeff_dict_x["axis"] = "x"
            coeff_dict_x["order_deg"] = orderDeg[0]
            coeff_dict_x["wavelength_deg"] = wavelengthDeg[0]
            coeff_dict_x["slit_deg"] = slitDeg[0]
            n_coeff = 0
            for i in range(0, orderDeg[0] + 1):
                for j in range(0, wavelengthDeg[0] + 1):
                    for k in range(0, slitDeg[0] + 1):
                        coeff_dict_x[f'c{i}{j}{k}'] = xcoeff[n_coeff]
                        n_coeff += 1
        else:
            coeff_dict_x = {}
            coeff_dict_x["axis"] = "x"
            coeff_dict_x["order_deg"] = orderDeg
            coeff_dict_x["wavelength_deg"] = wavelengthDeg
            coeff_dict_x["slit_deg"] = slitDeg
            n_coeff = 0
            for i in range(0, orderDeg + 1):
                for j in range(0, wavelengthDeg + 1):
                    for k in range(0, slitDeg + 1):
                        coeff_dict_x[f'c{i}{j}{k}'] = xcoeff[n_coeff]
                        n_coeff += 1

        # SORT Y COEFFICIENT OUTPUT TO WRITE TO FILE
        if isinstance(orderDeg, list):
            coeff_dict_y = {}
            coeff_dict_y["axis"] = "y"
            coeff_dict_y["order_deg"] = orderDeg[1]
            coeff_dict_y["wavelength_deg"] = wavelengthDeg[1]
            coeff_dict_y["slit_deg"] = slitDeg[1]
            n_coeff = 0
            for i in range(0, orderDeg[1] + 1):
                for j in range(0, wavelengthDeg[1] + 1):
                    for k in range(0, slitDeg[1] + 1):
                        coeff_dict_y[f'c{i}{j}{k}'] = ycoeff[n_coeff]
                        n_coeff += 1
        else:
            coeff_dict_y = {}
            coeff_dict_y["axis"] = "y"
            coeff_dict_y["order_deg"] = orderDeg
            coeff_dict_y["wavelength_deg"] = wavelengthDeg
            coeff_dict_y["slit_deg"] = slitDeg
            n_coeff = 0
            for i in range(0, orderDeg + 1):
                for j in range(0, wavelengthDeg + 1):
                    for k in range(0, slitDeg + 1):
                        coeff_dict_y[f'c{i}{j}{k}'] = ycoeff[n_coeff]
                        n_coeff += 1

        # DETERMINE WHERE TO WRITE THE FILE
        home = expanduser("~")
        outDir = self.settings["workspace-root-dir"].replace("~", home)
        outDir += f"/reduced/{self.startNightDate}/{self.recipeName}/"
        outDir = outDir.replace("//", "/")
        # Recursively create missing directories
        if not os.path.exists(outDir):
            os.makedirs(outDir)

        if not self.sofName:
            filename = filenamer(
                log=self.log,
                frame=self.pinholeFrame,
                settings=self.settings
            )
        else:
            filename = self.sofName + ".fits"

        header = copy.deepcopy(self.pinholeFrame.header)
        header.pop(kw("DPR_TECH"))
        header.pop(kw("DPR_CATG"))
        header.pop(kw("DPR_TYPE"))

        with suppress(KeyError):
            header.pop(kw("DET_READ_SPEED"))
        with suppress(KeyError, LookupError):
            header.pop(kw("CONAD"))
        with suppress(KeyError):
            header.pop(kw("GAIN"))
        with suppress(KeyError):
            header.pop(kw("RON"))

        if slitDeg == 0 or slitDeg == [0, 0]:
            # filename = filename.split("ARC")[0] + "DISP_MAP.fits"
            header[kw("PRO_TECH").upper()] = "ECHELLE,PINHOLE"
        else:
            # filename = filename.split("ARC")[0] + "2D_MAP.fits"
            header[kw("PRO_TECH").upper()] = "ECHELLE,MULTI-PINHOLE"
        filePath = f"{outDir}/{filename}"

        df = pd.DataFrame([coeff_dict_x, coeff_dict_y])
        t = Table.from_pandas(df)

        # SORT COLUMNS
        cols = list(t.columns)
        startList = ['axis', 'order_deg', 'wavelength_deg', 'slit_deg']
        cols = [c for c in cols if c not in startList]
        cols.sort()
        cols = startList + cols
        t = t[cols]

        BinTableHDU = fits.table_to_hdu(t)

        header[kw("SEQ_ARM").upper()] = arm
        header[kw("PRO_TYPE").upper()] = "REDUCED"
        header[kw("PRO_CATG").upper()] = f"DISP_TAB_{arm}".upper()
        self.dispMapHeader = header

        # WRITE QCs TO HEADERS
        for n, v, c, h in zip(self.qc["qc_name"].values, self.qc["qc_value"].values, self.qc["qc_comment"].values, self.qc["to_header"].values):
            if h:
                header[f"ESO QC {n}".upper()] = (v, c)

        priHDU = fits.PrimaryHDU(header=header)

        hduList = fits.HDUList([priHDU, BinTableHDU])
        hduList.writeto(filePath, checksum=True, overwrite=True)

        if False:
            cache = self.settings["workspace-root-dir"].replace("~", home) + "/.cache"
            # Recursively create missing directories
            if not os.path.exists(cache):
                os.makedirs(cache)
            polyOrders = [orderDeg, wavelengthDeg, slitDeg]
            if isinstance(orderDeg, list):
                merged_list = []
                for sublist in polyOrders:
                    merged_list.extend(sublist)
                polyOrders = merged_list
            polyOrders[:] = [str(l) for l in polyOrders]
            polyOrders = "".join(polyOrders)
            filename = f"{self.recipeName}_{self.arm}_{polyOrders}.fits"
            cacheFilePath = f"{cache}/{filename}"
            hduList.writeto(cacheFilePath, checksum=True, overwrite=True)

        self.log.debug('completed the ``write_map_to_file`` method')
        return filePath

    def calculate_residuals(
            self,
            orderPixelTable,
            xcoeff,
            ycoeff,
            orderDeg,
            wavelengthDeg,
            slitDeg,
            writeQCs=False,
            pixelRange=False):
        """*calculate residuals of the polynomial fits against the observed line positions*

        **Key Arguments:**

        - ``orderPixelTable`` -- the predicted line list as a data frame
        - ``xcoeff`` -- the x-coefficients
        - ``ycoeff`` -- the y-coefficients
        - ``orderDeg`` -- degree of the order fitting
        - ``wavelengthDeg`` -- degree of wavelength fitting
        - ``slitDeg`` -- degree of the slit fitting (False for single pinhole)
        - ``writeQCs`` -- write the QCs to dataframe? Default *False*
        - ``pixelRange`` -- return centre pixel *and* +- 2nm from the centre pixel (to measure the pixel scale)

        **Return:**

        - ``residuals`` -- combined x-y residuals
        - ``mean`` -- the mean of the combine residuals
        - ``std`` -- the stdev of the combine residuals
        - ``median`` -- the median of the combine residuals
        """
        self.log.debug('starting the ``calculate_residuals`` method')

        import numpy as np
        import pandas as pd

        arm = self.arm

        utcnow = datetime.utcnow()
        utcnow = utcnow.strftime("%Y-%m-%dT%H:%M:%S")

        # ADD EXPONENTS TO ORDERTABLE UP-FRONT
        if isinstance(orderDeg, list):
            orderDegx = orderDeg[0]
            orderDegy = orderDeg[1]
            wavelengthDegx = wavelengthDeg[0]
            wavelengthDegy = wavelengthDeg[1]
            slitDegx = slitDeg[0]
            slitDegy = slitDeg[1]
        else:
            orderDegx = orderDeg
            orderDegy = orderDeg
            wavelengthDegx = wavelengthDeg
            wavelengthDegy = wavelengthDeg
            slitDegx = slitDeg
            slitDegy = slitDeg

        polyx = chebyshev_order_wavelength_polynomials(
            log=self.log, orderDeg=orderDegx, wavelengthDeg=wavelengthDegx, slitDeg=slitDegx, exponentsIncluded=True, axis="x").poly
        polyy = chebyshev_order_wavelength_polynomials(
            log=self.log, orderDeg=orderDegy, wavelengthDeg=wavelengthDegy, slitDeg=slitDegy, exponentsIncluded=True, axis="y").poly

        # CALCULATE X & Y RESIDUALS BETWEEN OBSERVED LINE POSITIONS AND POLY
        # FITTED POSITIONS
        orderPixelTable["fit_x"] = polyx(orderPixelTable, *xcoeff)
        orderPixelTable["fit_y"] = polyy(orderPixelTable, *ycoeff)

        if pixelRange == True:
            polyx = chebyshev_order_wavelength_polynomials(
                log=self.log, orderDeg=orderDegx, wavelengthDeg=wavelengthDegx, slitDeg=slitDegx, exponentsIncluded=False, axis="x").poly
            polyy = chebyshev_order_wavelength_polynomials(
                log=self.log, orderDeg=orderDegy, wavelengthDeg=wavelengthDegy, slitDeg=slitDegy, exponentsIncluded=False, axis="y").poly
            # GET THE PIXEL SCALE
            orderPixelTableHigh = orderPixelTable.copy()
            nmRange = 4.
            orderPixelTableHigh["wavelength"] = orderPixelTableHigh["wavelength"] + nmRange / 2.
            orderPixelTableHigh["fit_x"] = polyx(orderPixelTableHigh, *xcoeff)
            orderPixelTableHigh["fit_y"] = polyy(orderPixelTableHigh, *ycoeff)

            orderPixelTableLow = orderPixelTable.copy()
            orderPixelTableLow["wavelength"] = orderPixelTableLow["wavelength"] - nmRange / 2.
            orderPixelTableLow["fit_x"] = polyx(orderPixelTableLow, *xcoeff)
            orderPixelTableLow["fit_y"] = polyy(orderPixelTableLow, *ycoeff)

            orderPixelTable["fit_x_high"] = orderPixelTableHigh["fit_x"]
            orderPixelTable["fit_y_high"] = orderPixelTableHigh["fit_y"]
            orderPixelTable["fit_x_low"] = orderPixelTableLow["fit_x"]
            orderPixelTable["fit_y_low"] = orderPixelTableLow["fit_y"]

            orderPixelTable["pixelScaleNm"] = nmRange / np.power(np.power(orderPixelTable["fit_x_high"] - orderPixelTable["fit_x_low"], 2) + np.power(orderPixelTable["fit_y_high"] - orderPixelTable["fit_y_low"], 2), 0.5)
            orderPixelTable["delta_wavelength"] = orderPixelTable["pixelScaleNm"] * orderPixelTable["fwhm_pin_px"]
            orderPixelTable["R_pin"] = orderPixelTable["wavelength"] / orderPixelTable["delta_wavelength"]

            # REMOVE COLUMN FROM DATA FRAME
            orderPixelTable.drop(columns=['fit_x_high', 'fit_y_high', 'fit_x_low', 'fit_y_low'], inplace=True)

        orderPixelTable["residuals_x"] = orderPixelTable[
            "fit_x"] - orderPixelTable["observed_x"]
        orderPixelTable["residuals_y"] = orderPixelTable[
            "fit_y"] - orderPixelTable["observed_y"]

        # CALCULATE COMBINED RESIDUALS AND STATS
        orderPixelTable["residuals_xy"] = np.sqrt(np.square(
            orderPixelTable["residuals_x"]) + np.square(orderPixelTable["residuals_y"]))
        combined_res_mean = np.mean(orderPixelTable["residuals_xy"])
        combined_res_std = np.std(orderPixelTable["residuals_xy"])
        combined_res_median = np.median(orderPixelTable["residuals_xy"])

        #orderPixelTable['tilt'] = 0.0
        #orderPixelTable['distance_to_middle'] = 0.0

        # COMPUTING THE TILT ON THE SLIT IMAGE

        # for index, row in orderPixelTable.iterrows():
        # GETTING THE OBSERVED_X AND OBSERVED_Y POSITION IN THE MIDDLE OF THE SLIT
        #    if row['slit_index'] == 4:
        #        orderPixelTable.at[index,'tilt'] = 0.0
        #        orderPixelTable.at[index,'distance_to_middle'] = 0.0
        #    else:
        #        current_wave = row['wavelength']
        #        current_order = row['order']
        #        middle_row = orderPixelTable.query('wavelength == @current_wave and order == @current_order and slit_index == 4 ')
        #        middle_row = middle_row.iloc[0]#

        #        current_tilt = np.arctan((row['observed_y'] - middle_row['observed_y'])/(row['observed_x'] - middle_row['observed_x']))
        #        orderPixelTable.at[index,'tilt'] = np.pi/2 + float(current_tilt)
        #        orderPixelTable.at[index,'distance_to_middle'] = np.sqrt(float( (row['observed_x'] - middle_row['observed_x'])**2 + (row['observed_y'] - middle_row['observed_y'])**2))

        if self.arcFrame:
            self.slitWidth = self.arcFrame.header[self.kw(f"SLIT_{arm}")].replace("SLIT", "").split("x")[0]
            orderPixelTable[["R_slit", "fwhm_slit_px"]] = orderPixelTable.apply(self._calculate_resolution_on_slit, axis=1)

        else:
            orderPixelTable["R_slit"] = 0.0
            orderPixelTable["fwhm_slit_px"] = 0.0
        if writeQCs:
            absx = abs(orderPixelTable["residuals_x"])
            absy = abs(orderPixelTable["residuals_y"])
            self.qc = pd.concat([self.qc, pd.Series({
                "soxspipe_recipe": self.recipeName,
                "qc_name": "XRESMIN",
                "qc_value": absx.min(),
                "qc_comment": "[px] Minimum residual in dispersion solution fit along x-axis",
                "qc_unit": "pixels",
                "obs_date_utc": self.dateObs,
                "reduction_date_utc": utcnow,
                "to_header": True
            }).to_frame().T], ignore_index=True)
            self.qc = pd.concat([self.qc, pd.Series({
                "soxspipe_recipe": self.recipeName,
                "qc_name": "XRESMAX",
                "qc_value": absx.max(),
                "qc_comment": "[px] Maximum residual in dispersion solution fit along x-axis",
                "qc_unit": "pixels",
                "obs_date_utc": self.dateObs,
                "reduction_date_utc": utcnow,
                "to_header": True
            }).to_frame().T], ignore_index=True)
            self.qc = pd.concat([self.qc, pd.Series({
                "soxspipe_recipe": self.recipeName,
                "qc_name": "XRESRMS",
                "qc_value": absx.std(),
                "qc_comment": "[px] Std-dev of residual in dispersion solution fit along x-axis",
                "qc_unit": "pixels",
                "obs_date_utc": self.dateObs,
                "reduction_date_utc": utcnow,
                "to_header": True
            }).to_frame().T], ignore_index=True)
            self.qc = pd.concat([self.qc, pd.Series({
                "soxspipe_recipe": self.recipeName,
                "qc_name": "YRESMIN",
                "qc_value": absy.min(),
                "qc_comment": "[px] Minimum residual in dispersion solution fit along y-axis",
                "qc_unit": "pixels",
                "obs_date_utc": self.dateObs,
                "reduction_date_utc": utcnow,
                "to_header": True
            }).to_frame().T], ignore_index=True)
            self.qc = pd.concat([self.qc, pd.Series({
                "soxspipe_recipe": self.recipeName,
                "qc_name": "YRESMAX",
                "qc_value": absy.max(),
                "qc_comment": "[px] Maximum residual in dispersion solution fit along y-axis",
                "qc_unit": "pixels",
                "obs_date_utc": self.dateObs,
                "reduction_date_utc": utcnow,
                "to_header": True
            }).to_frame().T], ignore_index=True)
            self.qc = pd.concat([self.qc, pd.Series({
                "soxspipe_recipe": self.recipeName,
                "qc_name": "YRESRMS",
                "qc_value": absy.std(),
                "qc_comment": "[px] Std-dev of residual in dispersion solution fit along y-axis",
                "qc_unit": "pixels",
                "obs_date_utc": self.dateObs,
                "reduction_date_utc": utcnow,
                "to_header": True
            }).to_frame().T], ignore_index=True)
            self.qc = pd.concat([self.qc, pd.Series({
                "soxspipe_recipe": self.recipeName,
                "qc_name": "XYRESMIN",
                "qc_value": orderPixelTable["residuals_xy"].min(),
                "qc_comment": "[px] Minimum residual in dispersion solution fit (XY combined)",
                "qc_unit": "pixels",
                "obs_date_utc": self.dateObs,
                "reduction_date_utc": utcnow,
                "to_header": True
            }).to_frame().T], ignore_index=True)
            self.qc = pd.concat([self.qc, pd.Series({
                "soxspipe_recipe": self.recipeName,
                "qc_name": "XYRESMAX",
                "qc_value": orderPixelTable["residuals_xy"].max(),
                "qc_comment": "[px] Maximum residual in dispersion solution fit (XY combined)",
                "qc_unit": "pixels",
                "obs_date_utc": self.dateObs,
                "reduction_date_utc": utcnow,
                "to_header": True
            }).to_frame().T], ignore_index=True)
            self.qc = pd.concat([self.qc, pd.Series({
                "soxspipe_recipe": self.recipeName,
                "qc_name": "XYRESRMS",
                "qc_value": orderPixelTable["residuals_xy"].std(),
                "qc_comment": "[px] Std-dev of residual in dispersion solution (XY combined)",
                "qc_unit": "pixels",
                "obs_date_utc": self.dateObs,
                "reduction_date_utc": utcnow,
                "to_header": True
            }).to_frame().T], ignore_index=True)

        self.log.debug('completed the ``calculate_residuals`` method')
        return combined_res_mean, combined_res_std, combined_res_median, orderPixelTable

    def _calculate_resolution_on_slit(
            self,
            row
    ):
        import math
        from astropy.modeling import models, fitting
        import numpy as np
        import matplotlib.pyplot as plt
        import pandas as pd
        stdToFwhm = 2 * (2 * math.log(2))**0.5

        # GOOD GUESS AT STD AND SLICE SIZE
        if self.arm == "NIR":
            guessStd = (float(self.slitWidth) / 0.25) / stdToFwhm
        else:
            guessStd = (float(self.slitWidth) / 0.29) / stdToFwhm
        sliceSize = int(guessStd * 5.)

        if self.detectorParams["dispersion-axis"] == "y":
            detector_row = self.arcFrame.data[int(row['observed_y']), int(row['observed_x']) - sliceSize:int(row['observed_x']) + sliceSize]
            detector_row = detector_row - np.median(detector_row)
            detector_row = np.where(detector_row < 0, 0, detector_row)
        else:
            detector_row = self.arcFrame.data[int(row['observed_y']) - sliceSize:int(row['observed_y']) + sliceSize, int(row['observed_x'])]
            detector_row = detector_row - np.median(detector_row)
            detector_row = np.where(detector_row < 0, 0, detector_row)

        # FITTING THE LINE WITH A GAUSSIAN PROFILE
        g_init = models.Gaussian1D(amplitude=1., mean=0, stddev=guessStd)
        fit_g = fitting.LevMarLSQFitter()

        try:
            g = fit_g(g_init, np.arange(0, len(detector_row)), detector_row)

            if g.stddev.value < guessStd / 2.:
                # 1.0 SLIT CANNOT BE LESS THAN 1.0 px
                raise Exception

            #stddev_corrected = np.sqrt(g.stddev.value*g.stddev.value - np.abs(13*np.sin(row['tilt'])*np.sin(row['tilt'])))
            stddev_corrected = float(g.stddev.value)
            fwhm = stddev_corrected * stdToFwhm

            import random

            random_number = random.randint(1, 5)
            if False and random_number == 2:
                gaussx = np.arange(0, len(detector_row), 0.05)
                plt.plot(np.arange(0, len(detector_row)), detector_row)
                stddev_corrected * stdToFwhm
                plt.plot(gaussx, g(gaussx), label=f'STD = {g.stddev.value:0.2f}, FWHM = {fwhm:0.2f}')
                plt.legend()
                plt.show()
        except Exception as e:
            return pd.Series([None, None])

        #stddev_corrected = np.sqrt(g.stddev.value*g.stddev.value - np.abs(13*np.sin(row['tilt'])*np.sin(row['tilt'])))
        stddev_corrected = float(g.stddev.value)

        # ENFORCING RESONABLE VALUES EXCLUDING OUTLIERS
        if stddev_corrected < guessStd / 2. or stddev_corrected > guessStd * 2.:
            return pd.Series([None, None])

        delta_wavelength = row["pixelScaleNm"] * fwhm
        resolution_line = row["wavelength"] / delta_wavelength

        return pd.Series([resolution_line, fwhm])

    def fit_polynomials(
            self,
            orderPixelTable,
            wavelengthDeg,
            orderDeg,
            slitDeg,
            missingLines=False):
        """*iteratively fit the dispersion map polynomials to the data, clipping residuals with each iteration*

        **Key Arguments:**

        - ``orderPixelTable`` -- data frame containing order, wavelengths, slit positions and observed pixel positions
        - ``wavelengthDeg`` -- degree of wavelength fitting
        - ``orderDeg`` -- degree of the order fitting
        - ``slitDeg`` -- degree of the slit fitting (0 for single pinhole)
        - ``missingLines`` -- lines not detected on the image

        **Return:**

        - ``xcoeff`` -- the x-coefficients post clipping
        - ``ycoeff`` -- the y-coefficients post clipping
        - ``goodLinesTable`` -- the fitted line-list with metrics
        - ``clippedLinesTable`` -- the lines that were sigma-clipped during polynomial fitting
        """
        self.log.debug('starting the ``fit_polynomials`` method')

        # FIRST REMOVE DROPPED LINES FILTERED ROWS FROM DATA FRAME
        allClippedLines = []
        mask = (orderPixelTable['dropped'] == True)
        allClippedLines.append(orderPixelTable.loc[mask])
        orderPixelTable = orderPixelTable.loc[~mask]

        import numpy as np
        from astropy.stats import sigma_clip
        from scipy.optimize import curve_fit
        import pandas as pd
        from soxspipe.commonutils import get_cached_coeffs

        arm = self.arm
        dp = self.detectorParams

        clippedCount = 1

        # ADD EXPONENTS TO ORDERTABLE UP-FRONT
        if isinstance(orderDeg, list):
            orderDegx = orderDeg[0]
            orderDegy = orderDeg[1]
            wavelengthDegx = wavelengthDeg[0]
            wavelengthDegy = wavelengthDeg[1]
            slitDegx = slitDeg[0]
            slitDegy = slitDeg[1]
        else:
            orderDegx = orderDeg
            orderDegy = orderDeg
            wavelengthDegx = wavelengthDeg
            wavelengthDegy = wavelengthDeg
            slitDegx = slitDeg
            slitDegy = slitDeg

        for i in range(0, orderDegx + 1):
            orderPixelTable[f"order_pow_x_{i}"] = orderPixelTable["order"].pow(i)
        for j in range(0, wavelengthDegx + 1):
            orderPixelTable[f"wavelength_pow_x_{j}"] = orderPixelTable["wavelength"].pow(j)
        for k in range(0, slitDegx + 1):
            orderPixelTable[f"slit_position_pow_x_{k}"] = orderPixelTable["slit_position"].pow(k)
        for i in range(0, orderDegy + 1):
            orderPixelTable[f"order_pow_y_{i}"] = orderPixelTable["order"].pow(i)
        for j in range(0, wavelengthDegy + 1):
            orderPixelTable[f"wavelength_pow_y_{j}"] = orderPixelTable["wavelength"].pow(j)
        for k in range(0, slitDegy + 1):
            orderPixelTable[f"slit_position_pow_y_{k}"] = orderPixelTable["slit_position"].pow(k)

        polyx = chebyshev_order_wavelength_polynomials(
            log=self.log, orderDeg=orderDegx, wavelengthDeg=wavelengthDegx, slitDeg=slitDegx, exponentsIncluded=True, axis="x").poly
        polyy = chebyshev_order_wavelength_polynomials(
            log=self.log, orderDeg=orderDegy, wavelengthDeg=wavelengthDegy, slitDeg=slitDegy, exponentsIncluded=True, axis="y").poly

        clippingSigma = self.recipeSettings["poly-fitting-residual-clipping-sigma"]
        clippingSigmaX = clippingSigma
        clippingSigmaY = clippingSigma
        clippingIterationLimit = self.recipeSettings["poly-clipping-iteration-limit"]
        if "poly-clipping-pinhole-sets" in self.recipeSettings and self.recipeSettings["poly-clipping-pinhole-sets"]:
            clipOnMphSets = True
        else:
            clipOnMphSets = False

        self.log.print("\n# FINDING DISPERSION SOLUTION")

        iteration = 0

        # CREATE A QUICK FIRST GUESS AT COEFFS ... SPEEDS UP FIRST ITERATION OF FULL SET
        tmpDF = orderPixelTable.copy()
        mean_res = 100.

        orderPixelTable['sigma_clipped'] = False
        while clippedCount > 0 and iteration < clippingIterationLimit:
            iteration += 1
            observed_x = orderPixelTable["observed_x"].to_numpy()
            observed_y = orderPixelTable["observed_y"].to_numpy()

            # IF mean_res < 10 WE WANT TO START FROM SCRATCH AGAIN SO NOT TO INFLUENCE THE FINAL RESULT
            if True and mean_res > 10:
                # FIND CACHED COEFF ELSE RETURN ARRAYS OF 1s
                xcoeff, ycoeff = get_cached_coeffs(
                    log=self.log,
                    arm=arm,
                    settings=self.settings,
                    recipeName=self.recipeName,
                    orderDeg=orderDeg,
                    wavelengthDeg=wavelengthDeg,
                    slitDeg=slitDeg
                )

            # USE LEAST-SQUARED CURVE FIT TO FIT CHEBY POLYS
            # FIRST X
            self.log.info("""curvefit x""" % locals())

            try:
                xcoeff, pcov_x = curve_fit(
                    polyx, xdata=orderPixelTable, ydata=observed_x, p0=xcoeff, maxfev=30000)
            except:
                return "xerror", None, None, None

            # NOW Y
            self.log.info("""curvefit y""" % locals())

            try:
                ycoeff, pcov_y = curve_fit(
                    polyy, xdata=orderPixelTable, ydata=observed_y, p0=ycoeff, maxfev=30000)
            except:
                return None, "yerror", None, None

            self.log.info("""calculate_residuals""" % locals())
            mean_res, std_res, median_res, orderPixelTable = self.calculate_residuals(
                orderPixelTable=orderPixelTable,
                xcoeff=xcoeff,
                ycoeff=ycoeff,
                orderDeg=orderDeg,
                wavelengthDeg=wavelengthDeg,
                slitDeg=slitDeg,
                pixelRange=True,
                writeQCs=False)

            # DO SOME CLIPPING ON THE PROFILES OF THE DETECTED LINES
            if iteration == -1:
                orderPixelTable = self._clip_on_measured_line_metrics(orderPixelTable)

            # COUNT THE CLIPPED LINES
            mask = (orderPixelTable['sigma_clipped'] == True)
            allClippedLines.append(orderPixelTable.loc[mask])
            mask = (orderPixelTable['sigma_clipped'] == True)
            orderPixelTable = orderPixelTable.loc[~mask]

            # SIGMA-CLIP THE DATA
            self.log.info("""sigma_clip""" % locals())

            if clipOnMphSets:

                columnsNoStrings = list(orderPixelTable.columns)
                try:
                    columnsNoStrings.remove("ion")
                except:
                    pass

                # GROUP BY ARC LINES (MPH SETS)
                lineGroups = orderPixelTable[columnsNoStrings].groupby(['wavelength', 'order']).mean()
                lineGroups = lineGroups.reset_index()

                # SIGMA-CLIP THE DATA ON SCATTER
                masked_residuals = sigma_clip(
                    lineGroups["residuals_x"].abs(), sigma_lower=3000, sigma_upper=clippingSigmaX, maxiters=1, cenfunc='median', stdfunc='mad_std')
                lineGroups["sigma_clipped_x"] = masked_residuals.mask
                masked_residuals = sigma_clip(
                    lineGroups["residuals_y"].abs(), sigma_lower=3000, sigma_upper=clippingSigmaY, maxiters=1, cenfunc='median', stdfunc='mad_std')
                lineGroups["sigma_clipped_y"] = masked_residuals.mask
                lineGroups.loc[((lineGroups["sigma_clipped_y"] == True) | (lineGroups["sigma_clipped_x"] == True)), "sigma_clipped"] = True

                if True:
                    # CLIP ALSO ON COMBINED RESIDUALS
                    masked_residuals = sigma_clip(
                        lineGroups["residuals_xy"], sigma_lower=5000, sigma_upper=clippingSigma, maxiters=1, cenfunc='median', stdfunc='mad_std')
                    lineGroups["sigma_clipped_xy"] = masked_residuals.mask
                    lineGroups.loc[((lineGroups["sigma_clipped_y"] == True) | (lineGroups["sigma_clipped_x"] == True) | (lineGroups["sigma_clipped_xy"] == True)), "sigma_clipped"] = True

                # REMOVE THE CLIPPED DATA BEFORE CLIPPING ON FLUX
                mask = (lineGroups["sigma_clipped"] == True)
                clippedGroups = lineGroups.loc[mask]
                clippedGroups = clippedGroups[['wavelength', 'order']]
                s = orderPixelTable[['wavelength', 'order']].merge(clippedGroups, indicator=True, how='left')
                s["clipped"] = False
                s.loc[(s["_merge"] == "both"), "clipped"] = True

                orderPixelTable["sigma_clipped"] = s["clipped"].values

                # CLIP THE MOST DEVIATE SINGLE PINHOLES
                if iteration == 1 and True:
                    masked_residuals = sigma_clip(
                        orderPixelTable["residuals_x"].abs(), sigma_lower=3000, sigma_upper=clippingSigmaX * 3, maxiters=1, cenfunc='median', stdfunc='mad_std')
                    orderPixelTable["sigma_clipped_x"] = masked_residuals.mask
                    masked_residuals = sigma_clip(
                        orderPixelTable["residuals_y"].abs(), sigma_lower=3000, sigma_upper=clippingSigmaY * 3, maxiters=1, cenfunc='median', stdfunc='mad_std')
                    orderPixelTable["sigma_clipped_y"] = masked_residuals.mask
                    masked_residuals = sigma_clip(
                        orderPixelTable["R_pin"], sigma_lower=3000, sigma_upper=10, maxiters=1, cenfunc='median', stdfunc='mad_std')
                    orderPixelTable["sigma_clipped_R"] = masked_residuals.mask
                    orderPixelTable.loc[((orderPixelTable["sigma_clipped_y"] == True) | (orderPixelTable["sigma_clipped_x"] == True) | (orderPixelTable["sigma_clipped_R"] == True)), "sigma_clipped"] = True

            else:
                masked_residuals = sigma_clip(
                    orderPixelTable["residuals_x"].abs(), sigma_lower=3000, sigma_upper=clippingSigmaX, maxiters=1, cenfunc='median', stdfunc='mad_std')
                orderPixelTable["sigma_clipped_x"] = masked_residuals.mask
                masked_residuals = sigma_clip(
                    orderPixelTable["residuals_y"].abs(), sigma_lower=3000, sigma_upper=clippingSigmaY, maxiters=1, cenfunc='median', stdfunc='mad_std')
                orderPixelTable["sigma_clipped_y"] = masked_residuals.mask
                orderPixelTable.loc[((orderPixelTable["sigma_clipped_y"] == True) | (orderPixelTable["sigma_clipped_x"] == True)), "sigma_clipped"] = True

                if False:
                    # CLIP ALSO ON COMBINED RESIDUALS
                    masked_residuals = sigma_clip(
                        orderPixelTable["residuals_xy"], sigma_lower=clippingSigma, sigma_upper=clippingSigma, maxiters=1, cenfunc='median', stdfunc='mad_std')
                    orderPixelTable["sigma_clipped_xy"] = masked_residuals.mask
                    orderPixelTable.loc[((orderPixelTable["sigma_clipped_y"] == True) | (orderPixelTable["sigma_clipped_x"] == True) | (orderPixelTable["sigma_clipped_xy"] == True)), "sigma_clipped"] = True

            # COUNT THE CLIPPED LINES
            mask = (orderPixelTable['sigma_clipped'] == True)
            allClippedLines.append(orderPixelTable.loc[mask])
            totalAllClippedLines = pd.concat(allClippedLines, ignore_index=True)

            # RETURN BREAKDOWN OF COLUMN VALUE COUNT
            valCounts = orderPixelTable[
                'sigma_clipped'].value_counts(normalize=False)
            if True in valCounts:
                clippedCount = valCounts[True]
            else:
                clippedCount = 0

            if iteration > 1:
                # Cursor up one line and clear line
                sys.stdout.flush()
                sys.stdout.write("\x1b[1A\x1b[2K")

            try:
                self.log.print(f'\tITERATION {iteration:02d}: {clippedCount} arc lines where clipped in this iteration of fitting a global dispersion map')
            except:
                pass

            mask = (orderPixelTable['sigma_clipped'] == True)
            orderPixelTable = orderPixelTable.loc[~mask]

        if len(allClippedLines):
            allClippedLines = pd.concat(allClippedLines, ignore_index=True)

        mean_res, std_res, median_res, orderPixelTable = self.calculate_residuals(
            orderPixelTable=orderPixelTable,
            xcoeff=xcoeff,
            ycoeff=ycoeff,
            orderDeg=orderDeg,
            wavelengthDeg=wavelengthDeg,
            slitDeg=slitDeg,
            writeQCs=False,
            pixelRange=True)

        self.log.debug('completed the ``fit_polynomials`` method')
        return xcoeff, ycoeff, orderPixelTable, allClippedLines

    def create_placeholder_images(
            self,
            order=False,
            plot=False,
            reverse=False):
        """*create CCDData objects as placeholders to host the 2D images of the wavelength and spatial solutions from dispersion solution map*

        **Key Arguments:**

        - ``order`` -- specific order to generate the placeholder pixels for. Inner-order pixels set to nan, else set to 0. Default *False* (generate all orders)
        - ``plot`` -- generate plots of placeholder images (for debugging). Default *False*.
        - ``reverse`` -- Inner-order pixels set to 0, else set to nan (reverse of default output).

        **Return:**

        - ``slitMap`` -- placeholder image to add pixel slit positions to
        - ``wlMap`` -- placeholder image to add pixel wavelength values to

        **Usage:**

        ```python
        slitMap, wlMap, orderMap = self._create_placeholder_images(order=order)
        ```
        """
        self.log.debug('starting the ``create_placeholder_images`` method')

        import numpy as np
        from astropy.nddata import CCDData

        kw = self.kw
        dp = self.detectorParams

        # UNPACK THE ORDER TABLE
        orderPolyTable, orderPixelTable, orderMetaTable = unpack_order_table(
            log=self.log, orderTablePath=self.orderTable, extend=0., order=order)

        # CREATE THE IMAGE SAME SIZE AS DETECTOR - NAN INSIDE ORDERS, 0 OUTSIDE
        science_pixels = dp["science-pixels"]
        xlen = science_pixels["columns"]["end"] - \
            science_pixels["columns"]["start"]
        ylen = science_pixels["rows"]["end"] - science_pixels["rows"]["start"]
        xlen, ylen

        if reverse:
            seedArray = np.empty((ylen, xlen))
            seedArray[:] = np.nan
        else:
            seedArray = np.zeros((ylen, xlen))
        wlMap = CCDData(seedArray, unit="adu")
        orderMap = wlMap.copy()
        uniqueOrders = orderPixelTable['order'].unique()
        expandEdges = 0

        if self.detectorParams["dispersion-axis"] == "x":
            axisALen = wlMap.data.shape[1]
            axisBLen = wlMap.data.shape[0]
        else:
            axisALen = wlMap.data.shape[0]
            axisBLen = wlMap.data.shape[1]

        for o in uniqueOrders:
            if order and o != order:
                continue
            axisBcoord = orderPixelTable.loc[
                (orderPixelTable["order"] == o)][f"{self.axisB}coord"]
            axisACoord_edgeup = orderPixelTable.loc[(orderPixelTable["order"] == o)][
                f"{self.axisA}coord_edgeup"] + expandEdges
            axisACoord_edgelow = orderPixelTable.loc[(orderPixelTable["order"] == o)][
                f"{self.axisA}coord_edgelow"] - expandEdges
            axisACoord_edgeup[axisACoord_edgeup > axisALen] = axisALen
            axisACoord_edgeup[axisACoord_edgeup < 0] = 0
            axisACoord_edgelow[axisACoord_edgelow > axisALen] = axisALen
            axisACoord_edgelow[axisACoord_edgelow < 0] = 0
            axisACoord_edgelow, axisACoord_edgeup, axisBcoord = zip(*[(l, u, b) for l, u, b in zip(axisACoord_edgelow, axisACoord_edgeup, axisBcoord) if l >= 0 and l <= axisALen and u >= 0 and u <= axisALen and b >= 0 and b < axisBLen])
            if reverse:
                for b, u, l in zip(axisBcoord, np.ceil(axisACoord_edgeup).astype(int), np.floor(axisACoord_edgelow).astype(int)):
                    if self.axisA == "x":
                        wlMap.data[b, l: u] = 0
                        orderMap.data[b, l: u] = o
                    else:
                        wlMap.data[l: u, b] = 0
                        orderMap.data[l: u, b] = o
            else:
                for b, u, l in zip(axisBcoord, np.ceil(axisACoord_edgeup).astype(int), np.floor(axisACoord_edgelow).astype(int)):
                    if self.axisA == "x":
                        wlMap.data[b, l: u] = np.nan
                        orderMap.data[b, l: u] = np.nan
                    else:
                        wlMap.data[l: u, b] = np.nan
                        orderMap.data[l: u, b] = np.nan

        # SLIT MAP PLACEHOLDER SAME AS WAVELENGTH MAP PLACEHOLDER
        slitMap = wlMap.copy()

        # PLOT CCDDATA OBJECT
        if False:
            import matplotlib.pyplot as plt
            rotatedImg = np.rot90(slitMap.data, 1)
            rotatedImg = slitMap.data
            std = np.nanstd(slitMap.data)
            mean = np.nanmean(slitMap.data)
            vmax = mean + 3 * std
            vmin = mean - 3 * std
            plt.figure(figsize=(12, 5))
            plt.imshow(rotatedImg, vmin=vmin, vmax=vmax,
                       cmap='gray', alpha=1)
            plt.colorbar()
            plt.xlabel(
                "y-axis", fontsize=10)
            plt.ylabel(
                "x-axis", fontsize=10)
            plt.show()
            plt.close("all")

        self.log.debug('completed the ``create_placeholder_images`` method')
        return slitMap, wlMap, orderMap

    def map_to_image(
            self,
            dispersionMapPath,
            orders=False):
        """*convert the dispersion map to images in the detector format showing pixel wavelength values and slit positions*

        **Key Arguments:**

        - ``dispersionMapPath`` -- path to the full dispersion map to convert to images
        - ``orders`` -- orders to add to the image map. List. Default *False* (add all orders)

        **Return:**

        - ``dispersion_image_filePath`` -- path to the FITS image with an extension for wavelength values and another for slit positions

        **Usage:**

        ```python
        mapImagePath = self.map_to_image(dispersionMapPath=mapPath)
        ```
        """
        self.log.debug('starting the ``map_to_image`` method')

        from soxspipe.commonutils.combiner import Combiner
        import numpy as np
        from astropy.io import fits
        import copy

        self.log.print("\n# CREATING 2D IMAGE MAP FROM DISPERSION SOLUTION\n\n")

        self.dispersionMapPath = dispersionMapPath
        kw = self.kw
        dp = self.detectorParams
        arm = self.arm

        self.map_to_image_displacement_threshold = self.recipeSettings["map_to_image_displacement_threshold"]
        # READ THE SPECTRAL FORMAT TABLE TO DETERMINE THE LIMITS OF THE TRACES
        orderNums, waveLengthMin, waveLengthMax = read_spectral_format(
            log=self.log, settings=self.settings, arm=self.arm)

        combinedSlitImage = False
        combinedWlImage = False

        if orders:
            theseOrders = orders
        else:
            theseOrders = orderNums

        # DEFINE AN INPUT ARRAY
        inputArray = [(order, minWl, maxWl) for order, minWl,
                      maxWl in zip(orderNums, waveLengthMin, waveLengthMax) if order in theseOrders]

        # NUMPY CAN BE TRICKY WITH MP
        numThreads = '1'
        os.environ['OPENBLAS_NUM_THREADS'] = numThreads
        os.environ['OMP_NUM_THREADS'] = numThreads
        os.environ['BLAS_NUM_THREADS'] = numThreads

        results = fmultiprocess(log=self.log, function=self.order_to_image,
                                inputArray=inputArray, poolSize=6, timeout=3600, turnOffMP=False)
        del os.environ['OPENBLAS_NUM_THREADS']
        del os.environ['OMP_NUM_THREADS']
        del os.environ['BLAS_NUM_THREADS']

        slitImages = [r[0] for r in results]
        wlImages = [r[1] for r in results]

        slitMap, wlMap, orderMap = self.create_placeholder_images(reverse=True)

        combinedSlitImage = Combiner(slitImages)
        combinedSlitImage = combinedSlitImage.sum_combine()
        combinedWlImage = Combiner(wlImages)
        combinedWlImage = combinedWlImage.sum_combine()

        combinedWlImage.data += wlMap.data
        combinedSlitImage.data += wlMap.data

        # DETERMINE WHERE TO WRITE THE FILE
        home = expanduser("~")
        outDir = self.settings["workspace-root-dir"].replace("~", home) + f"/reduced/{self.startNightDate}/{self.recipeName}"
        outDir = outDir.replace("//", "/")
        # RECURSIVELY CREATE MISSING DIRECTORIES
        if not os.path.exists(outDir):
            os.makedirs(outDir)

        # GET THE EXTENSION (WITH DOT PREFIX)
        extension = os.path.splitext(dispersionMapPath)[1]
        filename = os.path.basename(
            dispersionMapPath).replace(extension, "_IMAGE.fits")

        dispersion_image_filePath = f"{outDir}/{filename}"
        # WRITE CCDDATA OBJECT TO FILE
        # from astropy.io import fits
        header = copy.deepcopy(self.dispMapHeader)
        header[kw("PRO_CATG")] = f"DISP_IMAGE_{arm}".upper()
        primary_hdu = fits.PrimaryHDU(combinedWlImage.data, header=header)
        primary_hdu.header['EXTNAME'] = 'WAVELENGTH'
        image_hdu = fits.ImageHDU(combinedSlitImage.data)
        image_hdu.header['EXTNAME'] = 'SLIT'
        image_hdu2 = fits.ImageHDU(orderMap.data)
        image_hdu2.header['EXTNAME'] = 'ORDER'
        hdul = fits.HDUList([primary_hdu, image_hdu, image_hdu2])
        hdul.writeto(dispersion_image_filePath, output_verify='exception',
                     overwrite=True, checksum=True)

        self.log.debug('completed the ``map_to_image`` method')
        return dispersion_image_filePath

    def order_to_image(
            self,
            orderInfo):
        """*convert a single order in the dispersion map to wavelength and slit position images*

        **Key Arguments:**

        - ``orderInfo`` -- tuple containing the order number to generate the images for, the minimum wavelength to consider (from format table) and maximum wavelength to consider (from format table).

        **Return:**

        - ``slitMap`` -- the slit map with order values filled
        - ``wlMap`` -- the wavelengths map with order values filled

        **Usage:**

        ```python
        slitMap, wlMap = self.order_to_image(order=order,minWl=minWl, maxWl=maxWl)
        ```
        """
        self.log.debug('starting the ``order_to_image`` method')

        import numpy as np
        from scipy.interpolate import griddata

        (order, minWl, maxWl) = orderInfo
        slitMap, wlMap, orderMap = self.create_placeholder_images(order=order)

        if np.count_nonzero(wlMap.data) == 0:
            return slitMap, wlMap

        # FIRST GENERATE A WAVELENGTH SURFACE - FINE WL, CHUNKY SLIT-POSITION
        wlRange = maxWl - minWl
        if self.arm.lower == "nir":
            grid_res_wavelength = wlRange / 200
        else:
            grid_res_wavelength = wlRange / 200
        slitLength = self.detectorParams["slit_length"]
        grid_res_slit = slitLength / 20

        halfGrid = (slitLength / 2) * 1.2
        slitArray = np.arange(-halfGrid, halfGrid +
                              grid_res_slit, grid_res_slit)
        wlArray = np.arange(minWl, maxWl, grid_res_wavelength)

        # ONE SINGLE-VALUE SLIT ARRAY FOR EVERY WAVELENGTH ARRAY
        bigSlitArray = np.concatenate(
            [np.ones(wlArray.shape[0]) * slitArray[i] for i in range(0, slitArray.shape[0])])
        # NOW THE BIG WAVELEGTH ARRAY
        bigWlArray = np.tile(wlArray, np.shape(slitArray)[0])

        iteration = 0
        remainingPixels = 1
        iterationLimit = 20
        remainingCount = 1

        # GET A COMPLETE LIST OF THE PIXEL WE NEED
        nan_indexes = np.argwhere(np.isnan(slitMap.data))
        ally, allx = nan_indexes[:, 0], nan_indexes[:, 1]

        while remainingPixels and remainingCount and iteration < iterationLimit:
            iteration += 1

            # GENERATE THE ORDER PIXEL TABLE FROM WL AND SLIT-POSITION GRID .. IF WITHIN THRESHOLD OF CENTRE OF DETECTOR PIXEL THEN INJECT INTO MAPS
            orderPixelTable, remainingCount = self.convert_and_fit(
                order=order, bigWlArray=bigWlArray, bigSlitArray=bigSlitArray, slitMap=slitMap, wlMap=wlMap, iteration=iteration, plots=False)

            if remainingCount < 3:
                break

            orderPixelTable = orderPixelTable.drop_duplicates(subset=['pixel_x', 'pixel_y', 'order'])
            train_wlx = orderPixelTable["fit_x"].values
            train_wly = orderPixelTable["fit_y"].values
            train_wl = orderPixelTable["wavelength"].values
            train_sp = orderPixelTable["slit_position"].values

            targetX = orderPixelTable['pixel_x'].values
            targetY = orderPixelTable['pixel_y'].values

            if iteration == 1:
                # ADD MISSING PIXELS
                targetX = np.concatenate([targetX, allx])
                targetY = np.concatenate([targetY, ally])

            # USE CUBIC SPLINE NEAREST NEIGHBOUR TO SEED RESULTS
            bigWlArray = griddata((train_wlx, train_wly), train_wl, (targetX, targetY), method="cubic")
            bigSlitArray = griddata((train_wlx, train_wly), train_sp, (targetX, targetY), method="cubic")

        self.log.debug('completed the ``order_to_image`` method')
        return slitMap, wlMap

    def convert_and_fit(
            self,
            order,
            bigWlArray,
            bigSlitArray,
            slitMap,
            wlMap,
            iteration,
            plots=False):
        """*convert wavelength and slit position grids to pixels*

        **Key Arguments:**

        - ``order`` -- the order being considered
        - ``bigWlArray`` -- 1D array of all wavelengths to be converted
        - ``bigSlitArray`` -- 1D array of all split-positions to be converted (same length as `bigWlArray`)
        - ``slitMap`` -- place-holder image hosting fitted pixel slit-position values
        - ``wlMap`` -- place-holder image hosting fitted pixel wavelength values
        - ``iteration`` -- the iteration index (used for CL reporting)
        - ``plots`` -- show plot of the slit-map

        **Return:**

        - ``orderPixelTable`` -- dataframe containing unfitted pixel info
        - ``remainingCount`` -- number of remaining pixels in orderTable

        **Usage:**

        ```python
        orderPixelTable = self.convert_and_fit(
                order=order, bigWlArray=bigWlArray, bigSlitArray=bigSlitArray, slitMap=slitMap, wlMap=wlMap)
        ```
        """
        self.log.debug('starting the ``convert_and_fit`` method')

        import pandas as pd
        import numpy as np

        # CREATE PANDAS DATAFRAME WITH LARGE ARRAYS - ONE ROW PER
        # WAVELENGTH-SLIT GRID CELL
        myDict = {
            "order": np.ones(bigWlArray.shape[0]) * order,
            "wavelength": bigWlArray,
            "slit_position": bigSlitArray
        }
        orderPixelTable = pd.DataFrame(myDict)

        # GET DETECTOR PIXEL POSITIONS FOR ALL WAVELENGTH-SLIT GRID CELLS
        orderPixelTable = dispersion_map_to_pixel_arrays(
            log=self.log,
            dispersionMapPath=self.dispersionMapPath,
            orderPixelTable=orderPixelTable
        )

        # INTEGER PIXEL VALUES & FIT DISPLACEMENTS FROM PIXEL CENTRES
        orderPixelTable["pixel_x"] = np.round(orderPixelTable["fit_x"].values)
        orderPixelTable["pixel_y"] = np.round(orderPixelTable["fit_y"].values)
        orderPixelTable["residual_x"] = orderPixelTable[
            "fit_x"] - orderPixelTable["pixel_x"]
        orderPixelTable["residual_y"] = orderPixelTable[
            "fit_y"] - orderPixelTable["pixel_y"]
        orderPixelTable["residual_xy"] = np.sqrt(np.square(
            orderPixelTable["residual_x"]) + np.square(orderPixelTable["residual_y"]))

        # ADD A COUNT COLUMN FOR THE NUMBER OF SMALL SLIT/WL PIXELS FALLING IN
        # LARGE DETECTOR PIXELS
        count = orderPixelTable.groupby(
            ['pixel_x', 'pixel_y']).size().reset_index(name='count')
        orderPixelTable = pd.merge(orderPixelTable, count, how='left', left_on=[
            'pixel_x', 'pixel_y'], right_on=['pixel_x', 'pixel_y'])
        orderPixelTable = orderPixelTable.sort_values(
            ['order', 'pixel_x', 'pixel_y', 'residual_xy'])

        # FILTER TO WL/SLIT POSITION CLOSE ENOUGH TO CENTRE OF PIXEL
        mask = (orderPixelTable['residual_xy'] <
                self.map_to_image_displacement_threshold)

        # KEEP ONLY VALUES CLOSEST TO CENTRE OF PIXEL
        newPixelValue = orderPixelTable.loc[mask].drop_duplicates(
            subset=['pixel_x', 'pixel_y'], keep="first")

        # REMOVE PIXELS FOUND IN newPixelValue FROM orderPixelTable
        orderPixelTable = newPixelValue[['pixel_x', 'pixel_y']].merge(orderPixelTable, on=[
            'pixel_x', 'pixel_y'], how='right', indicator=True).query('_merge == "right_only"').drop(columns=['_merge'])

        remainingCount = orderPixelTable.drop_duplicates(
            subset=['pixel_x', 'pixel_y'], keep="first")

        # ADD FITTED PIXELS TO PLACE HOLDER IMAGES
        for xx, yy, wavelength, slit_position in zip(newPixelValue["pixel_x"].values.astype(int), newPixelValue["pixel_y"].values.astype(int), newPixelValue["wavelength"].values, newPixelValue["slit_position"].values):
            try:
                wlMap.data[yy, xx] = np.where(
                    np.isnan(wlMap.data[yy, xx]), wavelength, wlMap.data[yy, xx])
                slitMap.data[yy, xx] = np.where(
                    np.isnan(slitMap.data[yy, xx]), slit_position, slitMap.data[yy, xx])
            except (IndexError):
                # PIXELS OUTSIDE OF DETECTOR EDGES - IGNORE
                pass

        sys.stdout.flush()
        sys.stdout.write("\x1b[1A\x1b[2K")
        percentageFound = (1 - (np.count_nonzero(np.isnan(wlMap.data)) / np.count_nonzero(wlMap.data))) * 100
        try:
            self.log.print(f"ORDER {order:02d}, iteration {iteration:02d}. {percentageFound:0.2f}% order pixels now fitted.")
        except:
            pass
        # print(f"ORDER {order:02d}, iteration {iteration:02d}. {percentageFound:0.2f}% order pixels now fitted. Fit found for {len(newPixelValue.index)} new pixels, {len(remainingCount.index)} image pixel remain to be constrained ({np.count_nonzero(np.isnan(wlMap.data))} nans in place-holder image)")

        if plots:
            from matplotlib import cm
            import matplotlib.pyplot as plt
            # PLOT CCDDATA OBJECT
            rotatedImg = slitMap.data
            if self.axisA == "x":
                rotatedImg = np.rot90(rotatedImg, 1)
                rotatedImg = np.flipud(rotatedImg)
            std = np.nanstd(rotatedImg)
            mean = np.nanmean(rotatedImg)
            cmap = cm.gray
            cmap.set_bad(color='#ADD8E6')
            vmax = np.nanmax(rotatedImg)
            vmin = np.nanmin(rotatedImg)
            plt.figure(figsize=(24, 10))
            plt.imshow(rotatedImg, vmin=vmin, vmax=vmax,
                       cmap=cmap, alpha=1)
            if self.axisA == "x":
                plt.gca().invert_yaxis()
            plt.colorbar()
            plt.ylabel(f"{self.axisA}-axis", fontsize=12)
            plt.xlabel(f"{self.axisB}-axis", fontsize=12)
            plt.show()
            plt.close("all")

        remainingCount = len(remainingCount.index)

        self.log.debug('completed the ``convert_and_fit`` method')
        return orderPixelTable, remainingCount

    def _create_dispersion_map_qc_plot(
            self,
            xcoeff,
            ycoeff,
            orderDeg,
            wavelengthDeg,
            slitDeg,
            orderPixelTable,
            missingLines,
            allClippedLines,
            dispMap=False,
            dispMapImage=False):
        """*create the QC plot for the dispersion map solution*

        **Key Arguments:**

        - ``xcoeff`` -- the x-coefficients
        - ``ycoeff`` -- the y-coefficients
        - ``orderDeg`` -- degree of the order fitting
        - ``wavelengthDeg`` -- degree of wavelength fitting
        - ``slitDeg`` -- degree of the slit fitting (False for single pinhole)
        - ``orderPixelTable`` -- a panda's data-frame containing wavelength,order,slit_index,slit_position,detector_x,detector_y
        - ``missingLines`` -- lines not detected on the image
        - `allClippedLines` -- lines clipped during dispersion solution fitting
        - `dispMap` -- path to dispersion map. Default *False*
        - `dispMapImage` -- the 2D dispersion map image

        **Return:**

        - ``res_plots`` -- path the the output QC plot
        ```
        """
        self.log.debug('starting the ``create_dispersion_map_qc_plot`` method')

        import numpy as np
        from astropy.visualization import hist
        import matplotlib.pyplot as plt
        import pandas as pd
        from soxspipe.commonutils.toolkit import qc_settings_plot_tables
        from astropy.stats import sigma_clipped_stats

        arm = self.arm
        kw = self.kw
        dp = self.detectorParams

        rotateImage = dp["rotate-qc-plot"]
        flipImage = dp["flip-qc-plot"]

        gridLinePixelTable = False

        # CREATE THE GRID LINES FOR THE FULL SPATIAL SOLUTION PLOTS (MPH)
        if not isinstance(dispMapImage, bool):
            from soxspipe.commonutils.toolkit import create_dispersion_solution_grid_lines_for_plot
            gridLinePixelTable, interOrderMask = create_dispersion_solution_grid_lines_for_plot(
                log=self.log,
                dispMap=dispMap,
                dispMapImage=dispMapImage,
                associatedFrame=self.pinholeFrame,
                kw=kw,
                skylines=False,
                slitPositions=self.uniqueSlitPos
            )
        # DROP MISSING VALUES
        orderPixelTable.dropna(axis='index', how='any', subset=[
            'residuals_x'], inplace=True)
        orderPixelTable["residuals_xy"] = np.sqrt(np.square(
            orderPixelTable["residuals_x"]) + np.square(orderPixelTable["residuals_y"]))
        mean_res = np.mean(orderPixelTable["residuals_xy"])
        std_res = np.std(orderPixelTable["residuals_xy"])
        median_res = np.median(orderPixelTable["residuals_xy"])

        # mean_x_res = abs(orderPixelTable["residuals_x"]).mean()
        # mean_y_res = abs(orderPixelTable["residuals_y"]).mean()

        mean_x_res = abs(orderPixelTable["residuals_x"]).mean()
        mean_y_res = abs(orderPixelTable["residuals_y"]).mean()
        median_x_res = np.median(abs(orderPixelTable["residuals_x"]))
        median_y_res = np.median(abs(orderPixelTable["residuals_y"]))

        # ROTATE THE IMAGE FOR BETTER LAYOUT
        rotatedImg = np.ma.array(self.pinholeFrameMasked.data, mask=self.pinholeFrameMasked.mask, fill_value=0).filled()
        if rotateImage:
            rotatedImg = np.rot90(rotatedImg, rotateImage / 90)
        if flipImage:
            rotatedImg = np.flipud(rotatedImg)
            if not rotateImage:
                aLen = rotatedImg.shape[0]
                orderPixelTable[f"observed_{self.axisA}"] = aLen - orderPixelTable[f"observed_{self.axisA}"]
                missingLines[f"detector_{self.axisA}_shifted"] = aLen - missingLines[f"detector_{self.axisA}_shifted"]
                allClippedLines[f"detector_{self.axisA}_shifted"] = aLen - allClippedLines[f"detector_{self.axisA}_shifted"]
                orderPixelTable[f"detector_{self.axisA}_shifted"] = aLen - orderPixelTable[f"detector_{self.axisA}_shifted"]
                allClippedLines[f"detector_{self.axisA}"] = aLen - allClippedLines[f"detector_{self.axisA}"]
                missingLines[f"detector_{self.axisA}"] = aLen - missingLines[f"detector_{self.axisA}"]
                orderPixelTable[f"fit_{self.axisA}"] = aLen - orderPixelTable[f"fit_{self.axisA}"]
                allClippedLines[f"observed_{self.axisA}"] = aLen - allClippedLines[f"observed_{self.axisA}"]
                if not isinstance(gridLinePixelTable, bool):
                    gridLinePixelTable[f"fit_{self.axisA}"] = aLen - gridLinePixelTable[f"fit_{self.axisA}"]
                # orderPixelTable[f"observed_{self.axisA}"] = aLen - orderPixelTable[f"observed_{self.axisA}"]
                # orderPixelTable[f"observed_{self.axisA}"] = aLen - orderPixelTable[f"observed_{self.axisA}"]

        # a = plt.figure(figsize=(40, 15))

        if rotatedImg.shape[0] / rotatedImg.shape[1] > 0.8:
            fig = plt.figure(figsize=(6, 20), constrained_layout=True)
            # CREATE THE GRID OF AXES
            if self.arcFrame:
                gs = fig.add_gridspec(12, 4)
                sizeAx = fig.add_subplot(gs[8:9, :])
                gapAx = fig.add_subplot(gs[9:10, :])
                settingsAx = fig.add_subplot(gs[10:, 2:])
                qcAx = fig.add_subplot(gs[11:, 0:2])
            else:
                gs = fig.add_gridspec(10, 4)
                settingsAx = fig.add_subplot(gs[8:, 2:])
                qcAx = fig.add_subplot(gs[8:, 0:2])

            toprow = fig.add_subplot(gs[0:2, :])
            midrow = fig.add_subplot(gs[2:4, :])
            bottomleft = fig.add_subplot(gs[4:6, 0:2])
            bottomright = fig.add_subplot(gs[4:6, 2:])
            fwhmAx = fig.add_subplot(gs[6:7, :])
            resAx = fig.add_subplot(gs[7:8, :])

        else:
            fig = plt.figure(figsize=(6, 20), constrained_layout=True)
            # CREATE THE GRID OF AXES
            # CREATE THE GRID OF AXES
            if self.arcFrame:
                gs = fig.add_gridspec(12, 4)
                sizeAx = fig.add_subplot(gs[8:9, :])
                gapAx = fig.add_subplot(gs[9:10, :])
                settingsAx = fig.add_subplot(gs[10:, 2:])
                qcAx = fig.add_subplot(gs[11:, 0:2])
            else:
                gs = fig.add_gridspec(10, 4)
                settingsAx = fig.add_subplot(gs[8:, 2:])
                qcAx = fig.add_subplot(gs[8:, 0:2])

            toprow = fig.add_subplot(gs[0:2, :])
            midrow = fig.add_subplot(gs[2:4, :])
            bottomleft = fig.add_subplot(gs[4:6, 0:2])
            bottomright = fig.add_subplot(gs[4:6, 2:])
            fwhmAx = fig.add_subplot(gs[6:7, :])
            resAx = fig.add_subplot(gs[7:8, :])

        std = self.std
        mean = self.mean

        vmax = mean + 25 * std
        vmin = mean
        toprow.imshow(rotatedImg, vmin=vmin, vmax=vmax, cmap='gray', alpha=0.5)
        toprow.set_title(
            "observed arc-line positions (post-clipping)", fontsize=10)

        alphaBoost = 1.0
        if not self.firstGuessMap:
            alphaBoost = 1.7

        if False:
            # PLOT WHERE LINES GET SHIFTED TO
            toprow.scatter(missingLines[f"detector_{self.axisB}_shifted"], missingLines[f"detector_{self.axisA}_shifted"], marker='o', c='black', s=20, alpha=0.1 * alphaBoost, linewidths=0.5, label="missed lines")
            toprow.scatter(allClippedLines[f"detector_{self.axisB}_shifted"], allClippedLines[f"detector_{self.axisA}_shifted"], marker='o', c='black', s=20, alpha=0.1 * alphaBoost, linewidths=0.5, label="missed lines")
            toprow.scatter(orderPixelTable[f"detector_{self.axisB}_shifted"], orderPixelTable[f"detector_{self.axisA}_shifted"], marker='o', c='black', s=20, alpha=0.1 * alphaBoost, linewidths=0.5, label="missed lines")

        else:

            if isinstance(missingLines, pd.core.frame.DataFrame):
                toprow.scatter(missingLines[f"detector_{self.axisB}"], missingLines[f"detector_{self.axisA}"], marker='o', c='black', s=20, alpha=0.1 * alphaBoost, linewidths=0.5, label="undetected line location")
            if len(allClippedLines.index):
                mask = (allClippedLines['dropped'] == True)
                if self.firstGuessMap:
                    toprow.scatter(allClippedLines.loc[mask][f"observed_{self.axisB}"], allClippedLines.loc[mask][f"detector_{self.axisA}"], marker='o', c='blue', s=5, alpha=0.3 * alphaBoost, linewidths=0.5, label="dropped multi-pinhole set")
                else:
                    toprow.scatter(allClippedLines.loc[mask][f"observed_{self.axisB}"], allClippedLines.loc[mask][f"detector_{self.axisA}"], marker='o', c='blue', s=5, alpha=0.3 * alphaBoost, linewidths=0.5, label="dropped pinhole")
                toprow.scatter(allClippedLines.loc[~mask][f"observed_{self.axisB}"], allClippedLines.loc[~mask][f"observed_{self.axisA}"], marker='o', c='green', s=5, alpha=0.3 * alphaBoost, linewidths=0.5 * alphaBoost, )
                toprow.scatter(allClippedLines.loc[~mask][f"observed_{self.axisB}"], allClippedLines.loc[~mask][f"observed_{self.axisA}"], marker='x', c='red', s=5, alpha=0.3 * alphaBoost, linewidths=0.5, label="clipped during dispersion solution fitting")
            if len(orderPixelTable.index):
                toprow.scatter(orderPixelTable[f"observed_{self.axisB}"], orderPixelTable[f"observed_{self.axisA}"], marker='o', c='green', s=5, alpha=0.3 * alphaBoost, linewidths=0.5 * alphaBoost, label="detected line location")

        toprow.set_ylabel(f"{self.axisA}-axis", fontsize=12)
        toprow.set_xlabel(f"{self.axisB}-axis", fontsize=12)
        toprow.tick_params(axis='both', which='major', labelsize=9)
        toprow.legend(loc='upper right', bbox_to_anchor=(1.0, -0.15), fontsize=4)

        toprow.set_xlim([0, rotatedImg.shape[1]])
        if self.axisA == "x":
            toprow.invert_yaxis()
        toprow.set_ylim([0, rotatedImg.shape[0]])

        midrow.imshow(rotatedImg, vmin=vmin, vmax=vmax, cmap='gray', alpha=0.5)
        midrow.set_title(
            "global dispersion solution", fontsize=10)

        # ADD FULL DISPERSION SOLUTION GRID-LINES TO PLOT
        if not isinstance(gridLinePixelTable, bool):
            for l in range(int(gridLinePixelTable['line'].max())):
                mask = (gridLinePixelTable['line'] == l)
                if l == 1:
                    midrow.plot(gridLinePixelTable.loc[mask][f"fit_{self.axisB}"], gridLinePixelTable.loc[mask][f"fit_{self.axisA}"], "w-", linewidth=0.2, alpha=0.5 * alphaBoost, color="blue", label="dispersion solution")
                else:
                    midrow.plot(gridLinePixelTable.loc[mask][f"fit_{self.axisB}"], gridLinePixelTable.loc[mask][f"fit_{self.axisA}"], "w-", linewidth=0.2, alpha=0.5 * alphaBoost, color="blue")
        else:
            midrow.scatter(orderPixelTable[f"fit_{self.axisB}"],
                           orderPixelTable[f"fit_{self.axisA}"], marker='o', c='blue', s=orderPixelTable[f"residuals_xy"] * 30, alpha=0.1 * alphaBoost, label="fitted line (size proportional to line-fit residual)")

        midrow.set_ylabel(f"{self.axisA}-axis", fontsize=12)
        midrow.set_xlabel(f"{self.axisB}-axis", fontsize=12)
        midrow.tick_params(axis='both', which='major', labelsize=9)
        midrow.set_xlim([0, rotatedImg.shape[1]])
        if self.axisA == "x":
            midrow.invert_yaxis()
        midrow.set_ylim([0, rotatedImg.shape[0]])

        midrow.legend(loc='upper right', bbox_to_anchor=(1.0, -0.15), fontsize=4)

        # PLOT THE RESIDUALS
        plt.subplots_adjust(top=0.92)
        bottomleft.scatter(orderPixelTable[f"residuals_{self.axisA}"], orderPixelTable[
            f"residuals_{self.axisB}"], alpha=0.1)
        bottomleft.set_xlabel(f'{self.axisA} residual (mean: {mean_x_res:2.2f} pix)')
        bottomleft.set_ylabel(f'{self.axisB} residual (mean: {mean_y_res:2.2f} pix)')
        bottomleft.tick_params(axis='both', which='major', labelsize=9)
        hist(orderPixelTable["residuals_xy"], bins='scott', ax=bottomright, histtype='stepfilled',
             alpha=0.7, density=True)
        bottomright.set_xlabel('xy residual')
        bottomright.tick_params(axis='both', which='major', labelsize=9)

        subtitle = f"mean res: {mean_res:2.2f} pix, res stdev: {std_res:2.2f}"
        if self.firstGuessMap:
            fig.suptitle(f"residuals of global dispersion solution fitting - {arm} multi-pinhole\n{subtitle}", fontsize=10, y=0.99)
        else:
            fig.suptitle(f"residuals of global dispersion solution fitting - {arm} single pinhole\n{subtitle}", fontsize=10, y=0.99)
        orderPixelTable_groups = orderPixelTable.groupby(['order'])

        if self.arcFrame:
            mean_fwhm, median, std_fwhm = sigma_clipped_stats(orderPixelTable["fwhm_slit_px"].values, sigma=3.0, stdfunc="std", cenfunc="mean", maxiters=3)
            mean_r, median, std_r = sigma_clipped_stats(orderPixelTable["R_slit"].values, sigma=3.0, stdfunc="std", cenfunc="mean", maxiters=3)
        else:
            mean_fwhm, median, std_fwhm = sigma_clipped_stats(orderPixelTable["fwhm_pin_px"].values, sigma=3.0, stdfunc="std", cenfunc="mean", maxiters=3)
            mean_r, median, std_r = sigma_clipped_stats(orderPixelTable["R_pin"].values, sigma=3.0, stdfunc="std", cenfunc="mean", maxiters=3)

        lowerFwhm = mean_fwhm - 3 * std_fwhm
        upperFwhm = mean_fwhm + 3 * std_fwhm
        lowerR = mean_r - 3 * std_r
        upperR = mean_r + 3 * std_r

        if self.arcFrame:

            # UNPACK ORDER TABLE TO GET SLIT HEIGHT IN PX
            orderPolyTable, orderGeoTable, orderMetaTable = unpack_order_table(
                log=self.log, orderTablePath=self.orderTable, extend=0.)

            orderGeoTable[f'{self.axisA}_u'] = orderGeoTable[f'{self.axisA}coord_edgeup'].astype(int)
            orderGeoTable[f'{self.axisA}_l'] = orderGeoTable[f'{self.axisA}coord_edgelow'].astype(int)

            dispMapDF, interOrderMask = twoD_disp_map_image_to_dataframe(log=self.log, slit_length=14, twoDMapPath=dispMapImage, associatedFrame=self.arcFrame, kw=self.kw)

            # MERGE DATAFRAMES
            orderGeoTableLower = orderGeoTable.merge(dispMapDF, left_on=[f'{self.axisA}_l', f'{self.axisB}coord'], right_on=[self.axisA, self.axisB])
            orderGeoTableLower.rename(columns={"order_x": "order"}, inplace=True)
            orderGeoTableLower = orderGeoTableLower[['order', 'x', 'y', 'wavelength', 'slit_position', 'flux', 'pixelScale', f'{self.axisA}coord_centre', f'{self.axisA}coord_edgelow']]

            orderGeoTableUpper = orderGeoTable.merge(dispMapDF, left_on=[f'{self.axisA}_u', f'{self.axisB}coord'], right_on=[self.axisA, self.axisB])
            orderGeoTableUpper.rename(columns={"order_x": "order"}, inplace=True)
            orderGeoTableUpper = orderGeoTableUpper[['order', 'x', 'y', 'wavelength', 'slit_position', 'flux', 'pixelScale', f'{self.axisA}coord_centre', f'{self.axisA}coord_edgeup']]

            orderGeoTable = orderGeoTableLower.merge(orderGeoTableUpper, left_on=[f'{self.axisA}coord_centre', self.axisB], right_on=[f'{self.axisA}coord_centre', self.axisB], suffixes=('_l', '_u'))
            orderGeoTable.rename(columns={"order_l": "order"}, inplace=True)

            orderGeoTable["wavelength"] = (orderGeoTable["wavelength_l"] + orderGeoTable["wavelength_u"]) / 2.
            orderGeoTable["slitLengthPixelsInt"] = np.abs(orderGeoTable[f"{self.axisA}_u"] - orderGeoTable[f"{self.axisA}_l"])
            orderGeoTable["slitLengthPixels"] = np.abs(orderGeoTable[f"{self.axisA}coord_edgeup"] - orderGeoTable[f"{self.axisA}coord_edgelow"])
            orderGeoTable["slitLengthArcsec"] = np.abs(orderGeoTable[f"slit_position_u"] - orderGeoTable[f"slit_position_l"])
            orderGeoTable["pixelScale"] = orderGeoTable["slitLengthArcsec"] / orderGeoTable["slitLengthPixelsInt"]
            orderGeoTable["slitLengthArcsec"] = orderGeoTable["slitLengthPixels"] * orderGeoTable["pixelScale"]

        for name, group in orderPixelTable_groups:

            thisOrder = group["order"].values[0]

            if self.arcFrame:

                slitWidth = self.arcFrame.header[kw(f"SLIT_{arm}")].replace("SLIT", "").split("x")[0]
                fwhmAx.set_title(f"Line FWHM measured via {slitWidth}\" slit arc-lamp frame", fontsize=9)
                fwhmAx.scatter(group["wavelength"], group["fwhm_slit_px"], alpha=0.1)
                mean_fwhm, median, std_fwhm = sigma_clipped_stats(group["fwhm_slit_px"].values, sigma=3.0, stdfunc="std", cenfunc="mean", maxiters=3)
                fwhmAx.set_ylabel("FWHM (px)", fontsize=9)
                mean_wavelength = group["wavelength"].mean()
                fwhmAx.errorbar(mean_wavelength, mean_fwhm, yerr=std_fwhm, fmt='o', color='black', alpha=0.6)
                fwhmAx.tick_params(axis='both', which='major', labelsize=8)
                x_limits = fwhmAx.get_xlim()

                resAx.set_title(f"Resolution measured via {slitWidth}\" slit arc-lamp frame", fontsize=9)
                resAx.scatter(group["wavelength"], group["R_slit"], alpha=0.1)
                mean_resol, median, std_resol = sigma_clipped_stats(group["R_slit"].values, sigma=3.0, stdfunc="std", cenfunc="mean", maxiters=3)
                # ADD TO THE POINT THE ERROR BAR CONTAINED IN STD_RED
                resAx.errorbar(mean_wavelength, mean_resol, yerr=std_resol, fmt='o', color='black', alpha=0.6)

                if thisOrder not in [24]:

                    # FILTER DATA FRAME
                    # FIRST CREATE THE MASK
                    mask = (orderGeoTable["order"] == thisOrder)
                    filteredDf = orderGeoTable.loc[mask]

                    sizeAx.plot(filteredDf["wavelength"], filteredDf["slitLengthArcsec"])
                    sizeAx.set_xlabel('wavelength (nm)', fontsize=9)
                    sizeAx.set_ylabel('slit height\n(arcsec)', fontsize=9)
                    sizeAx.set_xlim(x_limits)
                    sizeAx.tick_params(axis='both', which='major', labelsize=8)
                    sizeAx.set_title(f"Slit height as measured between the lower and upper order edges", fontsize=9)

                    # PLOTTING THE ORDER GAP DISTANCES
                    if thisOrder not in [23]:
                        mask = orderGeoTable['order'] == thisOrder
                        order_l = orderGeoTable[mask]
                        mask = orderGeoTable['order'] == thisOrder + 1
                        order_u = orderGeoTable[mask]
                        if len(order_l.index) and len(order_u.index):
                            # print(order_10)
                            # print(order_9)
                            merged_ul = order_l.merge(order_u, on=[self.axisB], suffixes=('_l', '_u'))

                            merged_ul['order_distance'] = merged_ul[f"{self.axisA}coord_edgelow_u"] - merged_ul[f"{self.axisA}coord_edgeup_l"]
                            # print(merged_ul)
                            # sys.exit(0)
                            pixel_scale_mean = (merged_ul['pixelScale_l'] + merged_ul['pixelScale_u']) / (2.0)
                            gapAx.plot(merged_ul['wavelength_l'], merged_ul['order_distance'] * pixel_scale_mean, label=thisOrder)
                            gapAx.set_xlabel('wavelength (nm)', fontsize=9)
                            gapAx.set_ylabel('inter-order\ngap (arcsec)', fontsize=9)

                        gapAx.set_xlim(x_limits)
                        gapAx.tick_params(axis='both', which='major', labelsize=8)
                        gapAx.set_title(f"Inter-order gap measured between adjacent orders", fontsize=9)

            else:
                fwhmAx.set_title("Line FWHM measured via 0.5\" pinhole  arc-lamp frame", fontsize=9)
                fwhmAx.scatter(group["wavelength"], group["fwhm_pin_px"], alpha=0.1)
                mean_fwhm = group["fwhm_pin_px"].mean()
                std_fwhm = group["fwhm_pin_px"].std()
                fwhmAx.set_ylabel("FWHM (px)", fontsize=9)
                mean_wavelength = group["wavelength"].mean()
                fwhmAx.errorbar(mean_wavelength, mean_fwhm, yerr=std_fwhm, fmt='o', color='black', alpha=0.6)
                fwhmAx.tick_params(axis='both', which='major', labelsize=8)
                x_limits = fwhmAx.get_xlim()

                resAx.set_title("Resolution as measured via 0.5\" pinhole arc-lamp frame", fontsize=9)
                resAx.scatter(group["wavelength"], group["R_pin"], alpha=0.1)
                # CALCULATE THE MEAN AND STD DEV OF THE GROUP AND ADD TO THE PLOT
                mean_resol = group["R_pin"].mean()
                std_resol = group["R_pin"].std()
                # ADD TO THE POINT THE ERROR BAR CONTAINED IN STD_RED
                resAx.errorbar(mean_wavelength, mean_resol, yerr=std_resol, fmt='o', color='black', alpha=0.6)

        resAx.set_ylabel("R", fontsize=9)
        resAx.tick_params(axis='both', which='major', labelsize=8)

        # resAx.scatter(orderPixelTable["wavelength"], orderPixelTable["R"], alpha=0.1)
        resAx.set_xlabel("wavelength (nm)", fontsize=9)

        fwhmAx.set_ylim([lowerFwhm, upperFwhm])
        resAx.set_ylim([lowerR, upperR])
        resAx.set_xlim(x_limits)

        utcnow = datetime.utcnow()
        utcnow = utcnow.strftime("%Y-%m-%dT%H:%M:%S")

        # GET FILENAME FOR THE RESIDUAL PLOT
        if not self.sofName:
            res_plots = filenamer(
                log=self.log,
                frame=self.pinholeFrame,
                settings=self.settings
            )
            res_plots = res_plots.replace(".fits", ".pdf")
        else:
            polyOrders = [orderDeg, wavelengthDeg, slitDeg]
            if isinstance(orderDeg, list):
                merged_list = []
                for sublist in polyOrders:
                    merged_list.extend(sublist)
                polyOrders = merged_list
            polyOrders[:] = [str(l) for l in polyOrders]
            polyOrders = "".join(polyOrders)
            res_plots = self.sofName + f"_RESIDUALS_{polyOrders}.pdf"

        if self.firstGuessMap:
            filePath = f"{self.qcDir}/{res_plots}"
        else:
            filePath = f"{self.qcDir}/{res_plots}"
        self.products = pd.concat([self.products, pd.Series({
            "soxspipe_recipe": self.recipeName,
            "product_label": "DISP_MAP_RES",
            "file_name": res_plots,
            "file_type": "PDF",
            "obs_date_utc": self.dateObs,
            "reduction_date_utc": utcnow,
            "product_desc": f"{self.arm} dispersion solution QC plots",
            "file_path": filePath,
            "label": "QC"
        }).to_frame().T], ignore_index=True)

        qc_settings_plot_tables(log=self.log, qc=self.qc, qcAx=qcAx, settings={**self.recipeSettings, **{"exptime": self.exptime}}, settingsAx=settingsAx)

        plt.tight_layout()
        # plt.show()
        if not self.settings["tune-pipeline"]:
            plt.savefig(filePath, dpi=720)

        plt.clf()
        plt.close(fig)

        if self.settings["tune-pipeline"]:
            import codecs
            filePath = f"residuals.txt"
            exists = os.path.exists(filePath)
            if not exists:
                with codecs.open(filePath, encoding='utf-8', mode='w') as writeFile:
                    writeFile.write(f"polyOrders,mean_x_res,mean_y_res,mean_res,std_res,median_res,median_x_res,median_y_res,CLINE \n")
            with codecs.open(filePath, encoding='utf-8', mode='a') as writeFile:
                writeFile.write(f"{polyOrders},{mean_x_res:0.4f},{mean_y_res:0.4f},{mean_res:2.4f},{std_res:2.4f},{median_res:2.4f},{median_x_res:2.4f},{median_y_res:2.4f},{self.CLINE}\n")

        self.log.debug('completed the ``create_dispersion_map_qc_plot`` method')
        return res_plots

    def _clip_on_measured_line_metrics(
            self,
            orderPixelTable):
        """*clip lines & sets of lines based on measured line metrics (from daostarfinder etc)*

        **Key Arguments:**

        - `orderPixelTable` -- panda's data-frame containing measure line metrics

        **Return:**

        - `orderPixelTable` -- the data-frame with clipped lines indicated in the "sigma_clipped" column
        """
        self.log.debug('starting the ``_clip_on_measured_line_metrics`` method')

        import matplotlib.pyplot as plt
        from astropy.stats import sigma_clip, sigma_clipped_stats
        from astropy.visualization import hist

        # LAYOUT THE FIGURE
        fig = plt.figure(figsize=(6, 12), constrained_layout=True)
        gs = fig.add_gridspec(8, 4)
        toprow = fig.add_subplot(gs[0:2, :])
        midrow = fig.add_subplot(gs[2:4, :])
        midrow2 = fig.add_subplot(gs[4:6, :])
        if self.firstGuessMap:
            bottomleft = fig.add_subplot(gs[6:8, 0:2])
            bottomright = fig.add_subplot(gs[6:8, 2:])

        toprow.set_title(
            "Pre-fitting QC Clipping on Pinhole-Sets", fontsize=10)

        columnsNoStrings = list(orderPixelTable.columns)
        try:
            columnsNoStrings.remove("ion")
        except:
            pass

        if self.firstGuessMap:

            # GROUP BY ARC LINES (MPH SETS)

            lineGroups = orderPixelTable.loc[(orderPixelTable["dropped"] == False)][columnsNoStrings].groupby(['wavelength', 'order']).std()
            lineGroups = lineGroups.reset_index()

            # SIGMA-CLIP THE DATA ON SCATTER
            # masked_residuals = sigma_clip(
            #     lineGroups["xy_diff"], sigma_lower=5000, sigma_upper=7, maxiters=5, cenfunc='median', stdfunc='mad_std')
            # lineGroups["sigma_clipped_scatter"] = masked_residuals.mask

            # SIGMA-CLIP THE DATA ON SCATTER
            lineGroups["sigma_clipped_scatter"] = False
            masked_residuals = sigma_clip(
                lineGroups["x_diff"], sigma_lower=5000, sigma_upper=7, maxiters=3, cenfunc='median', stdfunc='mad_std')
            lineGroups["sigma_clipped_x"] = masked_residuals.mask
            masked_residuals = sigma_clip(
                lineGroups["y_diff"], sigma_lower=5000, sigma_upper=7, maxiters=3, cenfunc='median', stdfunc='mad_std')
            lineGroups["sigma_clipped_y"] = masked_residuals.mask
            masked_residuals = sigma_clip(
                lineGroups["xy_diff"], sigma_lower=5000, sigma_upper=7, maxiters=3, cenfunc='median', stdfunc='mad_std')
            lineGroups["sigma_clipped_xy"] = masked_residuals.mask
            lineGroups.loc[((lineGroups["sigma_clipped_y"] == True) | (lineGroups["sigma_clipped_x"] == True) | (lineGroups["sigma_clipped_xy"] == True)), "sigma_clipped_scatter"] = True

            bottomleft.scatter(
                x=lineGroups["x_diff"],  # numpy array of x-points
                y=lineGroups["y_diff"],  # numpy array of y-points
                s=1,    # 1 number or array of areas for each datapoint (i.e. point size)
                c="black",    # color or sequence of color, optional, default
                marker='x',
                alpha=0.6,
                label="arc line shift from predicted position")

            bottomleft.scatter(
                x=lineGroups.loc[(lineGroups["sigma_clipped_scatter"] == True)]["x_diff"],  # numpy array of x-points
                y=lineGroups.loc[(lineGroups["sigma_clipped_scatter"] == True)]["y_diff"],  # numpy array of y-points
                s=5,    # 1 number or array of areas for each datapoint (i.e. point size)
                c="red",    # color or sequence of color, optional, default
                marker='x',
                alpha=0.6,
                label="clipped arc lines")

            bottomleft.set_ylabel(f"y-shift rms (px)", fontsize=12)
            bottomleft.set_xlabel(f"x-shift rms (px)", fontsize=12)
            bottomleft.tick_params(axis='both', which='major', labelsize=9)
            bottomleft.legend(loc='upper right', bbox_to_anchor=(1.0, -0.05), fontsize=4)

            hist(lineGroups[(lineGroups["sigma_clipped_scatter"] == False)]["xy_diff"], bins='scott', ax=bottomright, histtype='stepfilled',
                 alpha=0.7, density=True)
            bottomright.set_xlabel('xy residual')
            bottomright.tick_params(axis='both', which='major', labelsize=9)

            # REMOVE THE CLIPPED DATA BEFORE CLIPPING ON FWHM
            mask = (lineGroups["sigma_clipped_scatter"] == True)
            dropGroups = lineGroups.loc[mask]
            setsToDrop = dropGroups[['wavelength', 'order']]
            s = orderPixelTable[['wavelength', 'order']].merge(setsToDrop, indicator=True, how='left')
            s["dropped"] = False
            s.loc[(s["_merge"] == "both"), "dropped"] = True
            orderPixelTable["droppedOnScatter"] = s["dropped"].values
            orderPixelTable.loc[(orderPixelTable["droppedOnScatter"] == True), "dropped"] = True

        # SIGMA-CLIP THE DATA ON FWHM
        lineGroups = orderPixelTable.loc[(orderPixelTable["dropped"] == False)][columnsNoStrings].groupby(['wavelength', 'order']).mean()
        lineGroups = lineGroups.reset_index()
        masked_residuals = sigma_clip(
            lineGroups["fwhm_pin_px"], sigma_lower=2.5, sigma_upper=5, maxiters=3, cenfunc='median', stdfunc='mad_std')
        lineGroups["sigma_clipped_fwhm"] = masked_residuals.mask
        lineGroups["sigma_clipped"] = masked_residuals.mask

        toprow.scatter(
            x=lineGroups["wavelength"],  # numpy array of x-points
            y=lineGroups["fwhm_pin_px"],  # numpy array of y-points
            s=1,    # 1 number or array of areas for each datapoint (i.e. point size)
            c="black",    # color or sequence of color, optional, default
            marker='x',
            alpha=0.6,
            label="measured pinhole lines")

        toprow.scatter(
            x=lineGroups[(lineGroups["sigma_clipped_fwhm"] == True)]["wavelength"],  # numpy array of x-points
            y=lineGroups[(lineGroups["sigma_clipped_fwhm"] == True)]["fwhm_pin_px"],  # numpy array of y-points
            s=5,    # 1 number or array of areas for each datapoint (i.e. point size)
            c="red",    # color or sequence of color, optional, default
            marker='x',
            alpha=0.6,
            label="clipped pinhole lines")

        toprow.set_ylabel(f"fwhm (px)", fontsize=12)
        toprow.set_xlabel(f"wavelength (nm)", fontsize=12)
        toprow.tick_params(axis='both', which='major', labelsize=9)
        toprow.legend(loc='upper right', bbox_to_anchor=(1.0, -0.05), fontsize=4)

        # REMOVE THE CLIPPED DATA BEFORE CLIPPING ON FLUX
        mask = (lineGroups["sigma_clipped_fwhm"] == True)
        dropGroups = lineGroups.loc[mask]
        setsToDrop = dropGroups[['wavelength', 'order']]
        s = orderPixelTable[['wavelength', 'order']].merge(setsToDrop, indicator=True, how='left')
        s["dropped"] = False
        s.loc[(s["_merge"] == "both"), "dropped"] = True
        orderPixelTable["droppedOnFWHM"] = s["dropped"].values
        orderPixelTable.loc[(orderPixelTable["droppedOnFWHM"] == True), "dropped"] = True

        # SIGMA-CLIP THE DATA ON FLUX
        lineGroups = lineGroups.loc[~mask]
        masked_residuals = sigma_clip(
            lineGroups["flux"], sigma_lower=5, sigma_upper=5, maxiters=5, cenfunc='mean', stdfunc='std')
        lineGroups["sigma_clipped_flux"] = masked_residuals.mask

        midrow.scatter(
            x=lineGroups["wavelength"],  # numpy array of x-points
            y=lineGroups["flux"],  # numpy array of y-points
            s=1,    # 1 number or array of areas for each datapoint (i.e. point size)
            c="black",    # color or sequence of color, optional, default
            marker='x',
            alpha=0.6,
            label="pinhole flux")

        midrow.set_ylabel(f"pinhole flux", fontsize=12)
        midrow.set_xlabel(f"wavelength (nm)", fontsize=12)
        midrow.tick_params(axis='both', which='major', labelsize=9)
        midrow.legend(loc='upper right', bbox_to_anchor=(1.0, -0.05), fontsize=4)

        midrow.scatter(
            x=lineGroups[(lineGroups["sigma_clipped_flux"] == True)]["wavelength"],  # numpy array of x-points
            y=lineGroups[(lineGroups["sigma_clipped_flux"] == True)]["flux"],  # numpy array of y-points
            s=5,    # 1 number or array of areas for each datapoint (i.e. point size)
            c="red",    # color or sequence of color, optional, default
            marker='x',
            alpha=0.6,
            label="clipped pinhole lines")

        midrow.legend(loc='upper right', bbox_to_anchor=(1.0, -0.05), fontsize=4)

        # REMOVE THE CLIPPED DATA BEFORE CLIPPING ON PEAK
        mask = (lineGroups["sigma_clipped_flux"] == True)
        dropGroups = lineGroups.loc[mask]
        setsToDrop = dropGroups[['wavelength', 'order']]
        s = orderPixelTable[['wavelength', 'order']].merge(setsToDrop, indicator=True, how='left')
        s["dropped"] = False
        s.loc[(s["_merge"] == "both"), "dropped"] = True
        orderPixelTable["droppedOnFlux"] = s["dropped"].values
        orderPixelTable.loc[(orderPixelTable["droppedOnFlux"] == True), "dropped"] = True

        # SIGMA-CLIP THE DATA ON FLUX
        lineGroups = lineGroups.loc[~mask]
        lineGroups["peak"] = lineGroups["peak"] / lineGroups["flux"]
        masked_residuals = sigma_clip(
            lineGroups["peak"], sigma_lower=5000, sigma_upper=7, maxiters=5, cenfunc='mean', stdfunc='std')
        lineGroups["sigma_clipped_peak"] = masked_residuals.mask

        midrow2.scatter(
            x=lineGroups["wavelength"],  # numpy array of x-points
            y=lineGroups["peak"],  # numpy array of y-points
            s=1,    # 1 number or array of areas for each datapoint (i.e. point size)
            c="black",    # color or sequence of color, optional, default
            marker='x',
            alpha=0.6,
            label="pinhole peak flux / mean flux")

        midrow2.set_ylabel(f"pinhole peak flux", fontsize=12)
        midrow2.set_xlabel(f"wavelength (nm)", fontsize=12)
        midrow2.tick_params(axis='both', which='major', labelsize=9)
        midrow2.legend(loc='upper right', bbox_to_anchor=(1.0, -0.05), fontsize=4)

        midrow2.scatter(
            x=lineGroups[(lineGroups["sigma_clipped_peak"] == True)]["wavelength"],  # numpy array of x-points
            y=lineGroups[(lineGroups["sigma_clipped_peak"] == True)]["peak"],  # numpy array of y-points
            s=5,    # 1 number or array of areas for each datapoint (i.e. point size)
            c="red",    # color or sequence of color, optional, default
            marker='x',
            alpha=0.6,
            label="clipped pinhole lines")

        midrow2.legend(loc='upper right', bbox_to_anchor=(1.0, -0.05), fontsize=4)

        # REMOVE THE CLIPPED DATA
        mask = (lineGroups["sigma_clipped_peak"] == True)
        dropGroups = lineGroups.loc[mask]
        setsToDrop = dropGroups[['wavelength', 'order']]
        s = orderPixelTable[['wavelength', 'order']].merge(setsToDrop, indicator=True, how='left')
        s["dropped"] = False
        s.loc[(s["_merge"] == "both"), "dropped"] = True
        orderPixelTable["droppedOnPeak"] = s["dropped"].values
        orderPixelTable.loc[(orderPixelTable["droppedOnPeak"] == True), "dropped"] = True

        # plt.show()
        plt.close("all")

        self.log.debug('completed the ``_clip_on_measured_line_metrics`` method')
        return orderPixelTable

    def create_new_static_line_list(
            self,
            dispersionMapPath):
        """*using a first pass dispersion solution, use a line atlas to generate a more accurate and more complete static line list*

        **Key Arguments:**

        - `dispersionMapPath` -- path to the first pass dispersion solution

        **Return:**

        - `newPredictedLineList` -- a new predicted line list (to replace the static calibration line-list)
        """
        self.log.debug('starting the ``create_new_static_line_list`` method')

        from soxspipe.commonutils import dispersion_map_to_pixel_arrays
        from soxspipe.commonutils.toolkit import read_spectral_format
        import pandas as pd
        from astropy.table import Table

        dp = self.detectorParams

        # READ THE SPECTRAL FORMAT TABLE TO DETERMINE THE LIMITS OF THE TRACES
        orderNums, waveLengthMin, waveLengthMax = read_spectral_format(
            log=self.log, settings=self.settings, arm=self.arm)

        # FIND THE LINE ATLAS
        calibrationRootPath = get_calibrations_path(log=self.log, settings=self.settings)
        lineAtlas = calibrationRootPath + "/" + dp["line-atlas"]
        # LINE LIST TO PANDAS DATAFRAME
        lineAtlas = Table.read(lineAtlas, format='fits')
        lineAtlas = lineAtlas.to_pandas()

        # CLEAN UP LINE LIST DATA
        lineAtlas['ion'] = lineAtlas['ion'].str.decode('ascii')
        lineAtlas['source'] = lineAtlas['source'].str.decode('ascii')

        lineAtlas['wave'] = lineAtlas['wave'].round(5)

        # FILTER DATA FRAME
        # FIRST CREATE THE MASK

        # GET UNIQUE VALUES IN COLUMN
        uniqueions = lineAtlas['ion'].unique()

        # # mask = (lineAtlas['ion'].isin(["XeI", 'UNK']))
        # mask = (lineAtlas['ion'].isin(["HgI", 'UNK']))
        # # mask = (lineAtlas['ion'].isin(["NeI", "NeII", 'UNK']))
        # # mask = (lineAtlas['ion'].isin(["Ar", "ArI", "ArII", "ArIII", 'UNK']))
        # lineAtlas = lineAtlas.loc[mask]

        dfCollection = []
        for o, wmin, wmax in zip(orderNums, waveLengthMin, waveLengthMax):

            wrange = wmax - wmin
            wmin -= wrange / 5
            wmax += wrange / 5

            # FILTER DATA FRAME
            # FIRST CREATE THE MASK
            mask = (lineAtlas['wave'].between(wmin, wmax))
            filteredDf = lineAtlas.loc[mask]
            filteredDf = filteredDf.drop_duplicates(subset=["wave", "ion"])
            wave = filteredDf["wave"]
            ion = filteredDf["ion"]

            if not self.firstGuessMap:
                slit_positions = [0.0]
                slit_indexes = [4]
            else:
                slit_positions = self.uniqueSlitPos
                slit_indexes = list(range(0, len(self.uniqueSlitPos), 1))

            # xpd-update-filter-dataframe-column-values

            for si, sp in zip(slit_indexes, slit_positions):
                myDict = {
                    "order": [o] * len(wave),
                    "wavelength": wave,
                    "slit_position": [sp] * len(wave),
                    "slit_index": [si] * len(wave),
                    "ion": ion}

                orderPixelTable = pd.DataFrame(myDict)
                floats = ["wavelength", 'slit_position']
                ints = ['order', 'slit_index']
                for f in floats:
                    orderPixelTable[f] = orderPixelTable[f].astype(float)
                for i in ints:
                    orderPixelTable[i] = orderPixelTable[i].astype(int)

                orderPixelTable = dispersion_map_to_pixel_arrays(
                    log=self.log,
                    dispersionMapPath=dispersionMapPath,
                    orderPixelTable=orderPixelTable,
                    removeOffDetectorLocation=False
                )

                orderPixelTable.rename(columns={"fit_x": "detector_x", "fit_y": "detector_y"}, inplace=True)
                orderPixelTable = orderPixelTable[['ion', 'wavelength', 'order', 'slit_index', 'slit_position', 'detector_x', 'detector_y']]

                dfCollection.append(orderPixelTable)

        newPredictedLineList = pd.concat(dfCollection, ignore_index=True)

        from astropy.table import Table
        t = Table.from_pandas(newPredictedLineList)
        # t.write("Xe.fits", overwrite=True)
        t.write("Hg.fits", overwrite=True)
        # t.write("Ne.fits", overwrite=True)
        # t.write("Ar.fits", overwrite=True)

        # sys.exit(0)

        self.log.debug('completed the ``create_new_static_line_list`` method')
        return newPredictedLineList

    # use the tab-trigger below for new method
    # xt-class-method
