pymusepipe.config_pipe — pymusepipe 2.16.1 documentation

# Source code for pymusepipe.config_pipe

    # Licensed under a MIT license - see LICENSE
    """MUSE-PHANGS configuration module
    """

    # import modules
    import numpy as np

    __authors__ = "Eric Emsellem"
    __copyright = "(c) 2017-2019, ESO + CRAL"
    __license__ = "MIT License"
    __contact__ = " <eric.emsellem@eso.org>"

    # ======================================================================#
    # BEGIN
    #           FILTER LIST
    # ======================================================================#

    default_filter_list = ["white,Johnson_B,Johnson_V,Cousins_R,"
                           "SDSS_g,SDSS_r,SDSS_i"]
    default_short_filter_list = ["white,Cousins_R"]
    default_PHANGS_filter_list = ["white,Johnson_B,Johnson_V,Cousins_R,"
                                  "SDSS_g,SDSS_r,SDSS_i,WFI_BB,WFI_NB,DUPONT_R"]
    default_short_PHANGS_filter_list = ["white,WFI_BB,DUPONT_R"]

    # -- END ---------------------------------------------------------------#

    # ======================================================================#
    # BEGIN
    #           MUSE PIPELINE FITS KEYWORDS parameters
    # ======================================================================#
    # For musepipe module

    dict_expotypes = {'DARK': 'DARK', 'BIAS': 'BIAS', 'FLAT': 'FLAT,LAMP',
                      'ILLUM': 'FLAT,LAMP,ILLUM', 'TWILIGHT': 'FLAT,SKY',
                      'WAVE': 'WAVE', 'STD': 'STD', 'AST': 'AST',
                      'OBJECT': 'OBJECT', 'SKY': 'SKY',
                      'ASTROMETRY': 'ASTROMETRY',
                      'GEOMETRY': 'GEOMETRY'
                      }

    dict_astrogeo = {'ASTROMETRY': 'Astrometric calibration (ASTROMETRY)',
                     'GEOMETRY': 'WAVE,MASK'
                     }

    # This dictionary contains the types
    dict_listMaster = {'DARK': 'MASTER_DARK',
                       'BIAS': 'MASTER_BIAS',
                       'FLAT': 'MASTER_FLAT',
                       'TRACE': 'TRACE_TABLE',
                       'TWILIGHT': 'TWILIGHT_CUBE',
                       'WAVE': 'WAVECAL_TABLE',
                       'LSF': 'LSF_PROFILE',
                       'STD': 'PIXTABLE_STD'
                       }

    dict_listObject = {'OBJECT': 'PIXTABLE_OBJECT',
                       'SKY': 'PIXTABLE_SKY',
                       'STD': 'PIXTABLE_STD',
                       'REDUCED': 'PIXTABLE_REDUCED'
                       }

    dict_listMasterObject = {**dict_listMaster, **dict_listObject}

    listexpo_files = {
            "OBJECT": ['object', 'OBJECT', str, '20A'],
            "TYPE": ['type', 'ESO DPR TYPE', str, '20A'],
            "DATE":  ['mjd', 'MJD-OBS', np.float, 'E'],
            "MODE":  ['mode', 'ESO INS MODE', str, '10A'],
            "EXPTIME":  ['exptime', 'EXPTIME', float, 'E'],
            "TPLS":  ['tpls', 'ESO TPL START', str, '30A'],
            "TPLN":  ['tplnexp', 'ESO TPL NEXP', np.int, 'J'],
            "TPLNO":  ['tplno', 'ESO TPL EXPNO', np.int, 'J']
            }

    # Excluding files from the checking of their MUSE mode (AO, noAO, N/E etc)
    list_exclude_checkmode = ['BIAS', 'DARK', 'GEOMETRY']
    list_fieldspecific_checkmode = ['ILLUM', 'ASTROMETRY']

    # List of modes allowed for MUSE
    list_musemodes = ['WFM-AO-N', 'WFM-AO-E', 'WFM-NOAO-E', 'WFM-NOAO-N', 'NFM-AO-N']
    dict_musemodes = {'field': 0, 'ao': 1, 'lrange': 2}

    # Suffix for the pre/post-alignment files. Will be part of the output names
    suffix_prealign = "_prealign"
    suffix_checkalign = "_checkalign"

    # List of suffix you wish to have scanned
    suffix_rawfiles = ['fits.fz', 'fits']

    # List of names from the products
    dict_files_iexpo_products = {
            'ALIGN': ['SOURCE_LIST']
            }

    dict_files_products = {
            'STD': ['DATACUBE_STD', 'STD_FLUXES',
                    'STD_RESPONSE', 'STD_TELLURIC'],
            'TWILIGHT': ['DATACUBE_SKYFLAT', 'TWILIGHT_CUBE'],
            'SKY': ['SKY_MASK', 'SKY_IMAGE', 'SKY_LINES', 'SKY_SPECTRUM',
                    'SKY_CONTINUUM'],
            'ALIGN': ['OFFSET_LIST']
            }

    dict_products_scipost = {
            'cube': ['DATACUBE_FINAL', 'IMAGE_FOV'],
            'individual': ['PIXTABLE_REDUCED'],
            'stacked': ['OBJECT_RESAMPLED'],
            'positioned': ['PIXTABLE_POSITIONED'],
            'combined': ['PIXTABLE_COMBINED'],
            'skymodel': ['SKY_MASK', 'SKY_SPECTRUM',
                         'SKY_LINES', 'SKY_IMAGE'],
            'raman': ['RAMAN_IMAGES'],
            'autocal': ['AUTOCAL_FACTORS']
            }

    # -- END ---------------------------------------------------------------#

    # ======================================================================#
    # BEGIN
    #           Printing name configuration
    # ======================================================================#
    # Formatting the output name for dataset (e.g., OBs)
    # E.g., nint_mf = '02' means that 10 will be represented by '10' and 1
    # as '01' (always two digits for an integer)
    # The following  nint_mf = '02' and str_mf='P' will give
    # P01, P02, P03 etc as suffixes for the naming convention
    default_ndigits = 3
    default_str_dataset = 'OB'
    default_str_pointing = 'P'

    # -- END ---------------------------------------------------------------#

    # ======================================================================#
    # BEGIN
    #           DEFAULT PATHS for the Muse pipeline
    #
    # Default hard-coded folders
    # The setting of these folders can be overwritten
    # by a given rc file if provided
    # ======================================================================#
    # for init_musepipe module

    dict_user_folders = {
                # values provide the folder and whether or not this should be attempted to create
                # Muse calibration files (common to all)
                "musecalib": "/data/beegfs/astro-storage/groups/schinnerer/PHANGS/MUSE/soft/muse-pipelines/calib/'",
                # Time varying calibrations
                "musecalib_time": "/data/beegfs/astro-storage/groups/schinnerer/PHANGS/MUSE/time_calibrations/",
                # Calibration files (specific to OBs)
                "root": "/data/beegfs/astro-storage/groups/schinnerer/PHANGS/MUSE/live/",
                # Default prefix for Datasets
                "str_dataset": default_str_dataset,
                # Default prefix for Pointings
                "str_pointing": default_str_pointing,
                # Default number of digits for Blocks
                "ndigits": default_ndigits
                }

    # Default initialisation file
    default_rc_filename = "~/.musepiperc"

    PHANGS_reduc_config = {'fakemode': False,
                           'nocache': False,
                           'checkmode': False,
                           'overwrite_astropy_table': True,
                           'filter_list': "white,WFI_BB,DUPONT_R",
                           'filter_for_alignment': "WFI_BB"}

    PHANGS_combine_config = {'fakemode': False, 'nocache': False}

    # Extra filters which may be used in the course of the reduction
    dict_extra_filters = {
        # Narrow band filter
        "WFI_BB": "data/Filters/LaSilla_WFI_ESO844p.txt",
        # Broad band filter
        "WFI_NB": "data/Filters/LaSilla_WFI_ESO856p.txt",
        # DUPONT R filter
        "DUPONT_R": "data/Filters/LCO_SITe3_rp.txt",
        "LCO-HA663": "data/Filters/Ha663BP6_AA.txt",
        "LCO-HA657": "data/Filters/Ha657BP6_AA.txt"
        }

    # Default hard-coded fits files - Calibration Tables
    # These are also overwritten by the given calib input file (if provided)
    dict_calib_tables = {
        # Muse geometric file
        "geo_table": "geometry_table_wfm.fits",
        # Astrometry
        "astro_table": "astrometry_wcs_wfm.fits",
        # Bad pixel table
        "badpix_table": "badpix_table_2015-06-02.fits",
        # Vignetting
        "vignetting_mask": "vignetting_mask.fits",
        # Flux table
        "std_flux_table": "std_flux_table.fits",
        # Sky Flat files
        "extinct_table": "extinct_table.fits",
        # Line Catalog
        "line_catalog": "line_catalog.fits",
        # Sky lines
        "sky_lines": "sky_lines.fits",
        # Filter List
        "filter_list": "filter_list.fits",
        }

    # Default structure folders
    # If already existing, won't be created
    # If not, will be created automatically
    dict_input_folders = {
                # Raw Data files
                "rawfiles": "Raw/",
                # Config files
                "config": "Config/",
                # Tables
                "astro_tables": "Astro_tables/",
                # esores log files
                "esorex_log": "Esorex_log/",
                # Data Products - first writing
                "pipe_products": "Pipe_products/",
                # Log
                "log": "Log/"
                }

    # Values provide the folder names for the file structure
    # If already existing, won't be created
    # If not, will be created automatically
    dict_folders = {
                # Master Calibration files
                "master": "Master/",
                # Object files
                "object": "Object/",
                # Sky files
                "sky": "Sky/",
                # Std files
                "std": "Std/",
                # Reconstructed Maps
                "maps": "Maps/",
                # SOF folder
                "sof": "Sof/",
                # Figure
                "figures": "Figures/"
                }

    # This dictionary includes extra folders for certain specific task
    # e.g., alignment - associated with the target
    # Will be created automatically if not already existing
    dict_folders_target = {
            "alignment": "Alignment/"
            }

    dict_combined_folders = {
            # Sof files
            "sof": "Sof/",
            # Combined products
            "cubes": "Cubes/",
            # esores log files
            "esorex_log": "Esorex_log/",
            # Data Products - first writing
            "pipe_products": "Pipe_products/",
            # Log
            "log": "Log/"
            }

    # -- END ---------------------------------------------------------------#
    # ======================================================================#
    # BEGIN
    #           DEFAULT GEOMETRY AND ASTROWCS FILES
    #
    # These are provided if you need time varying astrometry files
    # ======================================================================#
    # For musepipe module

    dict_geo_astrowcs_table = {
        'comm1': ['2014-02-09', '2014-02-09'],
        'comm2a': ['2014-04-27', '2014-05-06'],
        'comm2b': ['2014-07-24', '2014-08-03'],
        'gto01': ['2014-09-13', '2014-09-26'],
        'gto02': ['2014-10-18', '2014-10-29'],
        'gto03': ['2014-11-16', '2014-11-27'],
        '2014d': ['2014-12-16', '2014-12-25'],
        'gto05': ['2015-04-14', '2015-04-25'],
        'gto06': ['2015-05-10', '2015-05-23'],
        'gto07': ['2015-08-17', '2015-08-24'],
        'gto08': ['2015-09-07', '2015-09-12'],
        'gto09': ['2015-10-09', '2015-10-16'],
        'gto10': ['2015-11-04', '2015-11-13'],
        'gto11': ['2016-01-31', '2016-02-06'],
        'gto12': ['2016-03-09', '2016-03-14'],
        'gto13': ['2016-04-05', '2016-04-10'],
        'gto14': ['2016-05-06', '2016-05-12'],
        'gto15': ['2016-08-29', '2016-09-06'],
        'gto16': ['2017-01-27', '2017-02-03'],
        # Initial gto17 is actually extended with gto18
        # which did not have a solution
        #    'gto17' :   ['2017-04-22', '2017-04-25'],
        #    'gto18' :   ['2017-05-20', '2017-05-23'],
        'gto17': ['2017-04-22', '2017-05-23'],
        'gto19': ['2017-09-19', '2017-09-25'],
        'gto20': ['2017-10-16', '2017-10-26'],
        'gto21': ['2017-11-15', '2017-11-20'],
        'gto22': ['2018-02-11', '2018-02-16'],
        'gto23': ['2018-03-14', '2018-03-19'],
        'gto24': ['2018-04-11', '2018-04-19'],
        'gto25': ['2018-05-10', '2018-05-13'],
        'gto26': ['2018-08-12', '2018-08-18'],
        'gto27': ['2018-09-05', '2018-09-15'],
        'gto28': ['2018-10-07', '2018-10-15'],
        # gto30 did not provide a solution hence
        # gto29 also includes gto30 run
        #    'gto29' :   ['2018-07-14', '2018-11-14'],
        #    'gto30' :   ['2018-12-06', '2018-12-12'],
        'gto29': ['2018-07-14', '2018-12-12'],
        'gto31': ['2019-01-02', '2019-01-09'],
        'gto32': ['2019-03-04', '2019-03-07'],
        'gto33': ['2019-04-05', '2019-04-11'],
        'gto34': ['2019-05-03', '2019-05-05']
        }
    # -- END ---------------------------------------------------------------#

    # ======================================================================#
    # BEGIN
    #           DEFAULT KEYWORDS and COLUMNS for OFFSET_TABLES
    #
    # Define useful keywords for fits table and images
    # ======================================================================#
    mjd_names = {'table': "MJD_OBS", 'image': "MJD-OBS"}
    date_names = {'table': "DATE_OBS", 'image': "DATE-OBS"}
    tpl_names = {'table': "TPL_START", 'image': "HIERARCH ESO TPL START"}
    iexpo_names = {'table': "IEXPO_OBS", 'image': "MUSEPIPE_IEXPO"}
    dataset_names = {'table': "DATASET_OBS", 'image': "MUSEPIPE_DATASET", 'oldimage': "MUSEPIPE_POINTING"}

    default_offset_table = {'date': [date_names['table'], 'S23', ""],
                            'mjd': [mjd_names['table'], 'f8', 0.0],
                            'tpls': [tpl_names['table'], 'S19', ""],
                            'iexpo': [iexpo_names['table'], 'i4', 0],
                            'dataset': [iexpo_names['table'], 'i4', 0],
                            'ora': ["RA_OFFSET", 'f8', 0.0],
                            'odec': ["DEC_OFFSET", 'f8', 0.0],
                            'scale': ["FLUX_SCALE", 'f8', 1.0]}
    # -- END ---------------------------------------------------------------#

    # --- Some useful functions using the input configuration --------------#
    def get_suffix_product(expotype):
        return dict_listMasterObject[expotype]

    default_prefix_wcs = "refwcs_"
    prefix_mosaic = "full"
    default_prefix_wcs_mosaic = "{0}{1}".format(prefix_mosaic, default_prefix_wcs)
    default_wave_wcs = 6500.0
    default_prefix_mask = "mask_"

    ao_mask_lambda = [5800, 5970]
    lambdaminmax_for_wcs = [6800, 6805]
    lambdaminmax_for_mosaic = [4700, 9400]

    # ===========================================
    # Recipes for data reduction
    dict_recipes_per_num = {1: 'bias',
                            2: 'flat',
                            3: 'wave',
                            4: 'lsf',
                            5: 'twilight',
                            6: 'scibasic_all',
                            7: 'standard',
                            8: 'sky',
                            9: 'prep_align',
                            10: 'align_bydataset',
                            11: 'align_bygroup',
                            12: 'scipost_perexpo',
                            13: 'scipost_sky',
                            14: 'combine_dataset'
                            }

    # Value by default for certain fixed parameters in recipes
    dict_default_for_recipes = {'fraction': 0.8,
                                'skymethod': 'model',
                                'illum': True,
                                'filter_for_alignment': 'Cousins_R',
                                'line': None,
                                'lambda_window': 10.0
                                }

    # and creating the inverse dictionary
    dict_recipes_per_name = {}
    for key in dict_recipes_per_num:
        dict_recipes_per_name[dict_recipes_per_num[key]] = key