"""
pyart.io.odim_h5_writer.py
=========================

Utilities for writing ODIM hdf5 files.

.. autosummary::
    :toctree: generated/

    write_odim_h5
    _to_str
    _get_sec_since_epoch
    _tree
    _check_file_exists
    _create_odim_h5_file
    _create_odim_h5_grp
    _create_odim_h5_sub_grp
    _create_odim_h5_attr
    _create_odim_h5_dataset
    _map_radar_quantity
    _get_data_from_field
    _map_radar_to_how_dict

"""

import datetime
import calendar
import time
from warnings import warn
from collections import defaultdict
from pathlib import Path
import numpy as np

from ..exceptions import MissingOptionalDependency
from ..config import get_fillvalue

try:
    import h5py
    _H5PY_AVAILABLE = True
except ImportError:
    _H5PY_AVAILABLE = False

#from ..config import FileMetadata
#from ..core.radar import Radar
#from ..lazydict import LazyLoadDict


def write_odim_h5(filename, radar, field_names=None, physical=True,
                  compression="gzip", compression_opts=6):
    """
    Write a Radar object to a EUMETNET OPERA compliant HDF5 file.

    The files produced by this routine follow the EUMETNET OPERA information
    model:
    http://eumetnet.eu/wp-content/uploads/2017/01/OPERA_hdf_description_2014.pdf

    Supported features:
      - Writing PPIs: PVOL and SCAN objects
        - Different sweeps are saved in different dataset groups
      - Writing sectorized PPIs and SCANs: AZIM objects
      - Writing RHIs: ELEV objects

    Not yet supported:
      - Mixed datasets (how group always on top level)
      - Single ray data (e.g. from fixed staring mode)
      - Profiles


    Parameters
    ----------
    filename : str
        Filename of file to create.
    radar : Radar
        Radar object to process.
    field_names : list of str
        The list of fields from the radar object to save. If none all fields
        in the radar object will be saved.
    physical : Bool
        If true the physical values are stored. nodata parameter is equal to
        the _FillValue parameter in the field metadata or the default Py-ART
        fill value. If false the data is converted into binary values using a
        linear conversion. The gain and offset are either specified in the
        metadata of the field with keywords 'scale_factor' and 'add_offset' or
        calculated on the fly. keyword '_Write_as_dtype' specifies the
        datatype. It can be either 'uint8' or 'uint16'. The default datatype
        is uint8. The 'undetect' parameter is not used
    compression : str
        The type of compression for the datasets. Typical are "gzip" and "lzf".
    compression_opts : any
        The compression options. In the case of gzip is the level between 0 to
        9 (recomended 1 to 6). In the case of lzf there are not options.

    """

    #Initialize hdf5 file
    hdf_id = _create_odim_h5_file(filename)

    #Determine odim object, number of datasets (PPIs, RHIs) and
    # check if sweep mode does not change between different sweeps
    if radar.scan_type == 'rhi':
        odim_object = 'ELEV'
        n_datasets = np.size(radar.fixed_angle['data'])
    else:
        if radar.scan_type == 'ppi':
            n_datasets = radar.nsweeps
            if len(set(radar.fixed_angle['data'])) <= 1:
                odim_object = 'SCAN'
            else:
                if radar.sweep_mode is not None:
                    if len(set(radar.sweep_mode['data'])) <= 1:
                        nray_in_sweep = radar.rays_per_sweep['data'][0]
                        ray_angle_res = radar.ray_angle_res['data'][0]
                        if nray_in_sweep * ray_angle_res < 360:
                            odim_object = 'AZIM'
                        else:
                            odim_object = 'PVOL'
                    else:
                        warn("Radar sweep mode changes, which is not " +
                             "yet supported to write ODIM HDF5 file.")
                        return
                else:
                    warn("ODIM object could not be identified.")
                    return

    #Determine number of different data types per dataset and list of fields
    radar_field_names = list(radar.fields.keys())
    if field_names is not None:
        # check that all desired fields are in radar object
        aux_field_names = []
        for field_name in field_names:
            if field_name not in radar_field_names:
                warn(field_name+' not in radar object')
            else:
                aux_field_names.append(field_name)
        if np.size(aux_field_names) == 0:
            warn('No matching field names available')
            return
        field_names = aux_field_names

    else:
        field_names = list(radar.fields.keys())
    n_datatypes = len(field_names)

    #Create level 1 group structure
    where1_grp = _create_odim_h5_grp(hdf_id, '/where')
    what1_grp = _create_odim_h5_grp(hdf_id, '/what')
    how1_grp = _create_odim_h5_grp(hdf_id, '/how')

    dataset_grps = list()
    for i in range(n_datasets):
        name = 'dataset'+str(i+1)
        grp_id = _create_odim_h5_grp(hdf_id, name)
        dataset_grps.append(grp_id)

    #Write ODIM Conventions attribute
    _create_odim_h5_attr(hdf_id, 'Conventions', 'ODIM_H5/V2_2')

    #where - lon, lat, height
    lon = radar.longitude['data']
    lat = radar.latitude['data']
    height = radar.altitude['data']
    _create_odim_h5_attr(where1_grp, 'lon', lon)
    _create_odim_h5_attr(where1_grp, 'lat', lat)
    _create_odim_h5_attr(where1_grp, 'height', height)

    #what - version, date, time, source, object
    odim_version = _to_str(radar.metadata['version'])
    odim_source = _to_str(radar.metadata['source'])

    #Time
    odim_time_struct = time.strptime(
        radar.time['units'], "seconds since %Y-%m-%dT%H:%M:%SZ")
    odim_datetime_struct = datetime.datetime.utcfromtimestamp(
        calendar.timegm(odim_time_struct))

    #Time relative to center of first gate?
    odim_dt = radar.time['data'][0]
    odim_datetime_start = odim_datetime_struct + datetime.timedelta(
        seconds=odim_dt)

    odim_time = datetime.datetime.strftime(odim_datetime_start, "%H%M%S")
    odim_date = datetime.datetime.strftime(odim_datetime_start, "%Y%m%d")

    #Create and fill what1 group attributes
    _create_odim_h5_attr(what1_grp, 'time', odim_time)
    _create_odim_h5_attr(what1_grp, 'date', odim_date)
    _create_odim_h5_attr(what1_grp, 'version', odim_version)
    _create_odim_h5_attr(what1_grp, 'source', odim_source)
    _create_odim_h5_attr(what1_grp, 'object', odim_object)

    #How variables
    #General
    how_var_general = ['system', 'software', 'sw_verison']

    #Individual radar
    how_var_instrument = [
        'beamwidth', 'wavelength', 'rpm', 'elevspeed', 'pulsewidth',
        'RXbandwidth', 'lowprf', 'midprf', 'highprf', 'TXlossH', 'TXlossV'
        'injectlossH', 'injectlossV', 'RXlossH', 'RXlossV',
        'radomelossH', 'radomelossV', 'antgainH', 'antgainV', 'beamwH',
        'beamwV', 'gasattn', 'radconstH', 'radconstV', 'nomTXpower', 'TXpower'
        'powerdiff', 'phasediff', 'NI', 'Vsamples']

    # Map radar.metadata to how1_dict if entries are available
    if any(x in how_var_general for x in radar.metadata):
        if 'system' in radar.metadata:
            how_var_gen = ['system']
        if 'software' in radar.metadata:
            how_var_gen.append('software')
        if 'sw_verison' in radar.metadata:
            how_var_gen.append('sw_version')
        how1_gen_dict = _map_radar_to_how_dict(radar.metadata)
        for name in how_var_gen:
            _create_odim_h5_attr(how1_grp, name, how1_gen_dict[name])

    #Map radar.instrument_parameters to how1_dict
    if radar.instrument_parameters is not None:
        radar_ins_obj = radar.instrument_parameters
        how1_ins_dict = _map_radar_to_how_dict(radar_ins_obj)
    else:
        how1_ins_dict = None
        warn("Instrument parameters not available in radar object")

    #Map radar.radar_calibration to how1_dict
    if radar.radar_calibration is not None:
        radar_cal_obj = radar.radar_calibration
        how1_cal_dict = _map_radar_to_how_dict(radar_cal_obj)
    else:
        how1_cal_dict = None
        warn("Radar calibration parameters not available in radar object")

    for name in how_var_instrument:
        if how1_ins_dict is not None:
            if name in how1_ins_dict:
                _create_odim_h5_attr(how1_grp, name, how1_ins_dict[name])
        elif how1_cal_dict is not None:
            if name in how1_cal_dict:
                _create_odim_h5_attr(how1_grp, name, how1_cal_dict[name])

    #Create level 2 group structure
    datatype_grps = [] # 2D list datatype group ids
    where2_grps = list() # 1D list of where groups
    what2_grps = list() # 1D list of what groups
    how2_grps = list() # 1D list of how groups

    for i in range(n_datasets):
        where2_id = _create_odim_h5_sub_grp(dataset_grps[i], 'where')
        where2_grps.append(where2_id)
        what2_id = _create_odim_h5_sub_grp(dataset_grps[i], 'what')
        what2_grps.append(what2_id)
        how2_id = _create_odim_h5_sub_grp(dataset_grps[i], 'how')
        how2_grps.append(how2_id)

        datatype_grps.append([]) #empty list for each row

        for j in range(n_datatypes):
            name = 'data'+str(j+1)
            datatype_ind = _create_odim_h5_sub_grp(dataset_grps[i], name)
            datatype_grps[i].append(datatype_ind)

    # Dataset specific what header Attributes
    what_var_dataset = ['product', 'prodpar', 'startdate', 'starttime',
                        'enddate', 'endtime']

    # Data attributes
    what_var_data = ['quantity', 'gain', 'offset', 'nodata', 'undetect']

    # Supported scan types
    odim_product_list = ['PPI', 'RHI']

    if odim_object in ['PVOL', 'AZIM', 'SCAN', 'ELEV']:
        #where2
        where2_variables = ['nbins', 'rstart', 'rscale', 'nrays']
        where2_dict = {}

        where2_dict['nbins'] = np.int64(np.repeat(radar.ngates, n_datasets))
        where2_dict['rstart'] = np.repeat(
            np.double((radar.range['data'][0])/(1000.)), n_datasets)  #[km]
        where2_dict['nrays'] = np.int64(radar.rays_per_sweep['data'])

        if len(set(radar.range['data'][1:] - radar.range['data'][0:-1])) <= 1:
            range_resolution = np.double(
                radar.range['data'][1]-radar.range['data'][0])
        else:
            range_resolution = np.median(
                radar.range['data'][1:] - radar.range['data'][0:-1])
            warn("Radar range resolution changes between gates. " +
                 "The median resolution is taken as reference.")
        where2_dict['rscale'] = np.repeat(range_resolution, n_datasets)

        if odim_object in ['PVOL', 'AZIM', 'SCAN']:
            where2_variables.extend(['elangle', 'a1gate'])
            where2_dict['elangle'] = np.double(radar.fixed_angle['data'])
            where2_dict['a1gate'] = np.int64(
                radar.sweep_start_ray_index['data'])

        #Sector Specific
        elif odim_object == 'AZIM':
            where2_variables.extend(['startaz', 'stopaz'])
            start_az = []
            stop_az = []
            for i in radar.sweep_start_ray_index['data']:
                start_az_tmp = radar.azimuth['data'][i]
                start_az.append(start_az_tmp)
            for i in radar.sweep_end_ray_index['data']:
                stop_az_tmp = radar.azimuth['data'][i]
                stop_az.append(stop_az_tmp)
            where2_dict['startaz'] = np.double(start_az)
            where2_dict['stopaz'] = np.double(stop_az)

        #RHI specific
        elif odim_object == 'ELEV':
            where2_variables.extend(['az_angle', 'angles', 'range'])
            where2_dict['range'] = np.repeat(
                np.double((np.max(radar.range['data']) +
                           (radar.range['data'][0])) / 1000.),
                n_datasets) #[km] to end of last gate

            az_angle = []
            for i in range(n_datasets):
                az_angle_tmp = radar.fixed_angle['data'][i]
                az_angle.append(az_angle_tmp)
            where2_dict['az_angle'] = np.double(az_angle)
            angles_mx = [
                [0 for x in range(radar.nrays)] for y in range(n_datasets)]
            for i in range(n_datasets):
                angles_mx[i][:] = radar.elevation['data']
            where2_dict['angles'] = np.double(angles_mx)

        # Write where2 group attributes
        ind = 0
        for i in where2_grps:
            for name in where2_variables:
                if where2_dict[name] is not None:
                    _create_odim_h5_attr(i, name, where2_dict[name][ind])
            ind = ind+1

        # what2
        scan_type = radar.scan_type
        what2_dict = {}
        what2_dict['product'] = np.repeat(scan_type.upper(), n_datasets)
        if scan_type.upper() in odim_product_list:
            if scan_type.upper() == 'PPI':
                # Eleveation angle/s (deg)
                what2_dict['prodpar'] = np.double(radar.fixed_angle['data'])
            elif scan_type.upper() == 'RHI':
                # Correct for Index to scalar error - only 1 azimuth -
                # create array
                if n_datasets <= 1:
                    what2_dict['prodpar'] = np.repeat(
                        np.double(radar.fixed_angle['data']), n_datasets)
                else:
                    what2_dict['prodpar'] = np.double(
                        radar.fixed_angle['data'])
            else:
                warn("Scan type "+scan_type+" not yet supported")
        else:
            warn("Scan type "+scan_type+" not yet supported")

        #time
        _time = radar.time['data']
        start_sweep_ind = radar.sweep_start_ray_index['data']
        end_sweep_ind = radar.sweep_end_ray_index['data']
        startdate = []
        starttime = []
        enddate = []
        endtime = []
        for i, start_ind in enumerate(start_sweep_ind):
            _startdate = odim_datetime_struct + datetime.timedelta(
                seconds=_time[start_ind])
            _startdate_str = datetime.datetime.strftime(_startdate, "%Y%m%d")
            _starttime_str = datetime.datetime.strftime(_startdate, "%H%M%S")
            startdate.append(_startdate_str)
            starttime.append(_starttime_str)
            _enddate = odim_datetime_struct + datetime.timedelta(
                seconds=_time[end_sweep_ind[i]])
            _enddate_str = datetime.datetime.strftime(_enddate, "%Y%m%d")
            _endtime_str = datetime.datetime.strftime(_enddate, "%H%M%S")
            enddate.append(_enddate_str)
            endtime.append(_endtime_str)

        what2_dict['startdate'] = startdate
        what2_dict['starttime'] = starttime
        what2_dict['enddate'] = enddate
        what2_dict['endtime'] = endtime

        #fill the what2 group attributes
        ind = 0
        for i in what2_grps:
            for name in what_var_dataset:
                if what2_dict[name] is not None:
                    _create_odim_h5_attr(i, name, what2_dict[name][ind])
            ind = ind + 1

        #Dataset specific how variables
        #Necessary?
        ssri = radar.sweep_start_ray_index['data']
        seri = radar.sweep_end_ray_index['data']

        if odim_object in ['PVOL', 'AZIM', 'SCAN']:
            how2_dict = {}
            #PPI
            how_var_dataset = ['elangles', 'startazT', 'stopazT']

            el_angle = []
            T_start_az = []
            T_stop_az = []

            for i in range(n_datasets):
                el_angle_tmp = radar.elevation['data'][ssri[i]:seri[i]+1]
                el_angle.append(el_angle_tmp)
                T_start_az_tmp = (
                    radar.time['data'][ssri[i]:seri[i]+1] -
                    radar.time['data'][0])
                T_stop_az_tmp = (
                    radar.time['data'][ssri[i]:seri[i]+1] +
                    radar.time['data'][0])
                t1_epo = []
                for sec in T_start_az_tmp:
                    t_tmp = odim_datetime_struct + datetime.timedelta(
                        seconds=sec)
                    t_epo_sec = _get_sec_since_epoch(t_tmp)
                    t1_epo.append(t_epo_sec)
                T_start_az.append(t1_epo)
                t2_epo = []
                for sec in T_stop_az_tmp:
                    t_tmp = odim_datetime_struct + datetime.timedelta(
                        seconds=sec)
                    t_epo_sec = _get_sec_since_epoch(t_tmp)
                    t2_epo.append(t_epo_sec)
                T_stop_az.append(t2_epo)

            how2_dict['elangles'] = np.double(el_angle)
            how2_dict['startazT'] = np.double(T_start_az)
            how2_dict['stopazT'] = np.double(T_stop_az)

            if radar.azimuth['data'] is not None:
                how_var_dataset.extend(['startazA', 'stopazA'])
                start_azA = []
                stop_azA = []
                for i in range(n_datasets):
                    start_azA_tmp = (
                        radar.azimuth['data'][ssri[i]:seri[i]+1] -
                        radar.ray_angle_res['data'][i] / 2.)
                    start_azA_tmp[start_azA_tmp < 0.] += 360. #[0 360]
                    stop_azA_tmp = (
                        radar.azimuth['data'][ssri[i]:seri[i]+1] +
                        radar.ray_angle_res['data'][i] / 2.)
                    stop_azA_tmp[stop_azA_tmp > 360.] -= 360. #[0 360]
                    start_azA.append(start_azA_tmp)
                    stop_azA.append(stop_azA_tmp)

                how2_dict['startazA'] = np.double(start_azA)
                how2_dict['stopazA'] = np.double(stop_azA)

        # ELEV - RHI
        # how_var_dataset = ['startelA', 'stopelA', 'startelT', 'stopelT']
        # Not supported by ODIM reader yet

            #fill the how2 group attributes
            ind = 0
            for i in how2_grps:
                for name in how_var_dataset:
                    if how2_dict[name] is not None:
                        _create_odim_h5_attr(i, name, how2_dict[name][ind])
                ind = ind + 1

        # create level 3 data and what group structure and fill data
        what3_grps = []
        what3_dict = _tree()

        for i in range(n_datasets):
            what3_grps.append([])
            for j, field_name in enumerate(field_names):
                what3_id = _create_odim_h5_sub_grp(
                    datatype_grps[i][j], 'what')
                what3_grps[i].append(what3_id)

                # ODIM field name
                radar_quantity = _map_radar_quantity(field_name)
                what3_dict[i][j]['quantity'] = radar_quantity

                # get data
                data_dict = _get_data_from_field(
                    radar, i, field_name, physical=physical)

                # write data
                what3_dict[i][j]['gain'] = data_dict['gain']
                what3_dict[i][j]['offset'] = data_dict['offset']

                if data_dict['nodata'] is not None:
                    what3_dict[i][j]['nodata'] = data_dict['nodata']
                if data_dict['undetect'] is not None:
                    what3_dict[i][j]['undetect'] = data_dict['undetect']

                _create_odim_h5_dataset(
                    datatype_grps[i][j], 'data', data_dict['data'],
                    make_legend=False, compression=compression,
                    compression_opts=compression_opts)

                # Add legend data if 'label' key is present in radar quantity
                # dictionary
                if ('labels' in radar.fields[field_name] and
                        'ticks' in radar.fields[field_name]):
                    labs = radar.fields[field_name]['labels']
                    tics = radar.fields[field_name]['ticks']
                    data_legend = []
                    for lab_tic in zip(labs, tics):
                        data_legend.append(lab_tic)
                    _create_odim_h5_dataset(
                        datatype_grps[i][j], 'legend', data_legend,
                        make_legend=True, compression=compression,
                        compression_opts=compression_opts)

        # fill the what3 group attributes of data
        for i in range(n_datasets):
            for j in range(n_datatypes):
                for name in what_var_data:
                    if name in what3_dict[i][j]:
                        rq_name = what3_dict[i][j][name]
                        _create_odim_h5_attr(what3_grps[i][j], name, rq_name)
                    # else:
                    #    warn("Attribute "+name+" not specified")

    #close HDF file
    hdf_id.close()
    _check_file_exists(filename)


def _to_str(text):
    """
    Converter: From byte arrays to string if necessary.

    Parameters:
    -----------
    text: byte array
        Input for conversion

    Returns:
    -------
    text : str
        String of input array

    """
    if hasattr(text, 'decode'):
        return text.decode('utf-8')

    return text


def _check_file_exists(filename):
    """
    Check for ODIM h5 file existence

    Parameters:
    -----------
    filename : str
        Filename string pointing to ODIM h5 file that should be present.

    """
    myfile = Path(filename)
    if myfile.is_file():
        print('ODIM h5 file '+filename+' written.')
    else:
        print('Warning: ODIM h5 file '+filename+' not written.')


def _get_sec_since_epoch(time_f):
    """
    Calculate seconds since 1970-01-01 epoch with microseconds precision

    Parameters:
    -----------
    time_f : datetime
        Datetime object to process

    Returns:
    --------
    sec_since_epoch : double
        Seconds since 1970 epoch

    """
    if hasattr(time_f, 'microsecond'):
        sec_since_epoch = (
            calendar.timegm(time_f.timetuple()) +
            time_f.microsecond/1000000.0)
    else:
        sec_since_epoch = calendar.timegm(time_f.timetuple())

    return sec_since_epoch


def _tree():
    """
    Initialize a tree structure for a multidimensional dictionary

    """
    return defaultdict(_tree)


def _create_odim_h5_file(filename, access_mode='w', driver=None):
    """
    Initialize HDF5 file with h5py (https://www.h5py.org/) to write ODIM
    compliant data structure.

    Parameters:
    -----------
    filename : str
        Filename of ODIM HDF5 file to create
    access_mode : str
        'w' for going into write model
    driver : str, optional
        Optional driver specification in h5py

    Returns:
    --------
    file_id : object
        h5py file ID object

    """
    # Check that h5py module is available
    if not _H5PY_AVAILABLE:
        raise MissingOptionalDependency(
            "h5py is required to use the ODIM HDF5 writer, '" +
            "'but is not installed")

    file_id = h5py.File(filename, access_mode, driver)

    return file_id


def _create_odim_h5_grp(file_id, grp_name):
    """
    Create HDF5 group for a specified file ID.

    Parameters:
    -----------
    file_id : object
        File ID object pointing to h5py file where group should be created.
    grp_name : str
        Group name to be created

    Returns:
    --------
    grp_id : obejct
        h5py group ID object

    """
    grp_id = file_id.create_group(grp_name)

    return grp_id


def _create_odim_h5_sub_grp(grp_id, sub_grp_name):
    """
    Create HDF5 subgroup within a specified group.

    Parameters:
    -----------
    grp_id : object
        Group ID object pointing to h5py group where subgroup should be
        created.
    sub_grp_name : str
        Subgroup name to be created

    Returns:
    --------
    sub_grp_id : obejct
        h5py subgroup ID object

    """
    sub_grp_id = grp_id.create_group(sub_grp_name)

    return sub_grp_id


def _create_odim_h5_attr(grp_id, name, data):
    """
    Create and fill group (subgroup) attributes with metadata.

    Parameters:
    -----------
    grp_id : object
        h5py object where attributes shoul be created.
    name : str
        Name of the attribute to be created.
    data : data (str, int, double, array)
        Data to be saved in h5py group attribute

    """
    #Check if data is str
    if isinstance(data, str):
        string_dt = h5py.special_dtype(vlen=str)
        grp_id.attrs.create(
            name, np.array(data, dtype=string_dt), dtype=string_dt)
    else:
        grp_id.attrs.create(name, data)


def _create_odim_h5_dataset(ID, name, data_arr, make_legend=False,
                            compression="gzip", compression_opts=6):
    """
    Create and save radar field data array to h5py dataset.

    Parameters:
    -----------
    ID : object
        h5py object ID pointing to dataset to be created and filled.
    name : str
        Name of the dataset to be created
    data_arr : array
        Array containing the data
    make_legend : Bool
        If true a legend is created
    compression : str
        compression type and level. Can be "gzip", "lzf"
    compression_opts : int
        if "gzip" is choosen, the level of compression. ODIM recommends
        levels between 1 and 6
    """
    if make_legend:
        dt_tmp = [
            ('label', h5py.special_dtype(vlen=str)), ('tick', np.float)]
        ds = ID.create_dataset(
            name, (1, len(data_arr)), dtype=dt_tmp, compression=compression,
            compression_opts=compression_opts)
        data_np_arr = np.array(data_arr, dtype=dt_tmp)
        ds[0] = data_np_arr
    else:
        ID.create_dataset(
            name, data=data_arr, compression=compression,
            compression_opts=compression_opts)


def _map_radar_quantity(field_name):
    """
    Map radar field quantities to ODIM compliant quantities.

    Parameters:
    -----------
    field_name : str
        the  radar field names

    Returns:
    --------
    ODIM_field_name : str
        ODIM compliant radar field name

    """
    odim_quantity_dict = {
        'unfiltered_reflectivity': 'TH',
        'unfiltered_reflectivity_vv': 'TV',
        'reflectivity': 'DBZH',
        'reflectivity_vv': 'DBZV',
        'corrected_reflectivity': 'DBZHC',  # Non standard ODIM
        'corrected_reflectivity_vv': 'DBZVC',  # Non standard ODIM
        'corrected_unfiltered_reflectivity': 'THC',  # Non standard ODIM
        'corrected_unfiltered_reflectivity_vv': 'TVC',  # Non standard ODIM
        'reflectivity_bias': 'ZBIAS',  # Non standard ODIM
        'volumetric_reflectivity': 'etah',  # Non standard ODIM
        'volumetric_reflectivity_vv': 'etav',  # Non standard ODIM
        'radar_cross_section_hh': 'RCSH',  # Non standard ODIM
        'radar_cross_section_vv': 'RCSV', # Non standard ODIM
        'differential_reflectivity': 'ZDR',
        'unfiltered_differential_reflectivity': 'ZDRU',  # Non standard ODIM
        'corrected_differential_reflectivity': 'ZDRC',  # Non standard ODIM
        'corrected_unfiltered_differential_reflectivity': 'ZDRUC',  # Non standard ODIM
        'differential_reflectivity_in_precipitation': 'ZDRPREC', # Non standard ODIM
        'differential_reflectivity_in_snow': 'ZDRSNOW', # Non standard ODIM
        'signal_power_hh': 'DBMH',  # Non standard ODIM
        'signal_power_vv': 'DBMV',  # Non standard ODIM
        'noisedBZ_hh': 'NDBZH', # Non standard ODIM
        'noisedBZ_vv': 'NDBZV', # Non standard ODIM
        'sun_hit_power_h': 'DBM_SUNHIT', # Non standard ODIM
        'sun_hit_power_v': 'DBMV_SUNHIT', # Non standard ODIM
        'sun_hit_differential_reflectivity': 'ZDR_SUNHIT', # Non standard ODIM
        'sun_est_power_h': 'DBM_SUNEST',  # Non standard ODIM
        'sun_est_power_v': 'DBMV_SUNEST',  # Non standard ODIM
        'sun_est_differential_reflectivity': 'ZDR_SUNEST', # Non standard ODIM
        'sun_hit_h': 'POSH_SUNHIT', # Non standard ODIM
        'sun_hit_v': 'POSV_SUNHIT', # Non standard ODIM
        'sun_hit_zdr': 'POSZDR_SUNHIT', # Non standard ODIM
        'cross_correlation_ratio': 'RHOHV',
        'uncorrected_cross_correlation_ratio': 'URHOHV', # Non standard ODIM
        'corrected_cross_correlation_ratio': 'RHOHVC',  # Non standard ODIM
        'cross_correlation_ratio_in_rain': 'RHOHVRAIN', # Non standard ODIM
        'logarithmic_cross_correlation_ratio': 'LRHOHV', # Non standard ODIM
        'circular_depolarization_ratio': 'CDR',  # Non standard ODIM
        'linear_polarization_ratio': 'LDR',
        'differential_phase': 'PHIDP',
        'uncorrected_differential_phase': 'UPHIDP', # Non standard ODIM
        'corrected_differential_phase': 'PHIDPC',  # Non standard ODIM
        'system_differential_phase': 'PHIDP0',  # Non standard ODIM
        'first_gate_differential_phase': 'PHIDP0_BIN',  # Non standard ODIM
        'specific_differential_phase': 'KDP',
        'corrected_specific_differential_phase': 'KDPC',  # Non standard ODIM
        'normalized_coherent_power': 'SQIH',
        'normalized_coherent_power_vv': 'SQIV',
        'signal_to_noise_ratio_hh': 'SNRH',
        'signal_to_noise_ratio_vv': 'SNRV',
        'clutter_correction_hh': 'CCORH',  # Not used in Pyrad
        'clutter_correction_vv': 'CCORV', # Not used in Pyrad
        'radar_estimated_rain_rate': 'RATE',
        'uncorrected_rain_rate': 'URATE', # Not used in Pyrad
        'hail_intensity': 'HI',  # Not used in Pyrad
        'hail_probability': 'HP', # Not used in Pyrad
        'accumulated_precipitation': 'ACRR', # Not used in Pyrad
        'echotop_height': 'HGHT', # Not used in Pyrad
        'vertical_integrated_liquid_water': 'VIL', # Not used in Pyrad
        'velocity': 'VRADH',
        'velocity_vv': 'VRADV',
        'corrected_velocity': 'VRADHC',  # Non standard ODIM
        'dealiased_velocity': 'VRADDH',
        'dealiased_corrected_velocity': 'VRADDHC',  # Non standard ODIM
        'dealiased_velocity_vv': 'VRADDV',
        'retrieved_velocity': 'VRADEST',  # Non standard ODIM
        'retrieved_velocity_std': 'sd_vvp',  # special vol2bird
        'velocity_difference': 'VDIFF',  # Non standard ODIM
        'spectrum_width': 'WRADH',
        'corrected_spectrum_width': 'WRADHC',  # Non standard ODIM
        'spectrum_width_vv': 'WRADV',
        'eastward_wind_component': 'UWND',
        'northward_wind_component': 'VWND',
        'azimuthal_horizontal_wind_component': 'AHWND',  # Non standard ODIM
        'vertical_wind_component': 'w', # Standard for vertical profile
        'radial_wind_shear': 'RSHR',  # Not used in Pyrad
        'azimuthal_wind_shear': 'ASHR',  # Not used in Pyrad
        'range_azimuthal_wind_shear': 'CSHR',  # Not used in Pyrad
        'elevation_wind_shear': 'ESHR',  # Not used in Pyrad
        'range_elevation_wind_shear': 'OSHR',  # Not used in Pyrad
        'horizontal_wind_shear': 'HSHR',  # Not used in Pyrad
        'vertical_wind_shear': 'VSHR',
        'three_dimensional_shear': 'TSHR', # Not used in Pyrad
        'wind_speed': 'ff', # Standard for vertical profile
        'wind_direction': 'dd',  # Standard for vertical profile
        'specific_attenuation': 'AH',  # Non standard ODIM
        'corrected_specific_attenuation': 'AHC',  # Non standard ODIM
        'path_integrated_attenuation': 'PIA',  # Non standard ODIM
        'corrected_path_integrated_attenuation': 'PIAC',  # Non standard ODIM
        'specific_differential_attenuation': 'ADP',  # Non standard ODIM
        'corrected_specific_differential_attenuation': 'ADPC',  # Non standard ODIM
        'path_integrated_differential_attenuation': 'PIDA',  # Non standard ODIM
        'corrected_path_integrated_differential_attenuation': 'PIDAC',  # Non standard ODIM
        'temperature': 'TEMP',  # Non standard ODIM
        'iso0': 'ISO0',  # Non standard ODIM
        'height_over_iso0': 'HISO0',  # Non standard ODIM
        'cosmo_index': 'COSMOIND',  # Non standard ODIM
        'hzt_index': 'HZTIND',  # Non standard ODIM
        'melting_layer': 'ML',  # Non standard ODIM
        'visibility': 'VIS',  # Non standard ODIM
        'radar_echo_id': 'ECHOID',  # Non standard ODIM
        'clutter_exit_code': 'CLT',  # Non standard ODIM
        'occurrence': 'OCC',  # Non standard ODIM
        'frequency_of_occurrence': 'OCCFREQ',  # Non standard ODIM
        'radar_border': 'BRDR',  # Not used in Pyrad
        'signal_quality_index': 'QUIND',
        'radar_echo_classification': 'CLASS',
        'hydroclass_entropy': 'ENTROPY',  # Non standard ODIM
        'proportion_AG': 'propAG',  # Non standard ODIM
        'proportion_CR': 'propCR',  # Non standard ODIM
        'proportion_LR': 'propLR',  # Non standard ODIM
        'proportion_RP': 'propRP',  # Non standard ODIM
        'proportion_RN': 'propRN',  # Non standard ODIM
        'proportion_VI': 'propVI',  # Non standard ODIM
        'proportion_WS': 'propWS',  # Non standard ODIM
        'proportion_MH': 'propMH',  # Non standard ODIM
        'proportion_IH': 'propIH',  # Non standard ODIM
        'time_avg_flag': 'TAFLAG',  # Non standard ODIM
        'colocated_gates': 'COLGATES',  # Non standard ODIM
        'number_of_samples': 'ns',  # standard vertical profile
        'bird_density': 'dens',  # standard vol2bird
        'standard_deviation': 'STD',  # Non standard ODIM
        'sum': 'SUM',  # Non standard ODIM
        'sum_squared': 'SUM2',  # Non standard ODIM
        'height_resolution': 'width',  # Special vol2bird
        'gap': 'gap',  # Special vol2bird
        'bird_reflectivity': 'eta',  # Special vol2bird
        'number_of_samples_velocity': 'n',  # Special vol2bird
        'number_of_samples_reflectivity': 'n_dbz',  # Special vol2bird
        'number_of_samples_velocity_all': 'n_all',  # Special vol2bird
        'number_of_samples_reflectivity_all': 'n_dbz_all'  # Special vol2bird
    }
    if field_name not in odim_quantity_dict:
        warn('Field name '+field_name+' does not have an ODIM quantity ' +
             'equivalent. The same field name will be used.')
        return field_name

    return odim_quantity_dict[field_name]


def _get_data_from_field(radar, sweep_ind, field_name, physical=True):
    """
    Extract data from radar field object with respect to different datasets
    and datatypes.

    Parameters:
    -----------
    radar : object
        Radar object
    sweep_ind : int
        sweep index, equal to ODIM dataset index
    field_name : str
        name of field to convert
    physical : bool
        If true the data will be stored in physical units. Otherwise it will
        be converted in binary

    Returns:
    --------
    data_dict : dict
        Dictionary containing the data, gain, offset, nodata and undetect

    """
    data_ph = np.ma.asarray(radar.get_field(sweep_ind, field_name, copy=True))

    if physical:
        fill_value = radar.fields[field_name].get(
            '_FillValue', np.double(get_fillvalue()))
        if fill_value is not None:
            data = data_ph.filled(fill_value)
            nodata = fill_value
        else:
            data = np.asarray(data_ph)
            nodata = None
        undetect = None
        gain = 1.
        offset = 0.
    else:
        dtype = 'uint8'
        nvals = 256
        if '_Write_as_dtype' in radar.fields[field_name]:
            dtype = radar.fields[field_name]['_Write_as_dtype']
            if dtype not in ('uint8', 'uint16'):
                warn('Type '+dtype+' not supported. uint8 will be used')
                dtype = 'uint8'
                nvals = 256
            elif dtype == 'uint8':
                nvals = 256
            else:
                nvals = 65536

        nodata = radar.fields[field_name].get('nodata', 0)
        undetect = radar.fields[field_name].get('undetect', None)

        reserved = 0
        if nodata is not None:
            reserved = 1
        if undetect is not None:
            reserved += 1

        shift_up = 0
        if nodata == 0 or undetect == 0:
            shift_up = 1
        if nodata == 1 or undetect == 1:
            shift_up = 2

        if (('scale_factor' in radar.fields[field_name]) and
                ('add_offset' in radar.fields[field_name])):
            gain = radar.fields[field_name]['scale_factor']
            offset = radar.fields[field_name]['add_offset']

            shift_down = 0
            if nodata == nvals-1 or undetect == nvals-1:
                shift_down = 1
            if nodata == nvals-2 or undetect == nvals-2:
                shift_down = 2
            val_min = gain*shift_up+offset
            val_max = gain*(nvals-1-shift_down)+offset

            # Check if data within range
            ind = np.ma.where(data_ph < val_min)
            if ind[0].size > 0:
                warn(str(ind[0].size)+' data points below the minimum value ' +
                     str(val_min))
                data_ph[ind] = val_min
            ind = np.ma.where(data_ph > val_max)
            if ind[0].size > 0:
                warn(str(ind[0].size)+' data points above the maximum value ' +
                     str(val_max))
                print(data_ph[ind])
                data_ph[ind] = val_max
        else:
            val_min = np.ma.min(data_ph)
            val_max = np.ma.max(data_ph)
            nsteps = nvals-reserved
            gain = (val_max-val_min)/nsteps
            offset = val_min-shift_up*gain

            warn('gain and offset not specified. Optimal scale with gain ' +
                 str(gain)+' and offset '+str(offset)+' will be used')

        data = (data_ph-offset)/gain
        if nodata is not None:
            data = data.filled(nodata)
        data = np.asarray(data, dtype=eval('np.'+dtype))

    data_dict = {
        'data': data,
        'gain': gain,
        'offset': offset,
        'nodata': nodata,
        'undetect': undetect
    }
    return data_dict


def _map_radar_to_how_dict(radar_obj):
    """
    Tries to map data in a radar sub object (e.g. radar.instrument_parameters)
    to ODIM how attributes.

    Parameters:
    -----------
    radar_obj : Radar
        Containing a radar sub object

    Returns:
    --------
    dict_odim : dictionary
        Dictionary of ODIM how variables

    """
    # Variables in the instrument_parameter convention and
    # radar_parameters sub-convention that will be read from and written to
    # ODIM H5 files using Py-ART.
    # The meta_group attribute cannot be used to identify these parameters as
    # it is often set incorrectly.

    _INSTRUMENT_PARAMS = [
        #instrument parameters sub-convention
        'frequency',
        'pulse_width',
        'prt',
        'nyquist_velocity',
        'n_samples',
        # radar_parameters sub-convention
        'radar_antenna_gain_h',
        'radar_antenna_gain_v',
        'calibration_constant_hh',
        'calibration_constant_vv',
        'radar_beam_width_h',
        'radar_beam_width_v',
        'radar_receiver_bandwidth',
        'radar_rx_bandwidth'    # non-standard
        #'radar_measured_transmit_power_h', # not found in radar object
        #'radar_measured_transmit_power_v', # not found in radar object
        #'measured_transmit_power_v',    # not found in radar object
        #'measured_transmit_power_h'     # not found in radar object
        ]

    _RADAR_METADATA = [
        'system',
        'software',
        'sw_version'
        ]

    c = 299792458 #Speed of light

    dict_odim = {}

    for key in radar_obj.keys():
        if key in _INSTRUMENT_PARAMS:
            if key == 'frequency':
                dict_odim['wavelength'] = c/np.double(radar_obj[key]['data'])
            if key == 'pulse_width':
                dict_odim['pulsewidth'] = (
                    np.double(radar_obj[key]['data'])*1e6)
            if key == 'prt':
                dict_odim['highprf'] = 1 / np.double(radar_obj[key]['data'])
            if key == 'nyquist_velocity':
                dict_odim['NI'] = np.double(radar_obj[key]['data'])
            if key == 'n_samples':
                dict_odim['Vsamples'] = np.int64(radar_obj[key]['data'])
            if key == 'radar_antenna_gain_h':
                dict_odim['antgainH'] = np.double(radar_obj[key]['data'])
            if key == 'radar_antenna_gain_v':
                dict_odim['antgainV'] = np.double(radar_obj[key]['data'])
            if key == 'calibration_constant_hh':
                dict_odim['radconstH'] = np.double(radar_obj[key]['data'])
            if key == 'calibration_constant_vv':
                dict_odim['radconstV'] = np.double(radar_obj[key]['data'])
            if key == 'radar_beam_width_h':
                dict_odim['beamwH'] = np.double(radar_obj[key]['data'])
            if key == 'radar_beam_width_v':
                dict_odim['beamwV'] = np.double(radar_obj[key]['data'])
            if key in ['radar_receiver_bandwidth', 'radar_rx_bandwidth']:
                dict_odim['RXbandwidth'] = np.double(radar_obj[key]['data'])
        elif key in _RADAR_METADATA:
            dict_odim[key] = _to_str(radar_obj[key])
        else:
            warn("Unknown how parameter: %s, "%(key)+"not written to file.")

    return dict_odim
