grib2io.utils API documentation

# grib2io.utils

Collection of utility functions to assist in the encoding and decoding of GRIB2 Messages.

View Source

    """
    Collection of utility functions to assist in the encoding and decoding
    of GRIB2 Messages.
    """

    import g2clib
    import datetime
    import numpy as np
    import struct

    from .. import tables

    def int2bin(i,nbits=8,output=str):
        """
        Convert integer to binary string or list

        Parameters
        ----------

        **`i`**: Integer value to convert to binary representation.

        **`nbits`**: Number of bits to return.  Valid values are 8 [DEFAULT], 16,
        32, and 64.

        **`output`**: Return data as `str` [DEFAULT] or `list` (list of ints).

        Returns
        -------

        `str` or `list` (list of ints) of binary representation of the integer value.
        """
        i = int(i) if not isinstance(i,int) else i
        assert nbits in [8,16,32,64]
        bitstr = "{0:b}".format(i).zfill(nbits)
        if output is str:
            return bitstr
        elif output is list:
            return [int(b) for b in bitstr]

    def putieeeint(r):
        """
        Convert a float to a IEEE format 32 bit integer
        """
        ra = np.array([r],'f')
        ia = np.empty(1,'i')
        g2clib.rtoi_ieee(ra,ia)
        return ia[0]

    def getieeeint(i):
        """
        Convert an IEEE format 32 bit integer to a float
        """
        ia = np.array([i],'i')
        ra = np.empty(1,'f')
        g2clib.itor_ieee(ia,ra)
        return ra[0]

    def getmd5str(a):
        """
        Generate a MD5 hash string from input list
        """
        import hashlib
        assert isinstance(a,list) or isinstance(a,bytes)
        return hashlib.md5(''.join([str(i) for i in a]).encode()).hexdigest()

    def getdate(year,month,day,hour,minute=None,second=None):
        """
        Build an integer date from component input.

        **`year`**: Year in 4-digit format.

        **`month`**: Month in 2-digit format.

        **`day`**: Day in 2-digit format.

        **`hour`**: Hour in 2-digit format.

        **`minute`**: Minute in 2-digit format. This argument is required if second is provided, otherwise
        it is optional.

        **`second`**: Second in 2-digit format [OPTIONAL].
        """
        year_exp = 6
        month_exp = 4
        day_exp = 2
        hour_exp = 0
        #if second is not None and minute is None:
        #    raise ValueError("Must provide minute argument if second argument is provided.")
        #year_exp = 6
        #month_exp = 4
        #day_exp = 2
        #hour_exp = 0
        #minute_exp = -2
        #second_exp = -4
        #if minute is not None:
        #    assert minute >= 0 and minute <= 60
        #    year_exp += 2
        #    month_exp += 2
        #    day_exp += 2
        #    hour_exp += 2
        #    minute_exp += 2
        #    second_exp += 2
        #if second is not None:
        #    assert second >= 0 and second <= 60
        #    year_exp += 2
        #    month_exp += 2
        #    day_exp += 2
        #    hour_exp += 2
        #    minute_exp += 2
        #    second_exp += 2
        idate = (year*pow(10,year_exp))+(month*pow(10,month_exp))+\
                (day*pow(10,day_exp))+(hour*pow(10,hour_exp))
        #if minute is not None:
        #    idate += minute*pow(10,minute_exp)
        #if second is not None:
        #    idate += second*pow(10,second_exp)
        return idate

    def getleadtime(idsec,pdtn,pdt):
        """
        Computes the lead time (in units of hours) from using information from
        GRIB2 Identification Section (Section 1), Product Definition Template
        Number, and Product Definition Template (Section 4).

        Parameters
        ----------

        **`idsec`**: seqeunce containing GRIB2 Identification Section (Section 1).

        **`pdtn`**: GRIB2 Product Definition Template Number

        **`idsec`**: seqeunce containing GRIB2 Product Definition Template (Section 4).

        Returns
        -------

        **`lt`**: Lead time in units of hours
        """
        refdate = datetime.datetime(*idsec[5:11])
        if pdtn == 8:
            enddate = datetime.datetime(*pdt[15:21])
            td = enddate - refdate
            lt = (td).total_seconds()/3600.0
        elif pdtn == 9:
            enddate = datetime.datetime(*pdt[22:28])
            td = enddate - refdate
            lt = (td).total_seconds()/3600.0
        elif pdtn == 10:
            enddate = datetime.datetime(*pdt[16:22])
            td = enddate - refdate
            lt = (td).total_seconds()/3600.0
        elif pdtn == 11:
            enddate = datetime.datetime(*pdt[18:24])
            td = enddate - refdate
            lt = (td).total_seconds()/3600.0
        elif pdtn == 12:
            enddate = datetime.datetime(*pdt[17:23])
            td = enddate - refdate
            lt = (td).total_seconds()/3600.0
        else:
            lt = pdt[8]*(tables.get_value_from_table(pdt[7],'scale_time_hours'))
        return int(lt)

    def getduration(pdtn,pdt):
        """
        Computes the duration time (in units of hours) from using information from
        Product Definition Template Number, and Product Definition Template (Section 4).

        Parameters
        ----------

        **`pdtn`**: GRIB2 Product Definition Template Number

        **`pdt`**: sequence containing GRIB2 Product Definition Template (Section 4).

        Returns
        -------

        **`dur`**: Duration time in units of hours
        """
        if pdtn == 8:
            dur = pdt[26]*(tables.get_value_from_table(pdt[25],'scale_time_hours'))
        elif pdtn == 9:
            dur = pdt[32]*(tables.get_value_from_table(pdt[31],'scale_time_hours'))
        elif pdtn == 10:
            dur = pdt[27]*(tables.get_value_from_table(pdt[26],'scale_time_hours'))
        elif pdtn == 11:
            dur = pdt[29]*(tables.get_value_from_table(pdt[28],'scale_time_hours'))
        elif pdtn == 12:
            dur = pdt[28]*(tables.get_value_from_table(pdt[27],'scale_time_hours'))
        else:
            dur = 0
        return int(dur)

    def decode_mdl_wx_strings(lus):
        """
        Decode GRIB2 Local Use Section to obtain MDL Weather Strings.  The
        decode procedure is defined here:

        https://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc/grib2_mdl_temp2-1.shtml

        Parameters
        ----------

        **`lus`**: GRIB2 Local Use Section containing MDL weather strings.

        Returns
        -------

        **`list`**: List of weather strings.
        """
        assert lus[0] == 1
        # Unpack information related to the simple packing method
        # the packed weather string data.
        ngroups = struct.unpack('>h',lus[1:3])[0]
        nvalues = struct.unpack('>i',lus[3:7])[0]
        refvalue = struct.unpack('>i',lus[7:11])[0]
        dsf = struct.unpack('>h',lus[11:13])[0]
        nbits = lus[13]
        datatype = lus[14]
        if datatype == 0: # Floating point
            refvalue = np.float32(getieeeint(refvalue)*10**-dsf)
        elif datatype == 1: # Integer
            refvalue = np.int32(getieeeint(refvalue)*10**-dsf)
        #print("TEST:",ngroups,nvalues,refvalue,dsf,nbits,datatype)
        # Store the "data" part of the packed weather strings as
        # a binary string.
        b = bin(int.from_bytes(lus[15:],byteorder='big'))
        # Generated begin and end values. Note the offset begins at 2
        # due to the format nature of b (binary string).
        idxb = list(range(2,len(b),nbits))[0:nvalues-1]
        idxe = list(range(2+nbits,len(b)+nbits,nbits))[0:nvalues-1]
        # Iterate over the packed data, converting the nbits space to
        # and integer, then convert integer to an ASCII character.
        wxstring = ''
        for ib,ie in zip(idxb,idxe):
            wxstring += chr(int(b[ib:ie],2)+refvalue)
        # Return string as list, split by null character.
        return wxstring.split('\0')

    def decode_ndfd_wx_strings(lus):
        """
        Decode GRIB2 Local Use Section to obtain NDFD Weather Strings.  The
        decode procedure is defined here:

        https://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc/grib2_mdl_temp2-1.shtml

        NOTE: From testing, it seems how the process for packing Section 2 for NDFD
              Weather grids is different than GMOS and thus the need for a different
              method for decoding NDFD Wx vs. GMOS Wx.

        Parameters
        ----------

        **`lus`**: GRIB2 Local Use Section containing NDFD weather strings.

        Returns
        -------

        **`list`**: List of NDFD weather strings.
        """
        assert lus[0] == 1
        # Unpack information related to the simple packing method
        # the packed weather string data.
        ngroups = struct.unpack('>h',lus[1:3])[0]
        nvalues = struct.unpack('>i',lus[3:7])[0]
        refvalue = struct.unpack('>i',lus[7:11])[0]
        dsf = struct.unpack('>h',lus[11:13])[0]
        nbits = lus[13]
        datatype = lus[14]
        if datatype == 0: # Floating point
            refvalue = np.float32(getieeeint(refvalue)*10**-dsf)
        elif datatype == 1: # Integer
            refvalue = np.int32(getieeeint(refvalue)*10**-dsf)
        # Iterate over the data of section 2 in 4-byte (32-bit)
        # words OR 2-byte word at the end. Each word is unpacked
        # as an unsigned integer. Then convert the integer word
        # to a binary string and fill to the appropriate bit
        # size (16 or 32) and concatenate to b (binary string).
        b = ''
        for i in range(15,len(lus),4):
            if len(lus)-i < 4:
                iword = struct.unpack('>H',lus[i:i+2])[0]
                b += bin(iword).split('b')[1].zfill(16)
            else:
                iword = struct.unpack('>I',lus[i:i+4])[0]
                b += bin(iword).split('b')[1].zfill(32)
        # Iterate over the binary string (b). For each nbits
        # chunk, convert to an integer, including the refvalue,
        # and then convert the int to an ASCII character, then
        # concatenate to wxstring.
        wxstring = ''
        for i in range(0,len(b),nbits):
            wxstring += chr(int(b[i:i+nbits],2)+refvalue)
        # Return string as list, split by null character.
        return list(filter(None,wxstring.split('\0')))

def int2bin(i, nbits=8, output=<class 'str'>):
View Source

    def int2bin(i,nbits=8,output=str):
        """
        Convert integer to binary string or list

        Parameters
        ----------

        **`i`**: Integer value to convert to binary representation.

        **`nbits`**: Number of bits to return.  Valid values are 8 [DEFAULT], 16,
        32, and 64.

        **`output`**: Return data as `str` [DEFAULT] or `list` (list of ints).

        Returns
        -------

        `str` or `list` (list of ints) of binary representation of the integer value.
        """
        i = int(i) if not isinstance(i,int) else i
        assert nbits in [8,16,32,64]
        bitstr = "{0:b}".format(i).zfill(nbits)
        if output is str:
            return bitstr
        elif output is list:
            return [int(b) for b in bitstr]

Convert integer to binary string or list

## Parameters

`i`: Integer value to convert to binary representation.

`nbits`: Number of bits to return. Valid values are 8 [DEFAULT], 16, 32, and 64.

`output`: Return data as `str` [DEFAULT] or `list` (list of ints).

## Returns

`str` or `list` (list of ints) of binary representation of the integer value.

def putieeeint(r):
View Source

    def putieeeint(r):
        """
        Convert a float to a IEEE format 32 bit integer
        """
        ra = np.array([r],'f')
        ia = np.empty(1,'i')
        g2clib.rtoi_ieee(ra,ia)
        return ia[0]

Convert a float to a IEEE format 32 bit integer

def getieeeint(i):
View Source

    def getieeeint(i):
        """
        Convert an IEEE format 32 bit integer to a float
        """
        ia = np.array([i],'i')
        ra = np.empty(1,'f')
        g2clib.itor_ieee(ia,ra)
        return ra[0]

Convert an IEEE format 32 bit integer to a float

def getmd5str(a):
View Source

    def getmd5str(a):
        """
        Generate a MD5 hash string from input list
        """
        import hashlib
        assert isinstance(a,list) or isinstance(a,bytes)
        return hashlib.md5(''.join([str(i) for i in a]).encode()).hexdigest()

Generate a MD5 hash string from input list

def getdate(year, month, day, hour, minute=None, second=None):
View Source

    def getdate(year,month,day,hour,minute=None,second=None):
        """
        Build an integer date from component input.

        **`year`**: Year in 4-digit format.

        **`month`**: Month in 2-digit format.

        **`day`**: Day in 2-digit format.

        **`hour`**: Hour in 2-digit format.

        **`minute`**: Minute in 2-digit format. This argument is required if second is provided, otherwise
        it is optional.

        **`second`**: Second in 2-digit format [OPTIONAL].
        """
        year_exp = 6
        month_exp = 4
        day_exp = 2
        hour_exp = 0
        #if second is not None and minute is None:
        #    raise ValueError("Must provide minute argument if second argument is provided.")
        #year_exp = 6
        #month_exp = 4
        #day_exp = 2
        #hour_exp = 0
        #minute_exp = -2
        #second_exp = -4
        #if minute is not None:
        #    assert minute >= 0 and minute <= 60
        #    year_exp += 2
        #    month_exp += 2
        #    day_exp += 2
        #    hour_exp += 2
        #    minute_exp += 2
        #    second_exp += 2
        #if second is not None:
        #    assert second >= 0 and second <= 60
        #    year_exp += 2
        #    month_exp += 2
        #    day_exp += 2
        #    hour_exp += 2
        #    minute_exp += 2
        #    second_exp += 2
        idate = (year*pow(10,year_exp))+(month*pow(10,month_exp))+\
                (day*pow(10,day_exp))+(hour*pow(10,hour_exp))
        #if minute is not None:
        #    idate += minute*pow(10,minute_exp)
        #if second is not None:
        #    idate += second*pow(10,second_exp)
        return idate

Build an integer date from component input.

`year`: Year in 4-digit format.

`month`: Month in 2-digit format.

`day`: Day in 2-digit format.

`hour`: Hour in 2-digit format.

`minute`: Minute in 2-digit format. This argument is required if second is provided, otherwise it is optional.

`second`: Second in 2-digit format [OPTIONAL].

def getleadtime(idsec, pdtn, pdt):
View Source

    def getleadtime(idsec,pdtn,pdt):
        """
        Computes the lead time (in units of hours) from using information from
        GRIB2 Identification Section (Section 1), Product Definition Template
        Number, and Product Definition Template (Section 4).

        Parameters
        ----------

        **`idsec`**: seqeunce containing GRIB2 Identification Section (Section 1).

        **`pdtn`**: GRIB2 Product Definition Template Number

        **`idsec`**: seqeunce containing GRIB2 Product Definition Template (Section 4).

        Returns
        -------

        **`lt`**: Lead time in units of hours
        """
        refdate = datetime.datetime(*idsec[5:11])
        if pdtn == 8:
            enddate = datetime.datetime(*pdt[15:21])
            td = enddate - refdate
            lt = (td).total_seconds()/3600.0
        elif pdtn == 9:
            enddate = datetime.datetime(*pdt[22:28])
            td = enddate - refdate
            lt = (td).total_seconds()/3600.0
        elif pdtn == 10:
            enddate = datetime.datetime(*pdt[16:22])
            td = enddate - refdate
            lt = (td).total_seconds()/3600.0
        elif pdtn == 11:
            enddate = datetime.datetime(*pdt[18:24])
            td = enddate - refdate
            lt = (td).total_seconds()/3600.0
        elif pdtn == 12:
            enddate = datetime.datetime(*pdt[17:23])
            td = enddate - refdate
            lt = (td).total_seconds()/3600.0
        else:
            lt = pdt[8]*(tables.get_value_from_table(pdt[7],'scale_time_hours'))
        return int(lt)

Computes the lead time (in units of hours) from using information from GRIB2 Identification Section (Section 1), Product Definition Template Number, and Product Definition Template (Section 4).

## Parameters

`idsec`: seqeunce containing GRIB2 Identification Section (Section 1).

`pdtn`: GRIB2 Product Definition Template Number

`idsec`: seqeunce containing GRIB2 Product Definition Template (Section 4).

## Returns

`lt`: Lead time in units of hours

def getduration(pdtn, pdt):
View Source

    def getduration(pdtn,pdt):
        """
        Computes the duration time (in units of hours) from using information from
        Product Definition Template Number, and Product Definition Template (Section 4).

        Parameters
        ----------

        **`pdtn`**: GRIB2 Product Definition Template Number

        **`pdt`**: sequence containing GRIB2 Product Definition Template (Section 4).

        Returns
        -------

        **`dur`**: Duration time in units of hours
        """
        if pdtn == 8:
            dur = pdt[26]*(tables.get_value_from_table(pdt[25],'scale_time_hours'))
        elif pdtn == 9:
            dur = pdt[32]*(tables.get_value_from_table(pdt[31],'scale_time_hours'))
        elif pdtn == 10:
            dur = pdt[27]*(tables.get_value_from_table(pdt[26],'scale_time_hours'))
        elif pdtn == 11:
            dur = pdt[29]*(tables.get_value_from_table(pdt[28],'scale_time_hours'))
        elif pdtn == 12:
            dur = pdt[28]*(tables.get_value_from_table(pdt[27],'scale_time_hours'))
        else:
            dur = 0
        return int(dur)

Computes the duration time (in units of hours) from using information from Product Definition Template Number, and Product Definition Template (Section 4).

## Parameters

`pdtn`: GRIB2 Product Definition Template Number

`pdt`: sequence containing GRIB2 Product Definition Template (Section 4).

## Returns

`dur`: Duration time in units of hours

def decode_mdl_wx_strings(lus):
View Source

    def decode_mdl_wx_strings(lus):
        """
        Decode GRIB2 Local Use Section to obtain MDL Weather Strings.  The
        decode procedure is defined here:

        https://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc/grib2_mdl_temp2-1.shtml

        Parameters
        ----------

        **`lus`**: GRIB2 Local Use Section containing MDL weather strings.

        Returns
        -------

        **`list`**: List of weather strings.
        """
        assert lus[0] == 1
        # Unpack information related to the simple packing method
        # the packed weather string data.
        ngroups = struct.unpack('>h',lus[1:3])[0]
        nvalues = struct.unpack('>i',lus[3:7])[0]
        refvalue = struct.unpack('>i',lus[7:11])[0]
        dsf = struct.unpack('>h',lus[11:13])[0]
        nbits = lus[13]
        datatype = lus[14]
        if datatype == 0: # Floating point
            refvalue = np.float32(getieeeint(refvalue)*10**-dsf)
        elif datatype == 1: # Integer
            refvalue = np.int32(getieeeint(refvalue)*10**-dsf)
        #print("TEST:",ngroups,nvalues,refvalue,dsf,nbits,datatype)
        # Store the "data" part of the packed weather strings as
        # a binary string.
        b = bin(int.from_bytes(lus[15:],byteorder='big'))
        # Generated begin and end values. Note the offset begins at 2
        # due to the format nature of b (binary string).
        idxb = list(range(2,len(b),nbits))[0:nvalues-1]
        idxe = list(range(2+nbits,len(b)+nbits,nbits))[0:nvalues-1]
        # Iterate over the packed data, converting the nbits space to
        # and integer, then convert integer to an ASCII character.
        wxstring = ''
        for ib,ie in zip(idxb,idxe):
            wxstring += chr(int(b[ib:ie],2)+refvalue)
        # Return string as list, split by null character.
        return wxstring.split('\0')

Decode GRIB2 Local Use Section to obtain MDL Weather Strings. The decode procedure is defined here:

https://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc/grib2_mdl_temp2-1.shtml

## Parameters

`lus`: GRIB2 Local Use Section containing MDL weather strings.

## Returns

`list`: List of weather strings.

def decode_ndfd_wx_strings(lus):
View Source

    def decode_ndfd_wx_strings(lus):
        """
        Decode GRIB2 Local Use Section to obtain NDFD Weather Strings.  The
        decode procedure is defined here:

        https://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc/grib2_mdl_temp2-1.shtml

        NOTE: From testing, it seems how the process for packing Section 2 for NDFD
              Weather grids is different than GMOS and thus the need for a different
              method for decoding NDFD Wx vs. GMOS Wx.

        Parameters
        ----------

        **`lus`**: GRIB2 Local Use Section containing NDFD weather strings.

        Returns
        -------

        **`list`**: List of NDFD weather strings.
        """
        assert lus[0] == 1
        # Unpack information related to the simple packing method
        # the packed weather string data.
        ngroups = struct.unpack('>h',lus[1:3])[0]
        nvalues = struct.unpack('>i',lus[3:7])[0]
        refvalue = struct.unpack('>i',lus[7:11])[0]
        dsf = struct.unpack('>h',lus[11:13])[0]
        nbits = lus[13]
        datatype = lus[14]
        if datatype == 0: # Floating point
            refvalue = np.float32(getieeeint(refvalue)*10**-dsf)
        elif datatype == 1: # Integer
            refvalue = np.int32(getieeeint(refvalue)*10**-dsf)
        # Iterate over the data of section 2 in 4-byte (32-bit)
        # words OR 2-byte word at the end. Each word is unpacked
        # as an unsigned integer. Then convert the integer word
        # to a binary string and fill to the appropriate bit
        # size (16 or 32) and concatenate to b (binary string).
        b = ''
        for i in range(15,len(lus),4):
            if len(lus)-i < 4:
                iword = struct.unpack('>H',lus[i:i+2])[0]
                b += bin(iword).split('b')[1].zfill(16)
            else:
                iword = struct.unpack('>I',lus[i:i+4])[0]
                b += bin(iword).split('b')[1].zfill(32)
        # Iterate over the binary string (b). For each nbits
        # chunk, convert to an integer, including the refvalue,
        # and then convert the int to an ASCII character, then
        # concatenate to wxstring.
        wxstring = ''
        for i in range(0,len(b),nbits):
            wxstring += chr(int(b[i:i+nbits],2)+refvalue)
        # Return string as list, split by null character.
        return list(filter(None,wxstring.split('\0')))

Decode GRIB2 Local Use Section to obtain NDFD Weather Strings. The decode procedure is defined here:

https://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc/grib2_mdl_temp2-1.shtml

NOTE: From testing, it seems how the process for packing Section 2 for NDFD Weather grids is different than GMOS and thus the need for a different method for decoding NDFD Wx vs. GMOS Wx.

## Parameters

`lus`: GRIB2 Local Use Section containing NDFD weather strings.

## Returns

`list`: List of NDFD weather strings.