pyart.retrieve.kdp_proc — pyart-mch 0.4.1 documentation

# Source code for pyart.retrieve.kdp_proc

    """
    pyart.retrieve.kdp_proc
    =======================

    Module for retrieving specific differential phase (KDP) from radar total
    differential phase (PSIDP) measurements. Total differential phase is a function
    of propagation differential phase (PHIDP), backscatter differential phase
    (DELTAHV), and the system phase offset.

    .. autosummary::
        :toctree: generated/

        kdp_schneebeli
        kdp_vulpiani
        kdp_maesaka
        filter_psidp
        boundary_conditions_maesaka
        _kdp_estimation_backward_fixed
        _kdp_estimation_forward_fixed
        _kdp_kalman_profile
        _kdp_vulpiani_profile
        _cost_maesaka
        _jac_maesaka
        _forward_reverse_phidp
        _parse_range_resolution
        kdp_leastsquare_single_window
        kdp_leastsquare_double_window
        leastsquare_method
        leastsquare_method_scan

    """

    from functools import partial
    import time
    import warnings

    import numpy as np
    from scipy import optimize, stats, interpolate, linalg, signal

    from . import _kdp_proc
    from ..config import get_field_name, get_metadata, get_fillvalue
    from ..util import rolling_window

    # Constants in the Kalman filter retrieval method (generally no need to
    # modify them)
    SCALERS = [0.1, 10**(-0.8), 10**(-0.6), 10**(-0.4), 10**(-0.2), 1,
               10**(0.2), 10**(0.4), 10**(0.6), 10**(0.8), 1, 10]

    PADDING = 50  # Noise padding of the psidp signal (before and after signal)
    SHIFT = 13  # Shifting of the final signal

    def kdp_schneebeli(radar, gatefilter=None, fill_value=None, psidp_field=None,
                       kdp_field=None, phidp_field=None, band='C', rcov=0, pcov=0,
                       prefilter_psidp=False, filter_opt=None, parallel=True):
        """
        Estimates Kdp with the Kalman filter method by Schneebeli and al. (2014)
        for a set of psidp measurements.

        Parameters
        ----------
        radar : Radar
            Radar containing differential phase field.
        gatefilter : GateFilter, optional
            A GateFilter indicating radar gates that should be excluded when
            analysing differential phase measurements.
        fill_value : float, optional
            Value indicating missing or bad data in differential phase field, if
            not specified, the default in the Py-ART configuration file will be
            used.
        psidp_field : str, optional
            Total differential phase field. If None, the default field name must be
            specified in the Py-ART configuration file.
        kdp_field : str, optional
            Specific differential phase field. If None, the default field name must
            be specified in the Py-ART configuration file.
        phidp_field : str, optional
            Propagation differential phase field. If None, the default field name
            must be specified in the Py-ART configuration file.
        band : char, optional
           Radar frequency band string. Accepted "X", "C", "S" (capital
           or not). The band is used to compute intercepts -c and slope b of the
           delta = b*Kdp+c relation.
        rcov : 3x3 float array, optional
            Measurement error covariance matrix.
        pcov : 4x4 float array, optional
            Scaled state transition error covariance matrix.
        prefilter_psidp : bool, optional
            If set, the psidp measurements will first be filtered with the
            filter_psidp method, which can improve the quality of the final Kdp.
        filter_opt : dict, optional
            The arguments for the prefilter_psidp method, if empty, the defaults
            arguments of this method will be used.
        parallel : bool, optional
            Flag to enable parallel computation (one core for every psidp profile).

        Returns
        -------
        kdp_dict : dict
            Retrieved specific differential phase data and metadata.
        kdp_std_dict : dict
            Estimated specific differential phase standard dev. data and metadata.
        phidpr_dict,: dict
            Retrieved differential phase data and metadata.

        References
        ----------
        Schneebeli, M., Grazioli, J., and Berne, A.: Improved Estimation
        of the Specific Differential Phase SHIFT Using a Compilation of
        Kalman Filter Ensembles, IEEE T. Geosci. Remote Sens., 52,
        5137-5149, doi:10.1109/TGRS.2013.2287017, 2014.

        """

        # create parallel computing instance
        if parallel:
            import multiprocessing as mp

            pool = mp.Pool(processes=mp.cpu_count(), maxtasksperchild=1)

        # parse fill value
        if fill_value is None:
            fill_value = get_fillvalue()

        # parse field names
        if psidp_field is None:
            psidp_field = get_field_name('differential_phase')
        if kdp_field is None:
            kdp_field = get_field_name('specific_differential_phase')
        if phidp_field is None:
            phidp_field = get_field_name('differential_phase')

        # parse range resolution, length scale and low-pass filter constraint
        # weight
        dr = _parse_range_resolution(radar, check_uniform=True)

        # parse total differential phase measurements
        if prefilter_psidp:
            if filter_opt is None:
                filter_opt = {}
            # Assign psidp field to filter_psidp inputs
            filter_opt['psidp_field'] = psidp_field
            # Filter psidp
            psidp_o = filter_psidp(radar, **filter_opt)
        else:
            psidp_o = radar.fields[psidp_field]['data']

        # mask radar gates indicated by the gate filter
        if gatefilter is not None:
            psidp_o = np.ma.masked_where(gatefilter.gate_excluded, psidp_o)

        func = partial(_kdp_kalman_profile, dr=dr, band=band, rcov=rcov,
                       pcov=pcov)

        all_psidp_prof = list(psidp_o)

        if parallel:
            list_est = pool.map(func, all_psidp_prof)
        else:
            list_est = map(func, all_psidp_prof)

        kdp = np.zeros(psidp_o.shape) * np.nan
        kdp = np.ma.masked_array(kdp, fill_value=fill_value)

        kdp_stdev = np.zeros(psidp_o.shape) * np.nan
        kdp_stdev = np.ma.masked_array(kdp_stdev, fill_value=fill_value)

        phidp_rec = np.zeros(psidp_o.shape) * np.nan
        phidp_rec = np.ma.masked_array(phidp_rec, fill_value=fill_value)

        for i, l in enumerate(list_est):
            kdp[i, 0:len(l[0])] = l[0]
            kdp_stdev[i, 0:len(l[1])] = l[1]
            phidp_rec[i, 0:len(l[2])] = l[2]

        # Mask the estimated Kdp and reconstructed Phidp with the mask of original
        # psidp
        if isinstance(psidp_o, np.ma.masked_array):
            masked = psidp_o.mask
            kdp = np.ma.array(kdp, mask=masked, fill_value=fill_value)
            kdp_stdev = np.ma.array(kdp_stdev, mask=masked, fill_value=fill_value)
            phidp_rec = np.ma.array(phidp_rec, mask=masked, fill_value=fill_value)

        # create specific differential phase field dictionary and store data
        kdp_dict = get_metadata(kdp_field)
        kdp_dict['data'] = kdp
        # kdp_dict['valid_min'] = -1.0

        # create reconstructed differential phase field dictionary and store data
        phidpr_dict = get_metadata(phidp_field)
        phidpr_dict['data'] = phidp_rec
        # phidpr_dict['valid_min'] = 0.0

        # create specific phase stdev field dictionary and store data
        kdp_stdev_dict = {}
        kdp_stdev_dict['units'] = 'degrees/km'
        kdp_stdev_dict['standard_name'] = 'estimated KDP stdev'
        kdp_stdev_dict['long_name'] = 'Estimated stdev of spec. diff. phase (KDP)'
        kdp_stdev_dict['coordinates'] = 'elevation azimuth range'
        kdp_stdev_dict['data'] = kdp_stdev
        kdp_stdev_dict['valid_min'] = 0.0

        if parallel:
            pool.close()

        return kdp_dict, kdp_stdev_dict, phidpr_dict

    def _kdp_estimation_backward_fixed(
            psidp_in, rcov, pcov_scale, f, f_transposed, h_plus,
            c1, c2, b1, b2, kdp_th, mpsidp):
        """
        Processing one profile of Psidp and estimating Kdp and Phidp
        with the KFE algorithm described in Schneebeli et al, 2014
        IEEE_TGRS. This routine estimates Kdp in the backward
        direction given a set of matrices that define the Kalman
        filter.

        Parameters
        ----------
        psidp_in : ndarray
            One-dimensional vector of length -nrg- containining the input psidp
            [degrees].
        rcov : 3x3 float array
            Measurement error covariance matrix.
        pcov_scale : 4x4 float array
            Scaled state transition error covariance matrix.
        f : 4x4 float array
            Forward state prediction matrix [4x4].
        f_transposed : 4x4 float array
            Transpose of F.
        h_plus : 4x3 float array
            Measurement prediction matrix [4x3].
        c1, c2, b1, b2 : floats
            The values of the intercept of the relation c = b*Kdp - delta.
            This relation uses b1, c1 IF kdp is lower than a kdp_th and b2, c2
            otherwise kdp_th.
        kdp_th : float
            The kdp threshold which separates the two Kdp - delta regime
            i.e. the power law relating delta to Kdp will be different if Kdp is
            larger or smaller than kdp_th.
        mpsidp : float
            Final observed value of psidp along the radial (usually also
            the max value), needed for inverting the psidp vector.

        Returns
        -------
        kdp : ndarray
            Filtered Kdp [degrees/km]. Same length as Psidp.
        error_kdp : ndarray
            Estimated error on Kdp values.

        """

        # Define the input
        psidp = psidp_in

        # invert Psidp (backward estimation)
        psidp = mpsidp - psidp[::-1]
        nrg_new = len(psidp)

        # Initialize the state vector to 0
        s = np.zeros([4, 1])  # first state estimate

        # define measurement vector
        z = np.zeros([3, 1])

        # Define the identity matrix
        identity_i = np.eye(4)
        p = identity_i * 4.

        kdp = np.zeros([nrg_new])
        kdp_error = np.zeros([nrg_new])

        # Loop on all the gates and apply the filter

        for ii in range(0, nrg_new - 1):
            z[0] = psidp[ii]
            z[1] = psidp[ii + 1]

            s_pred = np.dot(f, s)  # state prediciton

            p_pred = np.dot(f, np.dot(p, f_transposed)) + \
                pcov_scale  # error prediction

            if s_pred[0] > kdp_th:
                h_plus[2, 0] = b2
                z[2] = c2
            else:
                h_plus[2, 0] = b1
                z[2] = c1

            # as far as i see aludc is symmetrical, so i do not transpose it
            aludc = (np.dot(h_plus, (np.dot(p_pred, h_plus.T))) + rcov)

            # below we get the transposed of B_mat directly
            b_mat = np.dot(h_plus, p_pred)

            # Cholesky decomposition
            cho = linalg.cho_factor(aludc)
            k = linalg.cho_solve(cho, b_mat, check_finite=False,
                                 overwrite_b=True).T

            # Update state and error
            s = np.dot(k, (np.dot(-h_plus, s_pred) + z)) + s_pred
            p = np.dot((identity_i - np.dot(k, h_plus)), p_pred)

            # Fill the output
            kdp[ii] = s[0]
            kdp_error[ii] = p[0, 0]

        # Shift
        n = len(kdp)
        dummy = np.copy(kdp)
        kdp[np.arange(SHIFT) + len(kdp) - SHIFT] = 0
        kdp[np.arange(n - 1 - SHIFT)] = dummy[np.arange(n - 1 - SHIFT) + SHIFT]

        # Reverse the estimates (backward direction)
        kdp = kdp[::-1]

        kdp_error = kdp_error[::-1]
        return kdp, kdp_error

    def _kdp_estimation_forward_fixed(
            psidp_in, rcov, pcov_scale, f, f_transposed, h_plus,
            c1, c2, b1, b2, kdp_th):
        """
        Processing one profile of Psidp and estimating Kdp and Phidp
        with the KFE algorithm described in Schneebeli et al, 2014
        IEEE_TGRS. This routine estimates Kdp in the forward
        direction given a set of matrices that define the Kalman
        filter.

        Parameters
        ----------
        psidp_in : ndarray
            One-dimensional vector of length -nrg- containining the input psidp
            [degrees].
        rcov : 3x3 float array.
            Measurement error covariance matrix.
        pcov_scale  : 4x4 float array
            Scaled state transition error covariance matrix.
        f : 4x4 float array
            Forward state prediction matrix [4x4].
        f_transposed : 4x4 float array
            Transpose of F.
        h_plus : 4x3 float array*np.nan
            Measurement prediction matrix [4x3].
        c1, c2, b1, b2 : floats
            The values of the intercept of the relation c = b*Kdp - delta.
            This relation uses b1, c1 IF kdp is lower than a kdp_th and b2, c2
            otherwise kdp_th.

        Returns
        -------
        kdp : ndarray
            Filtered Kdp [degrees/km]. Same length as Psidp.
        phidp : ndarray
            Estimated phidp (smooth psidp).
        error_kdp : ndarray
            Estimated error on Kdp values.

        """

        # Define the input
        psidp = np.ma.filled(psidp_in)
        nrg_new = len(psidp)

        # Initialize the state vector to 0
        s = np.zeros([4, 1])  # first state estimate

        # define measurement vector
        z = np.zeros([3, 1])

        # Define the identity matrix
        identity_i = np.eye(4)
        p = identity_i * 4.

        phidp = np.zeros([nrg_new])
        kdp = np.zeros([nrg_new])
        kdp_error = np.zeros([nrg_new])

        # Loop on all the gates and apply the filter
        for ii in range(0, nrg_new - 1):
            z[0] = psidp[ii]
            z[1] = psidp[ii + 1]

            s_pred = np.dot(f, s)  # state prediciton

            p_pred = np.dot(f, np.dot(p, f_transposed)) + \
                pcov_scale  # error prediction

            if s_pred[0] > kdp_th:
                h_plus[2, 0] = b2
                z[2] = c2
            else:
                h_plus[2, 0] = b1
                z[2] = c1

            # as far as i see aludc is symmetrical, so i do not transpose it
            aludc = (np.dot(h_plus, (np.dot(p_pred, h_plus.T))) + rcov)

            # below we get the transposed of B_mat directly
            b_mat = np.dot(h_plus, p_pred)

            # Cholesky decomposition
            cho = linalg.cho_factor(aludc)
            k = linalg.cho_solve(
                cho,
                b_mat,
                check_finite=False,
                overwrite_b=True).T

            # Update state and error
            s = np.dot(k, (np.dot(-h_plus, s_pred) + z)) + s_pred
            p = np.dot((identity_i - np.dot(k, h_plus)), p_pred)

            # Fill the output
            kdp[ii] = s[0]
            kdp_error[ii] = p[0, 0]
            phidp[ii] = s[2]

        # Shift
        dummy = np.copy(kdp)
        kdp[np.arange(SHIFT) + len(kdp) - SHIFT] = 0
        kdp[np.arange(len(kdp) - SHIFT)] = (
            dummy[np.arange(len(kdp) - SHIFT) + SHIFT])

        return kdp, phidp, kdp_error

    def _kdp_kalman_profile(psidp_in, dr, band='X', rcov=0, pcov=0):
        """
        Estimates Kdp with the Kalman filter method by Schneebeli and al. (2014)
        for a set of psidp measurements.

        Parameters
        ----------
        psidp_in : ndarray
            One-dimensional vector of length -nrg- containining the input psidp
            [degrees].
        dr : float
            Range resolution in meters.
        band : char, optional
           Radar frequency band string. Accepted "X", "C", "S" (capital
           or not). The band is used to compute intercepts -c and slope b of the
           delta = b*Kdp+c relation.
        rcov : 3x3 float array, optional
            Measurement error covariance matrix.
        pcov  : 4x4 float array, optional
            Scaled state transition error covariance matrix.

        Returns
        -------
        kdp_dict : ndarray
            Retrieved specific differential phase data.
        kdp_std_dict : ndarray
            Estimated specific differential phase standard dev. data.
        phidpr_dict : ndarray
            Retrieved differential phase data.

        References
        ----------
        Schneebeli, M., Grazioli, J., and Berne, A.: Improved Estimation
        of the Specific Differential Phase Shift Using a Compilation of
        Kalman Filter Ensembles, IEEE T. Geosci. Remote Sens., 52,
        5137-5149, doi:10.1109/TGRS.2013.2287017, 2014.

        """

        dr = dr / 1000.  # Convert rad. res. to km

        # NOTE! Parameters are not checked to save as much time as possible

        # Replace missing values with nans
        psidp_in = np.ma.filled(psidp_in, np.nan)
        # Check if psidp has at least one finite value
        if not np.isfinite(psidp_in).any():
            return psidp_in, psidp_in, psidp_in  # Return the NaNs...

        # Set default of the error covariance matrices
        if not isinstance(pcov, np.ndarray):
            pcov = np.array([[(0.11 + 1.56 * dr)**2,
                              (0.11 + 1.85 * dr)**2,
                              0,
                              (0.01 + 1.1 * dr)**2],
                             [(0.11 + 1.85 * dr)**2,
                              (0.18 + 3.03 * dr)**2,
                              0,
                              (0.01 + 1.23 * dr)**2],
                             [0,
                              0,
                              0,
                              0],
                             [(0.01 + 1.1 * dr)**2,
                              (0.01 + 1.23 * dr)**2,
                              0,
                              (-0.04 + 1.27 * dr)**2]])

        if not isinstance(rcov, np.ndarray):
            rcov = np.array([[4.10625, -0.0498779, -0.0634192],
                             [-0.0498779, 4.02369, -0.0421455],
                             [-0.0634192, -0.0421455, 1.44300]])

        # Define default parameters
        # Intercepts -c and slope b of delta=b*Kdp+c
        # According to the Kdp threshold selected

        if band == 'X':
            c1 = -0.054
            c2 = -6.155
            b1 = 2.3688
            b2 = 0.2734
            kdp_th = 2.5
        elif band == 'C':
            c1 = -0.036
            c2 = -1.03
            b1 = 0.53
            b2 = 0.15
            kdp_th = 2.5
        elif band == 'S':
            c1 = -0.024
            c2 = -0.15
            b1 = 0.19
            b2 = 0.019
            kdp_th = 1.1

        # Parameters for the final selection from the KF ensemble members
        fac1 = 1.2
        fac2 = 3.

        th1_comp = -0.15
        th2_comp = 0.15
        th1_final = -0.25

        # Kalman matrices
        # State matrix
        f = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 0, 1],
                      [2 * dr, 0, 0, 1]], dtype=float)
        f_transposed = f.T

        # Measurement prediction matrix--------------------------------
        # J. Grazioli modification 07.2015 --previous H_plus buggy--
        h_plus = np.array(
            [[-2 * dr, 1, 0, 1], [2 * dr, 1, 1, 0], [0, -1, 0, 0]], dtype=float)

        # Define the input
        psidp = psidp_in
        # Get indices of finite data
        real_data_ind = np.where(np.isfinite(psidp.ravel()))[0]
        offset = real_data_ind[0]

        if len(real_data_ind):
            mpsidp = psidp.ravel()[real_data_ind[-1]]
        else:
            mpsidp = np.nan

        psidp = psidp[offset:real_data_ind[-1] + 1]

        nrg = len(psidp)

        # Define the output
        kdp_filter_out = np.zeros([nrg, ])

        kdp_mat = np.zeros([nrg, 2 * len(SCALERS)])
        kdp_sim = np.zeros([nrg, len(SCALERS)])
        phidp_filter_out = np.zeros([nrg])

        # Prepare a longer array with some extra gates on each side
        # add values at the beginning and at the end of the profile
        psidp_long = np.zeros([nrg + PADDING * 2, ]) * np.nan

        nn = nrg + PADDING * 2

        noise = 2 * np.random.randn(PADDING)
        psidp_long[0:PADDING] = noise + psidp[0]
        psidp_long[nrg + PADDING: nrg + 2 * PADDING] = mpsidp + noise
        psidp_long[PADDING:nrg + PADDING] = psidp

        psidp = psidp_long

        # Get information of valid and non valid points in psidp the new psidp
        nonan = np.where(np.isfinite(psidp))[0]
        nan = np.where(np.isnan(psidp))[0]

        ranged = np.arange(0, nn)

        psidp_interp = psidp

        # interpolate
        if len(nan):
            interp = interpolate.interp1d(ranged[nonan], psidp[nonan], kind='zero')
            psidp_interp[nan] = interp(ranged[nan])
        else:
            psidp_interp = psidp

        # add noise
        if len(nan):
            psidp_interp[nan] = psidp_interp[nan] + 2 * np.random.randn(len(nan))

        # Define the final input and output
        psidp = psidp_interp

        # Smallest scaler
        scaler = 10 ** (-2.)

        # Backward
        kdp_dummy_b2, _ = _kdp_estimation_backward_fixed(psidp, rcov,
                                                         pcov * scaler, f,
                                                         f_transposed, h_plus,
                                                         c1, c2, b1, b2,
                                                         kdp_th, mpsidp)
        kdp002 = kdp_dummy_b2[PADDING:nrg + PADDING]

        # Forward
        kdp_dummy_b2, _, _ = _kdp_estimation_forward_fixed(psidp, rcov,
                                                           pcov * scaler, f,
                                                           f_transposed, h_plus,
                                                           c1, c2, b1, b2, kdp_th)
        kdp002f = kdp_dummy_b2[PADDING:nrg + PADDING]

        # Generate the ensemble of Kalman filters estimates in backward and
        # Forward directions
        for i, sc in enumerate(SCALERS):  # Loop on scalers
            # Forward
            kdp_dummy_f2, _, _ = _kdp_estimation_forward_fixed(psidp,
                                                               rcov, pcov * sc,
                                                               f, f_transposed,
                                                               h_plus, c1, c2,
                                                               b1, b2, kdp_th)
            kdp_mat[:, 2 * i] = kdp_dummy_f2[PADDING:nrg + PADDING]

            # Backward
            kdp_dummy_b2, _ = _kdp_estimation_backward_fixed(psidp, rcov,
                                                             pcov * sc, f,
                                                             f_transposed, h_plus,
                                                             c1, c2, b1, b2,
                                                             kdp_th, mpsidp)
            kdp_mat[:, 2 * i + 1] = kdp_dummy_b2[PADDING:nrg + PADDING]

        # Compile the final estimate
        # Get some reference mean values
        kdp_mean = np.nanmean(kdp_mat, axis=1)
        kdp_mean_shift = np.roll(kdp_mean, -1)
        diff_mean = kdp_mean - kdp_mean_shift

        kdp_std = np.nanstd(kdp_mat, axis=1)

        if len(diff_mean) < 4:
            size_filt = len(diff_mean)
        else:
            size_filt = 4

        diff_mean_smooth = np.convolve(diff_mean,
                                       np.ones((size_filt,)) / size_filt,
                                       mode='same')

        # Backward estimate if diff_mean greater than a defined threshold
        condi = np.where(diff_mean_smooth > th2_comp)[0]
        if len(condi):
            kdp_dummy = kdp_mat[:, np.arange(len(SCALERS)) * 2 + 1]
            kdp_sim[condi, :] = kdp_dummy[condi, :]

        # Forward estimate if diff_mean lower than a defined threshold
        condi = np.where(diff_mean_smooth < th1_comp)[0]
        if len(condi):
            kdp_dummy = kdp_mat[:, np.arange(len(SCALERS)) * 2]
            kdp_sim[condi, :] = kdp_dummy[condi, :]

        # Combination of the two in the middle
        condi = np.where(np.logical_and(diff_mean_smooth >= th1_comp,
                                        diff_mean_smooth <= th2_comp))[0]
        if len(condi):
            weight2 = (-0.5 / 0.15) * diff_mean_smooth + 0.5

            weight2 = np.tile(weight2, (len(SCALERS), 1)).T

            kdp_dummy = (1-weight2)*kdp_mat[:, np.arange(len(SCALERS))*2+1] \
                + weight2 * kdp_mat[:, np.arange(len(SCALERS)) * 2]
            kdp_sim[condi, :] = kdp_dummy[condi, :]

        # Now we reduced to 11 ensemble members: compile the final one
        kdp_mean_sim = np.nanmean(kdp_sim, axis=1)
        kdp_std_sim = np.nanstd(kdp_sim, axis=1)
        kdp_low_mean2 = np.nanmean(np.vstack((kdp002, kdp002f)).T, axis=1)

        # Get the range of ensemble members that compile
        # a final estimate

        # Lower bounds
        lower_bound = np.round(kdp_mean_sim * fac1) - np.round(kdp_std_sim * fac2)
        lower_bound = np.maximum(lower_bound, 0)
        lower_bound = np.minimum(lower_bound, len(SCALERS) - 1)

        # Upper bounds
        upper_bound = np.round(kdp_mean_sim * fac1) + np.round(kdp_std_sim * fac2)
        upper_bound = np.maximum(upper_bound, 0)
        upper_bound = np.minimum(upper_bound, len(SCALERS) - 1)

        # Final selection of the ensemble members
        for uu in range(0, nrg - 1):
            selection_vector = np.arange(
                upper_bound[uu] - lower_bound[uu] + 1) + lower_bound[uu]
            selection_vector = selection_vector.astype(int)
            kdp_filter_out[uu] = np.mean(kdp_sim[uu, selection_vector])

        # Final filtering of excessively negative values:
        # TO DO: It would be better to get rid of this filtering
        ind_lt_0 = np.where(kdp_filter_out < th1_final)[0]

        if len(ind_lt_0):
            kdp_filter_out[ind_lt_0] = kdp_low_mean2[ind_lt_0]

        # Compute phidp from Kdp
        phidp_filter_out = np.cumsum(kdp_filter_out) * 2. * dr

        phinan = np.where(np.isnan(psidp))[0]
        if len(phinan):
            phidp_filter_out[phinan] = np.nan
            kdp_filter_out[phinan] = np.nan
        phidp_filter_out[nrg - 1] = np.nan
        kdp_filter_out[nrg - 1] = np.nan

        # Pad with nan if offset > 0
        kdp_filter_out = np.pad(kdp_filter_out, (offset, 0), mode='constant',
                                constant_values=np.nan)
        kdp_std = np.pad(kdp_std, (offset, 0), mode='constant',
                         constant_values=np.nan)
        phidp_filter_out = np.pad(phidp_filter_out, (offset, 0), mode='constant',
                                  constant_values=np.nan)

        return kdp_filter_out, kdp_std, phidp_filter_out

    def kdp_vulpiani(radar, gatefilter=None, fill_value=None, psidp_field=None,
                     kdp_field=None, phidp_field=None, band='C', windsize=10,
                     n_iter=10, interp=False, prefilter_psidp=False,
                     filter_opt=None, parallel=False):
        """
        Estimates Kdp with the Vulpiani method for a 2D array of psidp measurements
        with the first dimension being the distance from radar and the second
        dimension being the angles (azimuths for PPI, elev for RHI).The input psidp
        is assumed to be pre-filtered (for ex. with the filter_psidp function)

        Parameters
        ----------
        radar : Radar
            Radar containing differential phase field.
        gatefilter : GateFilter, optional
            A GateFilter indicating radar gates that should be excluded when
            analysing differential phase measurements.
        fill_value : float, optional
            Value indicating missing or bad data in differential phase field, if
            not specified, the default in the Py-ART configuration file will be
            used
        psidp_field : str, optional
            Total differential phase field. If None, the default field name must be
            specified in the Py-ART configuration file.
        kdp_field : str, optional
            Specific differential phase field. If None, the default field name must
            be specified in the Py-ART configuration file.
        phidp_field : str, optional
            Propagation differential phase field. If None, the default field name
            must be specified in the Py-ART configuration file.
        band : char, optional
            Radar frequency band string. Accepted "X", "C", "S" (capital
            or not). It is used to set default boundaries for expected
            values of Kdp.
        windsize : int, optional
            Size in # of gates of the range derivative window. Should be even.
        n_iter : int, optional
            Number of iterations of the method. Default is 10.
        interp : bool, optional
            If True, all the nans are interpolated.The advantage is that less data
            are lost (the iterations in fact are "eating the edges") but some
            non-linear errors may be introduced.
        prefilter_psidp : bool, optional
            If set, the psidp measurements will first be filtered with the
            filter_psidp method, which can improve the quality of the final Kdp.
        filter_opt : dict, optional
            The arguments for the prefilter_psidp method, if empty, the defaults
            arguments of this method will be used.
        parallel : bool, optional
            Flag to enable parallel computation (one core for every psidp profile).

        Returns
        -------
        kdp_dict : dict
            Retrieved specific differential phase data and metadata.
        phidpr_dict,: dict
            Retrieved differential phase data and metadata.

        References
        ----------
        Gianfranco Vulpiani, Mario Montopoli, Luca Delli Passeri, Antonio G. Gioia,
        Pietro Giordano, and Frank S. Marzano, 2012: On the Use of Dual-Polarized
        C-Band Radar for Operational Rainfall Retrieval in Mountainous Areas.
        J. Appl. Meteor. Climatol., 51, 405-425, doi: 10.1175/JAMC-D-10-05024.1.

        """
        if np.mod(windsize, 2):
            warnings.warn('In the Vulpiani method, the windsize should be even. '
                          + 'Using default value, windsize = 10')
            windsize = 10

        if parallel:
            import multiprocessing as mp
            pool = mp.Pool(processes=mp.cpu_count(), maxtasksperchild=1)

        # parse fill value
        if fill_value is None:
            fill_value = get_fillvalue()

        # parse field names
        if psidp_field is None:
            psidp_field = get_field_name('differential_phase')
        if kdp_field is None:
            kdp_field = get_field_name('specific_differential_phase')
        if phidp_field is None:
            phidp_field = get_field_name('differential_phase')

        # parse range resolution, length scale and low-pass filter constraint
        # weight
        dr = _parse_range_resolution(radar, check_uniform=True)

        # parse total differential phase measurements
        if prefilter_psidp:
            if filter_opt is None:
                filter_opt = {}
            # Assign psidp field to filter_psidp inputs
            filter_opt['psidp_field'] = psidp_field
            # Filter psidp
            psidp_o = filter_psidp(radar, **filter_opt)
        else:
            psidp_o = radar.fields[psidp_field]['data']

        # mask radar gates indicated by the gate filter
        if gatefilter is not None:
            psidp_o = np.ma.masked_where(gatefilter.gate_excluded, psidp_o)

        func = partial(_kdp_vulpiani_profile, dr=dr, windsize=windsize,
                       band=band, n_iter=n_iter, interp=interp)

        all_psidp_prof = list(psidp_o)

        if parallel:
            list_est = pool.map(func, all_psidp_prof)
        else:
            list_est = map(func, all_psidp_prof)

        kdp = np.ma.zeros(psidp_o.shape)
        kdp[:] = np.ma.masked
        kdp.set_fill_value(fill_value)

        phidp_rec = np.ma.zeros(psidp_o.shape)
        phidp_rec[:] = np.ma.masked
        phidp_rec.set_fill_value(fill_value)

        for i, l in enumerate(list_est):
            kdp[i, 0:len(l[0])] = l[0]
            phidp_rec[i, 0:len(l[1])] = l[1]

        # Mask the estimated Kdp and reconstructed Phidp with the mask of original
        # psidp
        if isinstance(psidp_o, np.ma.masked_array):
            masked = psidp_o.mask
            kdp = np.ma.array(kdp, mask=masked, fill_value=fill_value)
            phidp_rec = np.ma.array(phidp_rec, mask=masked, fill_value=fill_value)

        # create specific differential phase field dictionary and store data
        kdp_dict = get_metadata(kdp_field)
        kdp_dict['data'] = kdp
        # kdp_dict['valid_min'] = 0.0

        # create reconstructed differential phase field dictionary and store data
        phidpr_dict = get_metadata(phidp_field)
        phidpr_dict['data'] = phidp_rec
        # phidpr_dict['valid_min'] = 0.0

        if parallel:
            pool.close()

        return kdp_dict, phidpr_dict

    def _kdp_vulpiani_profile(psidp_in, dr, windsize=10,
                              band='X', n_iter=10, interp=False):
        """
        Estimates Kdp with the Vulpiani method for a single profile of psidp
        measurements

        Parameters
        ----------
        psidp_in : ndarray
            Total differential phase measurements.
        dr : float
            Range resolution in meters.
        windsize : int, optional
            Size in # of gates of the range derivative window.
        band : char, optional
            Radar frequency band string. Accepted "X", "C", "S" (capital
            or not). It is used to set default boundaries for expected
            values of Kdp
        n_iter : int, optional
            Number of iterations of the method. Default is 10.
        interp : bool, optional
            If set all the nans are interpolated.The advantage is that less data
            are lost (the iterations in fact are "eating the edges") but some
            non-linear errors may be introduced

        Returns
        -------
        kdp_calc : ndarray
            Retrieved specific differential profile
        phidp_rec,: ndarray
            Retrieved differential phase profile

        """
        mask = np.ma.getmaskarray(psidp_in)
        l = windsize
        l2 = int(l/2)
        drm = dr/1000.

        if mask.all() is True:
            # Check if all elements are masked
            return psidp_in, psidp_in, psidp_in  # Return the NaNs...

        # Thresholds in kdp calculation
        if band == 'X':
            th1 = -2.
            th2 = 40.
            std_th = 5.
        elif band == 'C':
            th1 = -2.
            th2 = 20.
            std_th = 5.
        elif band == 'S':
            th1 = -2.
            th2 = 14.
            std_th = 5.
        else:
            print('Unexpected value set for the band keyword ')
            print(band)
            return None

        psidp = psidp_in
        nn = len(psidp_in)

        # Get information of valid and non valid points in psidp the new psidp
        valid = np.logical_not(mask)
        if interp:
            ranged = np.arange(0, nn)
            psidp_interp = psidp
            # interpolate
            if np.ma.is_masked(psidp):
                interp = interpolate.interp1d(ranged[valid], psidp[valid],
                                              kind='zero', bounds_error=False,
                                              fill_value=np.nan)
                psidp_interp[mask] = interp(ranged[mask])

            psidp = psidp_interp

        psidp = np.ma.filled(psidp, np.nan)
        kdp_calc = np.zeros([nn])

        # first guess
        # In the core of the profile
        kdp_calc[l2:nn-l2] = (psidp[l:nn]-psidp[0:nn-l])/(2.*l*drm)

        # set ray extremes to 0
        kdp_calc[0:l2] = 0.
        kdp_calc[nn-l2:] = 0.

        # set all non-valid data to 0
        kdp_calc[np.isnan(kdp_calc)] = 0.

        # apply thresholds
        kdp_calc[kdp_calc <= th1] = 0.
        kdp_calc[kdp_calc >= th2] = 0.

        # Remove bins with texture higher than treshold
        tex = np.ma.zeros(kdp_calc.shape)
        # compute the local standard deviation
        # (make sure that it is and odd window)
        tex_aux = np.ma.std(rolling_window(kdp_calc, l2*2+1), -1)
        tex[l2:-l2] = tex_aux
        kdp_calc[tex > std_th] = 0.

        # Loop over iterations
        for i in range(0, n_iter):
            phidp_rec = np.ma.cumsum(kdp_calc)*2.*drm

            # In the core of the profile
            kdp_calc[l2:nn-l2] = (phidp_rec[l:nn]-phidp_rec[0:nn-l])/(2.*l*drm)

            # set ray extremes to 0
            kdp_calc[0:l2] = 0.
            kdp_calc[nn-l2:] = 0.

            # apply thresholds
            kdp_calc[kdp_calc <= th1] = 0.
            kdp_calc[kdp_calc >= th2] = 0.

        # Censor Kdp where Psidp was not defined
        kdp_calc = np.ma.masked_where(mask, kdp_calc)

        # final reconstructed PhiDP from KDP
        phidp_rec = np.ma.cumsum(kdp_calc)*2.*drm

        return kdp_calc, phidp_rec

    def filter_psidp(radar, psidp_field=None, rhohv_field=None, minsize_seq=5,
                     median_filter_size=7, thresh_rhohv=0.65, max_discont=90):
        """
        Filter measured psidp to remove spurious data in four steps:
             1. Censor it where Rhohv is lower than threshold
             2. Unravel angles when strong discontinuities are detected
             3. Remove very short sequences of valid data
             4. Apply a median filter on every profile

        Parameters
        ----------
        radar : Radar
            Radar containing differential phase field.
        psidp_field : str, optional
            Total differential phase field. If None, the default field name must be
            specified in the Py-ART configuration file.
        rhohv_field : str, optional
            Cross correlation ratio field. If None, the default field name must
            be specified in the Py-ART configuration file.
        minsize_seq  : integer, optional
            Minimal len (in radar gates) of sequences of valid data to be accepted
        median_filter_size : integer, optional
            Size (in radar gates) of the median filter to be applied on psidp
        thresh_rhohv : float, optional
            Censoring threshold in rhohv (gates with rhohv < thresh_rhohv)
            will be rejected
        max_discont : int, optional
            Maximum discontinuity between psidp values, default is 90 deg

        Returns
        -------
        psidp_filt : ndarray
            Filtered psidp field

        """
        # parse field names
        if psidp_field is None:
            psidp_field = get_field_name('differential_phase')
        if rhohv_field is None:
            rhohv_field = get_field_name('cross_correlation_ratio')

        # parse total differential phase and cross corr. measurements
        psidp_o = radar.fields[psidp_field]['data']
        rhohv = radar.fields[rhohv_field]['data']

        if not isinstance(psidp_o, np.ma.masked_array):
            psidp_o = np.ma.array(psidp_o, mask=np.isnan(psidp_o))

        # Initialize mask
        mask = np.ones(psidp_o.shape) * False

        # Condition on rhohv
        mask += rhohv < thresh_rhohv
        # Get original mask
        mask += psidp_o.mask

        # Remove short sequences and unwrap
        psidp_filt = np.zeros(psidp_o.shape)
        for i, psi_row in enumerate(psidp_o):
            idx = np.where(~psi_row.mask)[0]  # idx of last valid gate
            if len(idx):
                psi_row = psi_row[0:idx[-1] + 1]
                psi_row[~psi_row.mask] = np.rad2deg(
                    np.unwrap(np.deg2rad(psi_row[~psi_row.mask]),
                              np.deg2rad(max_discont)))
                psi_row_with_nan = np.ma.filled(psi_row, np.nan)
                # To be sure to always have a left and right neighbour,
                # we need to pad the signal with NaN
                psi_row_with_nan = np.pad(
                    psi_row_with_nan, (1, 1), 'constant', constant_values=(
                        np.nan,))
                idx = np.where(np.isfinite(psi_row_with_nan))[0]
                nan_left = idx[np.where(np.isnan(psi_row_with_nan[idx - 1]))[0]]
                nan_right = idx[np.where(np.isnan(psi_row_with_nan[idx + 1]))[0]]

                len_sub = nan_right - nan_left

                for j, l in enumerate(len_sub):
                    if l < minsize_seq:
                        mask[i, nan_left[j] - 1:nan_right[j] + 1] = True

                # median filter
                psi_row = signal.medfilt(psi_row_with_nan, median_filter_size)
                psidp_filt[i, 0:len(psi_row[1:-1])] = psi_row[1:-1]

        psidp_filt = np.ma.masked_array(
            psidp_filt, mask=mask, fill_value=psidp_o.fill_value)

        return psidp_filt

    # Necessary and/or potential future improvements to the KDP module:
    #
    # * The near and far range gate boundary conditions necessary for the Maesaka
    #   et al. (2012) variational method are sensitive to the number of range gates
    #   used to define both boundaries as well as outliers in these samples. This
    #   may have to be further investigated and potentially improved upon.
    #
    # * It would be beneficial for the GateFilter class to have a method which can
    #   flag radar gates above the melting layer, maybe through the user supplying
    #   an estimate of the 0 deg Celsius level above mean sea level.
    #
    # * The backscatter differential phase parameter will likely have to be updated
    #   in the future to handle various parameterizations.
    #
    # * The addition of an azimuthal low-pass filter (smoothness) constraint would
    #   be beneficial to the variational Maesaka et al. (2012) algorithm.
    #
    # * Add the ability to record and/or store minimization data, e.g., current
    #   iteration number, functional value and functional gradient norm value as a
    #   function of iteration.

    def kdp_maesaka(radar, gatefilter=None, method='cg', backscatter=None,
                    Clpf=1.0, length_scale=None, first_guess=0.01,
                    finite_order='low', fill_value=None, proc=1, psidp_field=None,
                    kdp_field=None, phidp_field=None, debug=False, verbose=False,
                    **kwargs):
        """
        Compute the specific differential phase (KDP) from corrected (e.g.,
        unfolded) total differential phase data based on the variational method
        outlined in Maesaka et al. (2012). This method assumes a monotonically
        increasing propagation differential phase (PHIDP) with increasing range
        from the radar, and therefore is limited to rainfall below the melting
        layer and/or warm clouds at weather radar frequencies (e.g., S-, C-, and
        X-band). This method currently only supports radar data with constant range
        resolution.

        Following the notation of Maesaka et al. (2012), the primary control
        variable k is proportional to KDP,

                                    k**2 = 2 * KDP * dr

        which, because of the square, assumes that KDP always takes a positive
        value.

        Parameters
        ----------
        radar : Radar
            Radar containing differential phase field.
        gatefilter : GateFilter
            A GateFilter indicating radar gates that should be excluded when
            analysing differential phase measurements.
        method : str, optional
            Type of scipy.optimize method to use when minimizing the cost
            functional. The default method uses a nonlinear conjugate gradient
            algorithm. In Maesaka et al. (2012) they use the Broyden-Fletcher-
            Goldfarb-Shanno (BFGS) algorithm, however for large functional size
            (e.g., 100K+ variables) this algorithm is considerably slower than a
            conjugate gradient algorithm.
        backscatter : optional
            Define the backscatter differential phase. If None, the backscatter
            differential phase is set to zero for all range gates. Note that
            backscatter differential phase can be parameterized using attentuation
            corrected differential reflectivity.
        Clpf : float, optional
            The low-pass filter (radial smoothness) constraint weight as in
            equation (15) of Maesaka et al. (2012).
        length_scale : float, optional
            Length scale in meters used to bring the dimension and magnitude of the
            low-pass filter cost functional in line with the observation cost
            functional. If None, the length scale is set to the range resolution.
        first_guess : float, optional
            First guess for control variable k. Since k is proportional to the
            square root of KDP, the first guess should be close to zero to signify
            a KDP field close to 0 deg/km everywhere. However, the first guess
            should not be exactly zero in order to avoid convergence criteria after
            the first iteration. In fact it is recommended to use a value closer to
            one than zero.
        finite_order : 'low' or 'high', optional
            The finite difference accuracy to use when computing derivatives.
        maxiter : int, optional
            Maximum number of iterations to perform during cost functional
            minimization. The maximum number of iterations are only performed if
            convergence criteria are not met. For variational schemes such as this
            one, it is generally not recommended to try and achieve convergence
            criteria since the values of the cost functional and/or its gradient
            norm are somewhat arbitrary.
        fill_value : float, optional
            Value indicating missing or bad data in differential phase field.
        proc : int, optional
            The number of parallel threads (CPUs) to use. Currently no
            multiprocessing capability exists.
        psidp_field : str, optional
            Total differential phase field. If None, the default field name must be
            specified in the Py-ART configuration file.
        kdp_field : str, optional
            Specific differential phase field. If None, the default field name must
            be specified in the Py-ART configuration file.
        phidp_field : str, optional
            Propagation differential phase field. If None, the default field name
            must be specified in the Py-ART configuration file.
        debug : bool, optional
            True to print debugging information, False to suppress.
        verbose : bool, optional
            True to print relevant information, False to suppress.

        Returns
        -------
        kdp_dict : dict
            Retrieved specific differential phase data and metadata.
        phidpf_dict, phidpr_dict : dict
            Retrieved forward and reverse direction propagation differential phase
            data and metadata.

        References
        ----------
        Maesaka, T., Iwanami, K. and Maki, M., 2012: "Non-negative KDP Estimation
        by Monotone Increasing PHIDP Assumption below Melting Layer". The Seventh
        European Conference on Radar in Meteorology and Hydrology.

        """
        # parse fill value
        if fill_value is None:
            fill_value = get_fillvalue()

        # parse field names
        if psidp_field is None:
            psidp_field = get_field_name('differential_phase')
        if kdp_field is None:
            kdp_field = get_field_name('specific_differential_phase')
        if phidp_field is None:
            phidp_field = get_field_name('differential_phase')

        # parse range resolution, length scale and low-pass filter constraint
        # weight
        dr = _parse_range_resolution(radar, check_uniform=True, verbose=verbose)

        # parse length scale
        if length_scale is None:
            length_scale = dr

        # parse low-pass filter constraint weight
        # this brings the dimensions and magnitude of the low-pass filter
        # constraint in line with the differential phase measurement constraint
        Clpf *= length_scale**4

        # parse near and far range gate propagation differential phase boundary
        # conditions
        bcs = boundary_conditions_maesaka(
            radar, gatefilter=gatefilter, psidp_field=psidp_field, debug=debug,
            verbose=verbose, **kwargs)
        phi_near, phi_far = bcs[:2]
        idx_near, idx_far = bcs[4:]

        # parse total differential phase measurements
        psidp_o = radar.fields[psidp_field]['data']

        if debug:
            N = np.ma.count(psidp_o)
            print('Sample size before filtering: {}'.format(N))

        # mask radar gates indicated by the gate filter
        if gatefilter is not None:
            psidp_o = np.ma.masked_where(gatefilter.gate_excluded, psidp_o)

        # mask any radar gates which are closer (further) than the near (far)
        # boundary condition ranges
        for ray in range(radar.nrays):
            psidp_o[ray, :idx_near[ray]] = np.ma.masked
            psidp_o[ray, idx_far[ray] + 1:] = np.ma.masked

        if debug:
            N = np.ma.count(psidp_o)
            print('Sample size after filtering: {}'.format(N))

        # parse differential phase measurement weight
        Cobs = np.logical_not(np.ma.getmaskarray(psidp_o)).astype(psidp_o.dtype)
        psidp_o = np.ma.filled(psidp_o, fill_value)

        # parse backscattered differential phase data
        if backscatter is None:
            dhv = np.zeros_like(psidp_o, subok=False)

        # parse solver options
        options = {
            'maxiter': kwargs.get('maxiter', 50),
            'gtol': kwargs.get('gtol', 1.0e-5),
            'disp': verbose,
        }
        if debug:
            optimize.show_options(solver='minimize', method=method)

        # parse initial conditions (first guess)
        x0 = np.zeros_like(psidp_o, subok=False).flatten()
        x0.fill(first_guess)

        if verbose:
            print('Cost functional size: {}'.format(x0.size))

        # define arguments for cost functional and its Jacobian (gradient)
        args = (psidp_o, [phi_near, phi_far],
                dhv, dr, Cobs, Clpf,
                finite_order, fill_value,
                proc, debug, verbose)

        if debug:
            start = time.time()

        # minimize the cost functional
        xopt = optimize.minimize(
            _cost_maesaka, x0, args=args, method=method, jac=_jac_maesaka,
            hess=None, hessp=None, bounds=None, constraints=None, callback=None,
            options=options)

        if debug:
            elapsed = time.time() - start
            print('Elapsed time for minimization: {:.0f} sec'.format(elapsed))

        # parse control variables from optimized result
        k = xopt.x.reshape(psidp_o.shape)

        # compute specific differential phase from control variable k in deg/km
        kdp = k**2 / (2.0 * dr) * 1000.0

        if debug:
            print('Min retrieved KDP: {:.2f} deg/km'.format(kdp.min()))
            print('Max retrieved KDP: {:.2f} deg/km'.format(kdp.max()))
            print('Mean retrieved KDP: {:.2f} deg/km'.format(kdp.mean()))

        # create specific differential phase field dictionary and store data
        kdp_dict = get_metadata(kdp_field)
        kdp_dict['data'] = kdp
        kdp_dict['valid_min'] = 0.0
        kdp_dict['Clpf'] = Clpf

        # compute forward and reverse direction propagation differential phase
        phidp_f, phidp_r = _forward_reverse_phidp(
            k, [phi_near, phi_far], verbose=verbose)

        # create forward direction propagation differential phase field dictionary
        # and store data
        phidpf_dict = get_metadata(phidp_field)
        phidpf_dict['data'] = phidp_f
        phidpf_dict['comment'] = 'Retrieved in forward direction'

        # create reverse direction propagation differential phase field dictionary
        # and store data
        phidpr_dict = get_metadata(phidp_field)
        phidpr_dict['data'] = phidp_r
        phidpr_dict['comment'] = 'Retrieved in reverse direction'

        return kdp_dict, phidpf_dict, phidpr_dict

    def boundary_conditions_maesaka(
            radar, gatefilter=None, n=20, psidp_field=None, debug=False,
            verbose=False, **kwargs):
        """
        Determine near range gate and far range gate propagation differential phase
        boundary conditions. This follows the method outlined in Maesaka et al.
        (2012), except instead of using the mean we use the median which is less
        susceptible to outliers. This function can also be used to estimate the
        system phase offset.

        Parameters
        ----------
        radar : Radar
            Radar containing total differential phase measurements.
        gatefilter : GateFilter
            A GateFilter indicating radar gates that should be excluded when
            analysing differential phase measurements.
        n : int, optional
            The number of range gates necessary to define the near and far range
            gate boundary conditions. Maesaka et al. (2012) uses a value of 30. If
            this value is too small then a spurious spike in specific differential
            phase close to the radar may be retrieved.
        check_outliers : bool, optional
            True to check for near range gate boundary condition outliers. Outliers
            near the radar are primarily the result of ground clutter returns.
        psidp_field : str, optional
            Field name of total differential phase. If None, the default field name
            must be specified in the Py-ART configuration file.
        debug : bool, optional
            True to print debugging information, False to suppress.
        verbose : bool, optional
            True to print relevant information, False to suppress.

        Returns
        -------
        phi_near : ndarray
            The near range differential phase boundary condition for each ray.
        phi_far : ndarray
            The far range differential phase boundary condition for each ray.
        range_near : ndarray
            The near range gate in meters for each ray.
        range_far : ndarray
            The far range gate in meters for each ray.
        idx_near : ndarray
            Index of nearest range gate for each ray.
        idx_far : ndarray
            Index of furthest range gate for each ray.

        """
        # parse field names
        if psidp_field is None:
            psidp_field = get_field_name('differential_phase')

        # parse differential phase measurements
        psidp = radar.fields[psidp_field]['data']
        if gatefilter is not None:
            psidp = np.ma.masked_where(gatefilter.gate_excluded, psidp)

        # find contiguous unmasked data along each ray
        slices = np.ma.notmasked_contiguous(psidp, axis=1)
        if debug:
            N = sum([len(slc) for slc in slices if hasattr(slc, '__len__')])
            print('Total number of unique non-masked regions: {}'.format(N))

        phi_near = np.zeros(radar.nrays, dtype=psidp.dtype)
        phi_far = np.zeros(radar.nrays, dtype=psidp.dtype)
        range_near = np.zeros(radar.nrays, dtype=psidp.dtype)
        range_far = np.zeros(radar.nrays, dtype=psidp.dtype)
        idx_near = np.zeros(radar.nrays, dtype=np.int32)
        idx_far = np.zeros(radar.nrays, dtype=np.int32)

        for ray, regions in enumerate(slices):

            # check if all range gates are missing
            if regions is None:
                continue

            # check if all range gates available, i.e., no masked data
            if isinstance(regions, slice):
                regions = [regions]

            # near range gate boundary condition
            for slc in regions:

                # check if enough samples exist in slice
                if slc.stop - slc.start >= n:

                    # parse index and range of nearest range gate
                    idx = slice(slc.start, slc.start + n)
                    idx_near[ray] = idx.start
                    range_near[ray] = radar.range['data'][idx][0]

                    # parse data for linear regression and compute slope
                    x = radar.range['data'][idx]
                    y = psidp[ray, idx]
                    slope = stats.linregress(x, y)[0]

                    # if linear regression slope is positive, set near range gate
                    # boundary condition to first differential phase measurement,
                    # otherwise use the median value
                    if slope > 0.0:
                        phi_near[ray] = y[0]
                    else:
                        phi_near[ray] = np.median(y)
                    break

                else:
                    continue

            # far range gate boundary condition
            for slc in reversed(regions):

                if slc.stop - slc.start >= n:

                    # parse index and range of furthest range gate
                    idx = slice(slc.stop - n, slc.stop)
                    idx_far[ray] = idx.stop
                    range_far[ray] = radar.range['data'][idx][-1]

                    # parse data for linear regression and compute slope
                    x = radar.range['data'][idx]
                    y = psidp[ray, idx]
                    slope = stats.linregress(x, y)[0]

                    # if linear regression slope is positive, set far range gate
                    # boundary condition to last differential phase measurement,
                    # otherwise use the median value
                    if slope > 0.0:
                        phi_far[ray] = y[-1]
                    else:
                        phi_far[ray] = np.median(y)
                    break

                else:
                    continue

        # check for outliers in the near range boundary conditions, e.g., ground
        # clutter can introduce spurious values in certain rays

        # do not include missing values in the analysis
        phi_near_valid = phi_near[phi_near != 0.0]

        # skip the check if there are no valid values
        if kwargs.get('check_outliers', True) and (phi_near_valid.size != 0):

            # bin and count near range boundary condition values, i.e., create
            # a distribution of values
            # the default bin width is 5 deg
            counts, edges = np.histogram(
                phi_near_valid, bins=144, range=(-360, 360), density=False)

            # assume that the maximum counts corresponds to the maximum in the
            # system phase distribution
            # this assumption breaks down if there are a significant amount of
            # near range boundary conditions characterized by ground clutter
            system_phase_peak_left = edges[counts.argmax()]
            system_phase_peak_right = edges[counts.argmax() + 1]

            if debug:
                print('Peak of system phase distribution: {:.0f} deg'.format(
                    system_phase_peak_left))

            # determine left edge location of system phase distribution
            # we consider five counts or less to be insignificant
            is_left_side = np.logical_and(
                edges[:-1] < system_phase_peak_left, counts <= 5)
            left_edge = edges[:-1][is_left_side][-1]

            # determine right edge location of system phase distribution
            # we consider five counts or less to be insignificant
            is_right_side = np.logical_and(
                edges[1:] > system_phase_peak_right, counts <= 5)
            right_edge = edges[1:][is_right_side][0]

            if debug:
                print('Left edge of system phase distribution: {:.0f} deg'.format(
                    left_edge))
                print('Right edge of system phase distribution: {:.0f} deg'.format(
                    right_edge))

            # define the system phase offset as the median value of the system
            # phase distriubion
            is_system_phase = np.logical_or(
                phi_near_valid >= left_edge, phi_near_valid <= right_edge)
            system_phase_offset = np.median(phi_near_valid[is_system_phase])

            if debug:
                print('Estimated system phase offset: {:.0f} deg'.format(
                    system_phase_offset))

            for ray, bc in enumerate(phi_near):

                # if near range boundary condition does not draw from system phase
                # distribution then set it to the system phase offset
                if bc < left_edge or bc > right_edge:
                    phi_near[ray] = system_phase_offset

        # check for unphysical boundary conditions, i.e., propagation differential
        # phase should monotonically increase from the near boundary to the far
        # boundary
        is_unphysical = phi_far - phi_near < 0.0
        phi_far[is_unphysical] = phi_near[is_unphysical]
        if verbose:
            N = is_unphysical.sum()
            print('Rays with unphysical boundary conditions: {}'.format(N))

        return phi_near, phi_far, range_near, range_far, idx_near, idx_far

    def _cost_maesaka(x, psidp_o, bcs, dhv, dr, Cobs, Clpf, finite_order,
                      fill_value, proc, debug=False, verbose=False):
        """
        Compute the value of the cost functional similar to equations (12)-(15) in
        Maesaka et al. (2012).

        Parameters
        ----------
        x : ndarray
            Analysis vector containing control variable k.
        psidp_o : ndarray
            Total differential phase measurements.
        bcs : array_like
            The near and far range gate propagation differential phase boundary
            conditions.
        dhv : ndarray
            Backscatter differential phase.
        dr : float
            Range resolution in meters.
        Cobs : ndarray
            The differential phase measurement constraint weights. The weight
            should vanish where no differential phase measurements are available.
        Clpf : float
            The low-pass filter (radial smoothness) constraint weight as in
            equation (15) of Maesaka et al. (2012).
        finite_order : 'low' or 'high'
            The finite difference accuracy to use when computing derivatives.
        fill_value : float
            Value indicating missing or bad data in radar field data.
        proc : int
            The number of parallel threads (CPUs) to use.
        debug : bool, optional
            True to print debugging information, False to suppress.
        verbose : bool, optional
            True to print progress information, False to suppress.

        Returns
        -------
        J : float
            Value of total cost functional.

        """
        # parse control variable k from analysis vector
        nr, ng = psidp_o.shape
        k = x.reshape(nr, ng)

        # parse near and far range gate boundary conditions
        phi_near, phi_far = bcs

        # compute forward direction propagation differential phase from control
        # variable k
        phi_fa = np.zeros_like(k, subok=False)
        phi_fa[:, 1:] = np.cumsum(k[:, :-1]**2, axis=1)

        # compute reverse direction propagation differential phase from control
        # variable k
        phi_ra = np.zeros_like(k, subok=False)
        phi_ra[:, :-1] = np.cumsum(k[:, :0:-1]**2, axis=1)[:, ::-1]

        # compute forward and reverse propagation differential phase
        # from total differential phase observations
        phi_fo = psidp_o - dhv - phi_near[:, np.newaxis].repeat(ng, axis=1)
        phi_ro = phi_far[:, np.newaxis].repeat(ng, axis=1) - psidp_o + dhv

        # cost: forward direction differential phase observations
        Jof = 0.5 * np.sum(Cobs * (phi_fa - phi_fo)**2)

        # cost: reverse direction differential phase observations
        Jor = 0.5 * np.sum(Cobs * (phi_ra - phi_ro)**2)

        # prepare control variable k for Cython function
        k = np.ascontiguousarray(k, dtype=np.float64)

        # compute low-pass filter term, i.e., second order derivative of k with
        # respect to range
        d2kdr2 = np.empty_like(k)
        _kdp_proc.lowpass_maesaka_term(k, dr, finite_order, d2kdr2)

        # cost: low-pass filter, i.e., radial smoothness
        Jlpf = 0.5 * np.sum(Clpf * (d2kdr2)**2)

        # compute value of total cost functional
        J = Jof + Jor + Jlpf

        if verbose:
            print('Forward direction observation cost : {:1.3e}'.format(Jof))
            print('Reverse direction observation cost : {:1.3e}'.format(Jor))
            print('Low-pass filter cost ............. : {:1.3e}'.format(Jlpf))
            print('Total cost ....................... : {:1.3e}'.format(J))

        return J

    def _jac_maesaka(x, psidp_o, bcs, dhv, dr, Cobs, Clpf, finite_order,
                     fill_value, proc, debug=False, verbose=False):
        """
        Compute the Jacobian (gradient) of the cost functional similar to equations
        (16)-(18) in Maesaka et al. (2012).

        Parameters
        ----------
        x : ndarray
            Analysis vector containing control variable k.
        psidp_o : ndarray
            Total differential phase measurements.
        bcs : array_like
            The near and far range gate propagation differential phase boundary
            conditions.
        dhv : ndarray
            Backscatter differential phase.
        dr : float
            Range resolution in meters.
        Cobs : ndarray
            The differential phase measurement constraint weights. The weight
            should vanish where no differential phase measurements are available.
        Clpf : float
            The low-pass filter (radial smoothness) constraint weight as in
            equation (15) of Maesaka et al. (2012).
        finite_order : 'low' or 'high'
            The finite difference accuracy to use when computing derivatives.
        fill_value : float
            Value indicating missing or bad data in radar field data.
        proc : int
            The number of parallel threads (CPUs) to use.
        debug : bool, optional
            True to print debugging information, False to suppress.
        verbose : bool, optional
            True to print progress information, False to suppress.

        Returns
        -------
        jac : ndarray
            Jacobian of the cost functional.

        """
        # parse control variable k from analysis vector
        nr, ng = psidp_o.shape
        k = x.reshape(nr, ng)

        # parse near and far range gate boundary conditions
        phi_near, phi_far = bcs

        # compute forward direction propagation differential phase from control
        # variable k
        phi_fa = np.zeros_like(k, subok=False)
        phi_fa[:, 1:] = np.cumsum(k[:, :-1]**2, axis=1)

        # compute reverse direction propagation differential phase from control
        # variable k
        phi_ra = np.zeros_like(k, subok=False)
        phi_ra[:, :-1] = np.cumsum(k[:, :0:-1]**2, axis=1)[:, ::-1]

        # compute forward and reverse propagation differential phase
        # from total differential phase observations
        phi_fo = psidp_o - dhv - phi_near[:, np.newaxis].repeat(ng, axis=1)
        phi_ro = phi_far[:, np.newaxis].repeat(ng, axis=1) - psidp_o + dhv

        # prepare control variable k for Cython functions
        k = np.ascontiguousarray(k, dtype=np.float64)

        # cost: forward direction differential phase observations
        dJofdk = np.zeros_like(k, subok=False)
        dJofdk[:, :-1] = 2.0 * k[:, :-1] * np.cumsum(
            (Cobs[:, 1:] * (phi_fa[:, 1:] - phi_fo[:, 1:]))[:, ::-1],
            axis=1)[:, ::-1]

        # cost: reverse direction differential phase observations
        dJordk = np.zeros_like(k, subok=False)
        dJordk[:, 1:] = 2.0 * k[:, 1:] * np.cumsum(
            (Cobs[:, :-1] * (phi_ra[:, :-1] - phi_ro[:, :-1])), axis=1)

        # compute low-pass filter term, i.e., second order derivative of k with
        # respect to range
        d2kdr2 = np.empty_like(k)
        _kdp_proc.lowpass_maesaka_term(k, dr, finite_order, d2kdr2)

        # compute gradients of Jlpf with respect to the control variable k
        dJlpfdk = np.empty_like(d2kdr2)
        _kdp_proc.lowpass_maesaka_jac(d2kdr2, dr, Clpf, finite_order, dJlpfdk)

        # sum control variable derivative components
        dJdk = dJofdk + dJordk + dJlpfdk
        jac = dJdk.flatten()

        # compute the vector norm of the Jacobian
        if verbose:
            mag = np.linalg.norm(jac, ord=None, axis=None)
            print('Vector norm of Jacobian: {:1.3e}'.format(mag))

        return jac

    def _forward_reverse_phidp(k, bcs, verbose=False):
        """
        Compute the forward and reverse direction propagation differential phases
        from the control variable k and boundary conditions following equations (1)
        and (7) in Maesaka et al. (2012).

        Parameters
        ----------
        k : ndarray
            Control variable k of the Maesaka et al. (2012) method. The control
            variable k is proportional to the square root of specific differential
            phase.
        bcs : array_like
            The near and far range gate boundary conditions.
        verbose : bool, optional
            True to print relevant information, False to suppress.

        Returns
        -------
        phidp_f : ndarray
            Forward direction propagation differential phase.
        phidp_r : ndarray
            Reverse direction propagation differential phase.

        """
        # parse near and far range gate boundary conditions
        _, ng = k.shape
        phi_near, phi_far = bcs

        # compute forward direction propagation differential phase
        phi_f = np.zeros_like(k, subok=False)
        phi_f[:, 1:] = np.cumsum(k[:, :-1]**2, axis=1)
        phidp_f = phi_f + phi_near[:, np.newaxis].repeat(ng, axis=1)

        # compute reverse direction propagation differential phase
        phi_r = np.zeros_like(k, subok=False)
        phi_r[:, :-1] = np.cumsum(k[:, :0:-1]**2, axis=1)[:, ::-1]
        phidp_r = phi_far[:, np.newaxis].repeat(ng, axis=1) - phi_r

        # check quality of retrieval by comparing forward and reverse directions
        if verbose:
            phidp_mbe = np.ma.mean(phidp_f - phidp_r)
            phidp_mae = np.ma.mean(np.abs(phidp_f - phidp_r))
            print('Forward-reverse PHIDP MBE: {:.2f} deg'.format(phidp_mbe))
            print('Forward-reverse PHIDP MAE: {:.2f} deg'.format(phidp_mae))

        return phidp_f, phidp_r

    def _parse_range_resolution(
            radar, check_uniform=True, atol=1.0, verbose=False):
        """
        Parse the radar range gate resolution.

        Parameters
        ----------
        radar : Radar
            Radar containing range data.
        check_uniform : bool, optional
            True to check if all range gates are equally spaced, and if so return
            a scalar value for range resolution. If False, the resolution between
            each range gate is returned.
        atol : float, optional
            The absolute tolerance in meters allowed for discrepancies in range
            gate spacings. Only applicable when check_uniform is True. This
            parameter may be necessary to catch instances where range gate spacings
            differ by a few meters or so.
        verbose : bool, optional
            True to print the range gate resolution. Only valid if check_uniform is
            True.

        Returns
        -------
        dr : float or ndarray
            The radar range gate spacing in meters.

        """
        # parse radar range gate spacings
        dr = np.diff(radar.range['data'], n=1)

        # check for uniform resolution
        if check_uniform and np.allclose(np.diff(dr), 0.0, atol=atol):
            dr = dr[0]
            if verbose:
                print('Range resolution: {:.2f} m'.format(dr))
        else:
            raise ValueError('Radar gate spacing is not uniform')

        return dr

    def kdp_leastsquare_single_window(
            radar, wind_len=11, min_valid=6, phidp_field=None, kdp_field=None,
            vectorize=False):
        """
        Compute the specific differential phase (KDP) from differential phase data
        using a piecewise least square method. For optimal results PhiDP should
        be already smoothed and clutter filtered out.

        Parameters
        ----------
        radar : Radar
            Radar object.
        wind_len : int
            The lenght of the moving window.
        min_valid : int
            Minimum number of valid bins to consider the retrieval valid
        phidp_field : str
            Field name within the radar object which represent the differential
            phase shift. A value of None will use the default field name as
            defined in the Py-ART configuration file.
        kdp_field : str
            Field name within the radar object which represent the specific
            differential phase shift. A value of None will use the default field
            name as defined in the Py-ART configuration file.
        vectorize : bool
            whether to use a vectorized version of the least square method

        Returns
        -------
        kdp_dict : dict
            Retrieved specific differential phase data and metadata.

        """
        # parse the field parameters
        if phidp_field is None:
            phidp_field = get_field_name('differential_phase')
        if kdp_field is None:
            kdp_field = get_field_name('specific_differential_phase')

        # extract fields from radar
        radar.check_field_exists(phidp_field)
        phidp = radar.fields[phidp_field]['data']

        kdp_dict = get_metadata(kdp_field)
        if vectorize:
            kdp_dict['data'] = leastsquare_method_scan(
                phidp, radar.range['data'], wind_len=wind_len,
                min_valid=min_valid)
        else:
            kdp_dict['data'] = leastsquare_method(
                phidp, radar.range['data'], wind_len=wind_len,
                min_valid=min_valid)

        return kdp_dict

    def kdp_leastsquare_double_window(
            radar, swind_len=11, smin_valid=6, lwind_len=31, lmin_valid=16,
            zthr=40., phidp_field=None, refl_field=None, kdp_field=None,
            vectorize=False):
        """
        Compute the specific differential phase (KDP) from differential phase data
        using a piecewise least square method. For optimal results PhiDP should
        be already smoothed and clutter filtered out.

        Parameters
        ----------
        radar : Radar
            Radar object.
        swind_len : int
            The lenght of the short moving window.
        smin_valid : int
            Minimum number of valid bins to consider the retrieval valid when
            using the short moving window
        lwind_len : int
            The lenght of the long moving window.
        lmin_valid : int
            Minimum number of valid bins to consider the retrieval valid when
            using the long moving window
        zthr : float
            reflectivity value above which the short window is used
        phidp_field : str
            Field name within the radar object which represent the differential
            phase shift. A value of None will use the default field name as
            defined in the Py-ART configuration file.
        refl_field : str
            Field name within the radar object which represent the reflectivity.
            A value of None will use the default field name as defined in the
            Py-ART configuration file.
        kdp_field : str
            Field name within the radar object which represent the specific
            differential phase shift. A value of None will use the default field
            name as defined in the Py-ART configuration file.
        vectorize : bool
            whether to use a vectorized version of the least square method

        Returns
        -------
        kdp_dict : dict
            Retrieved specific differential phase data and metadata.

        """
        # parse the field parameters
        if phidp_field is None:
            phidp_field = get_field_name('differential_phase')
        if refl_field is None:
            refl_field = get_field_name('reflectivity')
        if kdp_field is None:
            kdp_field = get_field_name('specific_differential_phase')

        # extract fields from radar
        radar.check_field_exists(refl_field)
        radar.check_field_exists(phidp_field)

        refl = radar.fields[refl_field]['data']
        phidp = radar.fields[phidp_field]['data']

        if vectorize:
            skdp = leastsquare_method_scan(
                phidp, radar.range['data'], wind_len=swind_len,
                min_valid=smin_valid)

            kdp = leastsquare_method_scan(
                phidp, radar.range['data'], wind_len=lwind_len,
                min_valid=lmin_valid)
        else:
            skdp = leastsquare_method(
                phidp, radar.range['data'], wind_len=swind_len, min_valid=smin_valid)

            kdp = leastsquare_method(
                phidp, radar.range['data'], wind_len=lwind_len, min_valid=lmin_valid)

        # mix kdp
        is_short = refl > zthr
        kdp[is_short] = skdp[is_short]

        kdp_dict = get_metadata(kdp_field)
        kdp_dict['data'] = kdp

        return kdp_dict

    def leastsquare_method_scan(phidp, rng_m, wind_len=11, min_valid=6):
        """
        Compute the specific differential phase (KDP) from differential phase data
        using a piecewise least square method. For optimal results PhiDP should
        be already smoothed and clutter filtered out. This function computes the
        whole radar volume at once

        Parameters
        ----------
        phidp : masked array
            phidp field
        rng_m : array
            radar range in meters
        wind_len : int
            the window length
        min_valid : int
            Minimum number of valid bins to consider the retrieval valid

        Returns
        -------
        kdp : masked array
            Retrieved specific differential phase field

        """
        # we want an odd window
        if wind_len % 2 == 0:
            wind_len += 1
        half_wind = int((wind_len-1)/2)

        # initialize kdp
        nrays, nbins = np.shape(phidp)
        kdp = np.ma.zeros((nrays, nbins))
        kdp[:] = np.ma.masked
        kdp.set_fill_value(get_fillvalue())

        # check which gates are valid
        valid = np.logical_not(np.ma.getmaskarray(phidp))
        valid_wind = rolling_window(valid, wind_len)
        mask_wind = np.logical_not(valid_wind)
        nvalid = np.sum(valid_wind, axis=-1, dtype=int)
        ind_valid = np.logical_and(
            nvalid >= min_valid, valid[:, half_wind:-half_wind]).nonzero()
        nvalid = nvalid[ind_valid]
        del valid, valid_wind

        rng_mat = np.broadcast_to(rng_m.reshape(1, nbins), (nrays, nbins))
        rng_mat = rolling_window(rng_mat/1000., wind_len)
        rng_wind_ma = np.ma.masked_where(mask_wind, rng_mat, copy=False)
        phidp_wind = rolling_window(phidp, wind_len)

        rng_sum = np.ma.sum(rng_wind_ma, -1)[ind_valid]
        rng_sum2 = np.ma.sum(np.ma.power(rng_wind_ma, 2.), -1)[ind_valid]

        phidp_sum = np.ma.sum(phidp_wind, -1)[ind_valid]
        rphidp_sum = np.ma.sum(phidp_wind * rng_wind_ma, -1)[ind_valid]
        del rng_wind_ma, phidp_wind

        kdp[ind_valid[0], ind_valid[1]+half_wind] = (
            0.5*(rphidp_sum-rng_sum*phidp_sum/nvalid) /
            (rng_sum2-rng_sum*rng_sum/nvalid))

        return kdp

    def leastsquare_method(phidp, rng_m, wind_len=11, min_valid=6):
        """
        Compute the specific differential phase (KDP) from differential phase data
        using a piecewise least square method. For optimal results PhiDP should
        be already smoothed and clutter filtered out.

        Parameters
        ----------
        phidp : masked array
            phidp field
        rng_m : array
            radar range in meters
        wind_len : int
            the window length
        min_valid : int
            Minimum number of valid bins to consider the retrieval valid

        Returns
        -------
        kdp : masked array
            Retrieved specific differential phase field

        """
        # we want an odd window
        if wind_len % 2 == 0:
            wind_len += 1
        half_wind = int((wind_len-1)/2)

        # initialize kdp
        nrays, nbins = np.shape(phidp)
        kdp = np.ma.zeros((nrays, nbins))
        kdp[:] = np.ma.masked
        kdp.set_fill_value(get_fillvalue())

        rng_wind = rolling_window(rng_m/1000., wind_len)
        for ray in range(nrays):
            phidp_ray = phidp[ray, :]
            valid = np.logical_not(np.ma.getmaskarray(phidp_ray))
            valid_wind = rolling_window(valid, wind_len)
            mask_wind = np.logical_not(valid_wind)
            nvalid = np.sum(valid_wind, axis=-1, dtype=int)
            ind_valid = np.logical_and(
                nvalid >= min_valid, valid[half_wind:-half_wind]).nonzero()
            nvalid = nvalid[ind_valid]
            rng_wind_ma = np.ma.masked_where(mask_wind, rng_wind)
            phidp_wind = rolling_window(phidp_ray, wind_len)

            rng_sum = np.ma.sum(rng_wind_ma, -1)[ind_valid]
            rng_sum2 = np.ma.sum(np.ma.power(rng_wind_ma, 2.), -1)[ind_valid]

            phidp_sum = np.ma.sum(phidp_wind, -1)[ind_valid]
            rphidp_sum = np.ma.sum(phidp_wind * rng_wind_ma, -1)[ind_valid]

            kdp[ray, ind_valid[0]+half_wind] = (
                0.5*(rphidp_sum-rng_sum*phidp_sum/nvalid) /
                (rng_sum2-rng_sum*rng_sum/nvalid))

        return kdp