"""
pyart.correct.phase_proc
========================

Utilities for working with phase data.

Code based upon algorithm descriped in:
S. E. Giangrande et al, J. of Atmos. and Ocean. Tech., 2013, 30, 1716.

Adapted by Scott Collis and Scott Giangrande, refactored by Jonathan Helmus.

.. autosummary::
    :toctree: generated/

    det_sys_phase
    det_sys_phase_ray
    correct_sys_phase
    smooth_phidp_single_window
    smooth_phidp_double_window
    smooth_masked_scan
    smooth_masked
    fzl_index
    det_process_range
    snr
    unwrap_masked
    smooth_and_trim
    smooth_and_trim_scan
    noise
    get_phidp_unf
    construct_A_matrix
    construct_B_vectors
    LP_solver_cvxopt
    LP_solver_pyglpk
    solve_cylp
    LP_solver_cylp_mp
    LP_solver_cylp
    phase_proc_lp
    _det_sys_phase
    _det_sys_phase_ray
    _correct_sys_phase
    phase_proc_lp_gf
    get_phidp_unf_gf
    det_sys_phase_gf
    _det_sys_phase_gf

"""

from copy import deepcopy
from time import time

import numpy as np
from numpy import ma
import scipy.ndimage

from ..config import get_fillvalue, get_field_name, get_metadata
from ..filters import GateFilter
from ..util import rolling_window


def det_sys_phase(radar, ncp_lev=0.4, rhohv_lev=0.6,
                  ncp_field=None, rhv_field=None, phidp_field=None):
    """
    Determine the system phase.

    Parameters
    ----------
    radar : Radar
        Radar object for which to determine the system phase.
    ncp_lev : float, optional
        Miminum normal coherent power level. Regions below this value will
        not be included in the phase calculation.
    rhohv_lev : float, optional
        Miminum copolar coefficient level. Regions below this value will not
        be included in the phase calculation.
    ncp_field, rhv_field, phidp_field : str, optional
        Field names within the radar object which represent the normal
        coherent power, the copolar coefficient, and the differential phase
        shift. A value of None for any of these parameters will use the
        default field name as defined in the Py-ART configuration file.

    Returns
    -------
    sys_phase : float or None
        Estimate of the system phase. None is not estimate can be made.

    """
    # parse the field parameters
    if ncp_field is None:
        ncp_field = get_field_name('normalized_coherent_power')
    if rhv_field is None:
        rhv_field = get_field_name('cross_correlation_ratio')
    if phidp_field is None:
        phidp_field = get_field_name('differential_phase')

    ncp = radar.fields[ncp_field]['data'][:, 30:]
    rhv = radar.fields[rhv_field]['data'][:, 30:]
    phidp = radar.fields[phidp_field]['data'][:, 30:]
    last_ray_idx = radar.sweep_end_ray_index['data'][0]
    return _det_sys_phase(ncp, rhv, phidp, last_ray_idx, ncp_lev,
                          rhohv_lev)


def det_sys_phase_ray(radar, ind_rmin=10, ind_rmax=500, min_rcons=11,
                      zmin=20., zmax=40., phidp_field=None, refl_field=None):
    """
    Public method
    Alternative determination of the system phase.
    Assumes that the valid gates of phidp are only precipitation.
    A system phase value is found for each ray.

    Parameters
    ----------
    radar : Radar
        Radar object for which to determine the system phase.
    ind_rmin, ind_rmax : int
        Min and max range index where to look for continuous precipitation
    min_rcons : int
        The minimum number of consecutive gates to consider it a rain cell.
    zmin, zmax : float
        The minimum and maximum reflectivity to consider the radar bin
        suitable precipitation
    phidp_field : str
        Field name within the radar object which represent the differential
        phase shift. A value of None will use the default field name as
        defined in the Py-ART configuration file.
    refl_field : str
        Field name within the radar object which represent the reflectivity.
        A value of None will use the default field name as defined in the
        Py-ART configuration file.

    Returns
    -------
    phidp0_dict : dict
        Estimate of the system phase at each ray and metadata
    first_gates_dict : dict
        The first gate where PhiDP is valid and metadata

    """
    # parse the field parameters
    if phidp_field is None:
        phidp_field = get_field_name('differential_phase')
    if refl_field is None:
        refl_field = get_field_name('reflectivity')

    if phidp_field in radar.fields:
        phidp = radar.fields[phidp_field]['data']
    else:
        raise KeyError('Field not available: ' + phidp_field)
    if refl_field in radar.fields:
        refl = radar.fields[refl_field]['data']
    else:
        raise KeyError('Field not available: ' + refl_field)

    phidp0, first_gates = _det_sys_phase_ray(
        phidp, refl, radar.nrays, radar.ngates, ind_rmin=ind_rmin,
        ind_rmax=ind_rmax, min_rcons=min_rcons, zmin=zmin, zmax=zmax)

    phidp0_dict = get_metadata('system_differential_phase')
    phidp0_dict['data'] = phidp0

    first_gates_dict = get_metadata('first_gate_differential_phase')
    first_gates_dict['data'] = first_gates

    return phidp0_dict, first_gates_dict


def correct_sys_phase(radar, ind_rmin=10, ind_rmax=500, min_rcons=11,
                      zmin=20., zmax=40., psidp_field=None, refl_field=None,
                      phidp_field=None):
    """
    correction of the system offset. Public method

    Parameters
    ----------
    radar : Radar
        Radar object for which to determine the system phase.
    ind_rmin, ind_rmax : int
        Min and max range index where to look for continuous precipitation
    min_rcons : int
        The minimum number of consecutive gates to consider it a rain cell.
    zmin, zmax : float
        Minimum and maximum reflectivity to consider it a rain cell
    psidp_field : str
        Field name within the radar object which represent the differential
        phase shift. A value of None will use the default field name as
        defined in the Py-ART configuration file.
    refl_field : str
        Field name within the radar object which represent the reflectivity.
        A value of None will use the default field name as defined in the
        Py-ART configuration file.
    phidp_field : str
        Field name within the radar object which represent the corrected
        differential phase shift. A value of None will use the default field
        name as defined in the Py-ART configuration file.

    Returns
    -------
    phidp_dict : dict
        The corrected phidp field

    """
    # parse the field parameters
    if psidp_field is None:
        psidp_field = get_field_name('differential_phase')
    if refl_field is None:
        refl_field = get_field_name('reflectivity')
    if phidp_field is None:
        phidp_field = get_field_name('corrected_differential_phase')

    if psidp_field in radar.fields:
        psidp = radar.fields[psidp_field]['data']
    else:
        raise KeyError('Field not available: ' + psidp_field)
    if refl_field in radar.fields:
        refl = radar.fields[refl_field]['data']
    else:
        raise KeyError('Field not available: ' + refl_field)

    # correct phidp of system offset
    phidp = _correct_sys_phase(
        psidp, refl, radar.nsweeps, radar.nrays, radar.ngates,
        radar.sweep_start_ray_index['data'],
        radar.sweep_end_ray_index['data'], ind_rmin=ind_rmin,
        ind_rmax=ind_rmax, min_rcons=min_rcons, zmin=zmin, zmax=zmax)

    # create specific differential phase field dictionary and store data
    phidp_dict = get_metadata(phidp_field)
    phidp_dict['data'] = phidp

    return phidp_dict


def smooth_phidp_single_window(
        radar, ind_rmin=10, ind_rmax=500, min_rcons=11, zmin=20., zmax=40,
        wind_len=11, min_valid=6, psidp_field=None, refl_field=None,
        phidp_field=None):
    """
    correction of the system offset and smoothing using one window

    Parameters
    ----------
    radar : Radar
        Radar object for which to determine the system phase.
    ind_rmin, ind_rmax : int
        Min and max range index where to look for continuous precipitation
    min_rcons : int
        The minimum number of consecutive gates to consider it a rain cell.
    zmin, zmax : float
        Minimum and maximum reflectivity to consider it a rain cell
    wind_len : int
        Length of the moving window used to smooth
    min_valid : int
        Minimum number of valid bins to consider the smooth data valid
    psidp_field : str
        Field name within the radar object which represent the differential
        phase shift. A value of None will use the default field name as
        defined in the Py-ART configuration file.
    refl_field : str
        Field name within the radar object which represent the reflectivity.
        A value of None will use the default field name as defined in the
        Py-ART configuration file.
    phidp_field : str
        Field name within the radar object which represent the corrected
        differential phase shift. A value of None will use the default field
        name as defined in the Py-ART configuration file.

    Returns
    -------
    phidp_dict : dict
        The corrected phidp field

    """
    # parse the field parameters
    if psidp_field is None:
        psidp_field = get_field_name('differential_phase')
    if refl_field is None:
        refl_field = get_field_name('reflectivity')
    if phidp_field is None:
        phidp_field = get_field_name('corrected_differential_phase')

    if psidp_field in radar.fields:
        psidp = radar.fields[psidp_field]['data']
    else:
        raise KeyError('Field not available: ' + psidp_field)
    if refl_field in radar.fields:
        refl = radar.fields[refl_field]['data']
    else:
        raise KeyError('Field not available: ' + refl_field)

    # correction of system offset
    phidp = _correct_sys_phase(
        psidp, refl, radar.nsweeps, radar.nrays, radar.ngates,
        radar.sweep_start_ray_index['data'],
        radar.sweep_end_ray_index['data'], ind_rmin=ind_rmin, zmin=zmin,
        zmax=zmax, ind_rmax=ind_rmax, min_rcons=min_rcons)

    phidp = smooth_masked_scan(phidp, wind_len=wind_len, min_valid=min_valid,
                               wind_type='median')

    # create specific differential phase field dictionary and store data
    phidp_dict = get_metadata(phidp_field)
    phidp_dict['data'] = phidp

    return phidp_dict


def smooth_phidp_double_window(
        radar, ind_rmin=10, ind_rmax=500, min_rcons=11, zmin=20., zmax=40,
        swind_len=11, smin_valid=6, lwind_len=31, lmin_valid=16, zthr=40.,
        psidp_field=None, refl_field=None, phidp_field=None):
    """
    correction of the system offset and smoothing using two window

    Parameters
    ----------
    radar : Radar
        Radar object for which to determine the system phase.
    ind_rmin, ind_rmax : int
        Min and max range index where to look for continuous precipitation
    min_rcons : int
        The minimum number of consecutive gates to consider it a rain cell.
    zmin, zmax : float
        Minimum and maximum reflectivity to consider it a rain cell
    swind_len : int
        Length of the short moving window used to smooth
    smin_valid : int
        Minimum number of valid bins to consider the short window smooth data
        valid
    lwind_len : int
        Length of the long moving window used to smooth
    lmin_valid : int
        Minimum number of valid bins to consider the long window smooth data
        valid
    zthr : float
        reflectivity value above which the short window is used
    psidp_field : str
        Field name within the radar object which represent the differential
        phase shift. A value of None will use the default field name as
        defined in the Py-ART configuration file.
    refl_field : str
        Field name within the radar object which represent the reflectivity.
        A value of None will use the default field name as defined in the
        Py-ART configuration file.
    phidp_field : str
        Field name within the radar object which represent the corrected
        differential phase shift. A value of None will use the default field
        name as defined in the Py-ART configuration file.

    Returns
    -------
    phidp_dict : dict
        The corrected phidp field

    """
    # parse the field parameters
    if psidp_field is None:
        psidp_field = get_field_name('differential_phase')
    if refl_field is None:
        refl_field = get_field_name('reflectivity')
    if phidp_field is None:
        phidp_field = get_field_name('corrected_differential_phase')

    if psidp_field in radar.fields:
        psidp = radar.fields[psidp_field]['data']
    else:
        raise KeyError('Field not available: ' + psidp_field)
    if refl_field in radar.fields:
        refl = radar.fields[refl_field]['data']
    else:
        raise KeyError('Field not available: ' + refl_field)

    # correction of system offset
    phidp = _correct_sys_phase(
        psidp, refl, radar.nsweeps, radar.nrays, radar.ngates,
        radar.sweep_start_ray_index['data'],
        radar.sweep_end_ray_index['data'], ind_rmin=ind_rmin, zmin=zmin,
        zmax=zmax, ind_rmax=ind_rmax, min_rcons=min_rcons)

    sphidp = smooth_masked_scan(phidp, wind_len=swind_len,
                                min_valid=smin_valid, wind_type='median')
    phidp = smooth_masked_scan(phidp, wind_len=lwind_len,
                               min_valid=lmin_valid, wind_type='median')

    # mix phidp
    is_short = refl > zthr
    phidp[is_short] = sphidp[is_short]

    # create specific differential phase field dictionary and store data
    phidp_dict = get_metadata(phidp_field)
    phidp_dict['data'] = phidp

    return phidp_dict


def smooth_masked_scan(raw_data, wind_len=11, min_valid=6, wind_type='median'):
    """
    smoothes the data using a rolling window.
    data with less than n valid points is masked.
    Processess the entire scan at once

    Parameters
    ----------
    raw_data : float masked array
        The data to smooth.
    window_len : float
        Length of the moving window
    min_valid : float
        Minimum number of valid points for the smoothing to be valid
    wind_type : str
        type of window. Can be median or mean

    Returns
    -------
    data_smooth : float masked array
        smoothed data

    """
    valid_wind = ['median', 'mean']
    if wind_type not in valid_wind:
        raise ValueError(
            "Window "+wind_type+" is none of " + ' '.join(valid_wind))

    # we want an odd window
    if wind_len % 2 == 0:
        wind_len += 1
    half_wind = int((wind_len-1)/2)

    # initialize smoothed data
    nrays, nbins = np.shape(raw_data)
    data_smooth = np.ma.masked_all((nrays, nbins))

    # check which gates are valid
    valid = np.logical_not(np.ma.getmaskarray(raw_data))
    valid_rolled = rolling_window(valid, wind_len)
    nvalid = np.sum(valid_rolled, axis=-1, dtype=int)
    ind_valid = np.logical_and(
        nvalid >= min_valid, valid[:, half_wind:-half_wind]).nonzero()
    del valid, valid_rolled, nvalid

    # get rolling window and mask data
    data_wind = rolling_window(raw_data, wind_len)
    data_smooth[ind_valid[0], ind_valid[1]+half_wind] = eval(
        'np.ma.'+wind_type+'(data_wind, axis=-1)')[ind_valid]

    return data_smooth


def smooth_masked(raw_data, wind_len=11, min_valid=6, wind_type='median'):
    """
    smoothes the data using a rolling window.
    data with less than n valid points is masked.

    Parameters
    ----------
    raw_data : float masked array
        The data to smooth.
    window_len : float
        Length of the moving window
    min_valid : float
        Minimum number of valid points for the smoothing to be valid
    wind_type : str
        type of window. Can be median or mean

    Returns
    -------
    data_smooth : float masked array
        smoothed data

    """
    valid_wind = ['median', 'mean']
    if wind_type not in valid_wind:
        raise ValueError(
            "Window "+wind_type+" is none of " + ' '.join(valid_wind))

    # we want an odd window
    if wind_len % 2 == 0:
        wind_len += 1
    half_wind = int((wind_len-1)/2)

    # initialize smoothed data
    nrays, nbins = np.shape(raw_data)
    data_smooth = np.ma.masked_all((nrays, nbins))

    mask = np.ma.getmaskarray(raw_data)
    valid = np.logical_not(mask)

    mask_wind = rolling_window(mask, wind_len)
    valid_wind = np.logical_not(mask_wind).astype(int)
    nvalid = np.sum(valid_wind, -1)

    data_wind = rolling_window(raw_data, wind_len)

    # check which gates are valid
    ind_valid = np.logical_and(
        nvalid >= min_valid, valid[:, half_wind:-half_wind]).nonzero()

    data_smooth[ind_valid[0], ind_valid[1]+half_wind] = (
        eval('np.ma.'+wind_type+'(data_wind, axis=-1)')[ind_valid])

    return data_smooth


def fzl_index(fzl, ranges, elevation, radar_height):
    """
    Return the index of the last gate below a given altitude.

    Parameters
    ----------
    fzl : float
        Maximum altitude.
    ranges : array
        Range to measurement volume/gate in meters.
    elevation : float
        Elevation of antenna in degrees.
    radar_height :
        Altitude of radar in meters.

    Returns
    -------
    idx : int
        Index of last gate which has an altitude below `fzl`. -1 if
        all data is above the freezing level

    Notes
    -----
    Standard atmosphere is assumed, R = 4 / 3 * Re

    """
    Re = 6371.0 * 1000.0
    p_r = 4.0 * Re / 3.0
    z = radar_height + (ranges ** 2 + p_r ** 2 + 2.0 * ranges * p_r *
                        np.sin(elevation * np.pi / 180.0)) ** 0.5 - p_r
    # Make sure the freezing level isn't under the radar!
    # Return the minimum window size for the 5-pt filter
    if np.all(z > fzl):
        return 6
    else:
        return np.where(z < fzl)[0].max()


def det_process_range(radar, sweep, fzl, doc=10):
    """
    Determine the processing range for a given sweep.

    Queues the radar and returns the indices which can be used to slice
    the radar fields and select the desired sweep with gates which are
    below a given altitude.

    Parameters
    ----------
    radar : Radar
        Radar object from which ranges will be determined.
    sweep : int
        Sweep (0 indexed) for which to determine processing ranges.
    fzl : float
        Maximum altitude in meters. The determined range will not include
        gates which are above this limit.
    doc : int, optional
        Minimum number of gates which will be excluded from the determined
        range.

    Returns
    -------
    gate_end : int
        Index of last gate below `fzl` and satisfying the `doc` parameter.
        -1 if the entire volume is above the freezing level
    ray_start : int
        Ray index which defines the start of the region.
    ray_end : int
        Ray index which defined the end of the region.

    """
    # determine the index of the last valid gate
    ranges = radar.range['data']
    elevation = radar.fixed_angle['data'][sweep]
    radar_height = radar.altitude['data']
    gate_end = fzl_index(fzl, ranges, elevation, radar_height)
    if gate_end >= 0:
        gate_end = min(gate_end, len(ranges) - doc)

    ray_start = radar.sweep_start_ray_index['data'][sweep]
    ray_end = radar.sweep_end_ray_index['data'][sweep] + 1
    return gate_end, ray_start, ray_end


def snr(line, wl=11):
    """ Return the signal to noise ratio after smoothing. """
    signal = smooth_and_trim(line, window_len=wl)
    _noise = smooth_and_trim(np.sqrt((line - signal) ** 2), window_len=wl)
    return abs(signal) / _noise


def unwrap_masked(lon, centered=False, copy=True):
    """
    Unwrap a sequence of longitudes or headings in degrees.

    Parameters
    ----------
    lon : array
        Longtiudes or heading in degress. If masked output will also be
        masked.
    centered : bool, optional
        Center the unwrapping as close to zero as possible.
    copy : bool, optional.
        True to return a copy, False will avoid a copy when possible.

    Returns
    -------
    unwrap : array
        Array of unwrapped longtitudes or headings, in degrees.

    """
    masked_input = ma.isMaskedArray(lon)
    if masked_input:
        fill_value = lon.fill_value
        # masked_invalid loses the original fill_value (ma bug, 2011/01/20)
    lon = np.ma.masked_invalid(lon).astype(float)
    if lon.ndim != 1:
        raise ValueError("Only 1-D sequences are supported")
    if lon.shape[0] < 2:
        return lon
    x = lon.compressed()
    if len(x) < 2:
        return lon
    w = np.zeros(x.shape[0] - 1, int)
    ld = np.diff(x)
    np.putmask(w, ld > 180, -1)
    np.putmask(w, ld < -180, 1)
    x[1:] += (w.cumsum() * 360.0)
    if centered:
        x -= 360 * np.round(x.mean() / 360.0)
    if lon.mask is ma.nomask:
        lon[:] = x
    else:
        lon[~lon.mask] = x
    if masked_input:
        lon.fill_value = fill_value
        return lon

    return lon.filled(np.nan)


# this function adapted from the Scipy Cookbook:
# http://www.scipy.org/Cookbook/SignalSmooth
def smooth_and_trim(x, window_len=11, window='hanning'):
    """
    Smooth data using a window with requested size.

    This method is based on the convolution of a scaled window with the signal.
    The signal is prepared by introducing reflected copies of the signal
    (with the window size) in both ends so that transient parts are minimized
    in the begining and end part of the output signal.

    Parameters
    ----------
    x : array
        The input signal.
    window_len : int, optional
        The dimension of the smoothing window; should be an odd integer.
    window : str
        The type of window from 'flat', 'hanning', 'hamming', 'bartlett',
        'blackman', 'median' or 'sg_smooth'. A flat window will produce a
        moving average smoothing.

    Returns
    -------
    y : array
        The smoothed signal with length equal to the input signal.

    """
    from scipy.signal import medfilt

    if x.ndim != 1:
        raise ValueError("smooth only accepts 1 dimension arrays.")
    if x.size < window_len:
        raise ValueError("Input vector needs to be bigger than window size.")
    if window_len < 3:
        return x
    valid_windows = ['flat', 'hanning', 'hamming', 'bartlett', 'blackman',
                     'sg_smooth', 'median']
    if window not in valid_windows:
        raise ValueError(
            "Window "+window+" is none of " + ' '.join(valid_windows))

    s = np.r_[x[window_len - 1:0:-1], x, x[-1:-window_len:-1]]

    if window == 'median':
        if window_len % 2 == 0:
            window_len += 1
        y = medfilt(s, kernel_size=[window_len])
    else:
        if window == 'flat':  # moving average
            w = np.ones(int(window_len), 'd')
        elif window == 'sg_smooth':
            w = np.array([0.1, .25, .3, .25, .1])
        else:
            w = eval('np.' + window + '(window_len)')

        y = np.convolve(w / w.sum(), s, mode='valid')

    return y[int(window_len / 2):len(x) + int(window_len / 2)]


# adapted smooth and trim function to work with 2dimensional arrays
def smooth_and_trim_scan(x, window_len=11, window='hanning'):
    """
    Smooth data using a window with requested size.

    This method is based on the convolution of a scaled window with the signal.
    The signal is prepared by introducing reflected copies of the signal
    (with the window size) in both ends so that transient parts are minimized
    in the begining and end part of the output signal.

    Parameters
    ----------
    x : ndarray
        The input signal.
    window_len : int, optional
        The dimension of the smoothing window; should be an odd integer.
    window : str, optional
        The type of window from 'flat', 'hanning', 'hamming', 'bartlett',
        'blackman', 'median' or 'sg_smooth'. A flat window will produce a
        moving average smoothing.

    Returns
    -------
    y : ndarray
        The smoothed signal with length equal to the input signal.

    """
    from scipy.ndimage.filters import convolve1d
    from scipy.signal import medfilt2d

    if x.ndim != 2:
        raise ValueError("smooth only accepts 2 dimension arrays.")
    if x.shape[1] < window_len:
        mess = "Input dimension 1 needs to be bigger than window size."
        raise ValueError(mess)
    if window_len < 3:
        return x
    valid_windows = ['flat', 'hanning', 'hamming', 'bartlett', 'blackman',
                     'sg_smooth', 'median']
    if window not in valid_windows:
        raise ValueError(
            "Window "+window+" is none of " + ' '.join(valid_windows))

    if window == 'median':
        if window_len % 2 == 0:
            window_len += 1
        y = medfilt2d(x, kernel_size=[1, window_len])
    else:
        if window == 'flat':  # moving average
            w = np.ones(int(window_len), 'd')
        elif window == 'sg_smooth':
            w = np.array([0.1, .25, .3, .25, .1])
        else:
            w = eval('np.' + window + '(window_len)')

        y = convolve1d(x, w / w.sum(), axis=1)

    return y


def noise(line, wl=11):
    """ Return the noise after smoothing. """
    signal = smooth_and_trim(line, window_len=wl)
    _noise = np.sqrt((line - signal) ** 2)
    return _noise


def get_phidp_unf(radar, ncp_lev=0.4, rhohv_lev=0.6, debug=False, ncpts=20,
                  doc=-10, overide_sys_phase=False, sys_phase=-135,
                  nowrap=None, refl_field=None, ncp_field=None,
                  rhv_field=None, phidp_field=None):
    """
    Get Unfolded Phi differential phase

    Parameters
    ----------
    radar : Radar
        The input radar.
    ncp_lev : float, optional
        Miminum normal coherent power level. Regions below this value will
        not be included in the calculation.
    rhohv_lev : float, optional
        Miminum copolar coefficient level. Regions below this value will not
        be included in the calculation.
    debug : bool, optional
        True to print debugging information, False to supress printing.
    ncpts : int, optional
        Minimum number of points in a ray. Regions within a ray smaller than
        this or beginning before this gate number are excluded from
        calculations.
    doc : int or None, optional
        Index of first gate not to include in field data, None include all.
    overide_sys_phase : bool, optional
        True to use `sys_phase` as the system phase. False will determine a
        value automatically.
    sys_phase : float, optional
        System phase, not used if overide_sys_phase is False.
    nowrap : int or None, optional
        Gate number where unwrapping should begin. `None` will unwrap all
        gates.
    refl_field ncp_field, rhv_field, phidp_field : str, optional
        Field names within the radar object which represent the horizonal
        reflectivity, normal coherent power, the copolar coefficient, and the
        differential phase shift. A value of None for any of these parameters
        will use the default field name as defined in the Py-ART
        configuration file.

    Returns
    -------
    cordata : array
        Unwrapped phi differential phase.

    """
    # parse the field parameters
    if refl_field is None:
        refl_field = get_field_name('reflectivity')
    if ncp_field is None:
        ncp_field = get_field_name('normalized_coherent_power')
    if rhv_field is None:
        rhv_field = get_field_name('cross_correlation_ratio')
    if phidp_field is None:
        phidp_field = get_field_name('differential_phase')

    if doc is not None:
        my_phidp = radar.fields[phidp_field]['data'][:, 0:doc]
        my_rhv = radar.fields[rhv_field]['data'][:, 0:doc]
        my_ncp = radar.fields[ncp_field]['data'][:, 0:doc]
        my_z = radar.fields[refl_field]['data'][:, 0:doc]
    else:
        my_phidp = radar.fields[phidp_field]['data']
        my_rhv = radar.fields[rhv_field]['data']
        my_ncp = radar.fields[ncp_field]['data']
        my_z = radar.fields[refl_field]['data']
    t = time()
    if overide_sys_phase:
        system_zero = sys_phase
    else:
        system_zero = det_sys_phase(
            radar, ncp_field=ncp_field, rhv_field=rhv_field,
            phidp_field=phidp_field)
        if system_zero is None:
            system_zero = sys_phase
    cordata = np.zeros(my_rhv.shape, dtype=float)
    for radial in range(my_rhv.shape[0]):
        my_snr = snr(my_z[radial, :])
        notmeteo = np.logical_or(np.logical_or(
            my_ncp[radial, :] < ncp_lev,
            my_rhv[radial, :] < rhohv_lev), my_snr < 10.0)
        x_ma = ma.masked_where(notmeteo, my_phidp[radial, :])
        try:
            ma.notmasked_contiguous(x_ma)
            for slc in ma.notmasked_contiguous(x_ma):
                # so trying to get rid of clutter and small things that
                # should not add to phidp anyway
                if slc.stop - slc.start < ncpts or slc.start < ncpts:
                    x_ma.mask[slc.start - 1:slc.stop + 1] = True
            c = 0
        except TypeError:  # non sequence, no valid regions
            c = 1  # ie do nothing
            x_ma.mask[:] = True
        except AttributeError:
            # sys.stderr.write('No Valid Regions, ATTERR \n ')
            # sys.stderr.write(myfile.times['time_end'].isoformat() + '\n')
            # print x_ma
            # print x_ma.mask
            c = 1  # also do nothing
            x_ma.mask = True
        if 'nowrap' is not None:
            # Start the unfolding a bit later in order to avoid false
            # jumps based on clutter
            unwrapped = deepcopy(x_ma)
            end_unwrap = unwrap_masked(x_ma[nowrap::], centered=False)
            unwrapped[nowrap::] = end_unwrap
        else:
            unwrapped = unwrap_masked(x_ma, centered=False)
        # end so no clutter expected
        system_max = unwrapped[np.where(np.logical_not(
            notmeteo))][-10:-1].mean() - system_zero
        unwrapped_fixed = np.zeros(len(x_ma), dtype=float)
        based = unwrapped-system_zero
        based[0] = 0.0
        notmeteo[0] = False
        based[-1] = system_max
        notmeteo[-1] = False
        unwrapped_fixed[np.where(np.logical_not(based.mask))[0]] = \
            based[np.where(np.logical_not(based.mask))[0]]
        if len(based[np.where(np.logical_not(based.mask))[0]]) > 11:
            unwrapped_fixed[np.where(based.mask)[0]] = \
                np.interp(np.where(based.mask)[0],
                          np.where(np.logical_not(based.mask))[0],
                          smooth_and_trim(based[np.where(
                              np.logical_not(based.mask))[0]]))
        else:
            unwrapped_fixed[np.where(based.mask)[0]] = \
                np.interp(np.where(based.mask)[0],
                          np.where(np.logical_not(based.mask))[0],
                          based[np.where(np.logical_not(based.mask))[0]])
        if c != 1:
            cordata[radial, :] = unwrapped_fixed
        else:
            cordata[radial, :] = np.zeros(my_rhv.shape[1])
    if debug:
        print("Exec time: ", time() - t)
    return cordata


def construct_A_matrix(n_gates, filt):
    """
    Construct a row-augmented A matrix. Equation 5 in Giangrande et al, 2012.

    A is a block matrix given by:

    .. math::

        \\bf{A} = \\begin{bmatrix} \\bf{I} & \\bf{-I} \\\\\\\\
                  \\bf{-I} & \\bf{I} \\\\\\\\ \\bf{Z}
                  & \\bf{M} \\end{bmatrix}

    where
        :math:`\\bf{I}` is the identity matrix
        :math:`\\bf{Z}` is a matrix of zeros
        :math:`\\bf{M}` contains our differential constraints.

    Each block is of shape n_gates by n_gates making
    shape(:math:`\\bf{A}`) = (3 * n, 2 * n).

    Note that :math:`\\bf{M}` contains some side padding to deal with edge
    issues.

    Parameters
    ----------
    n_gates : int
        Number of gates, determines size of identity matrix.
    filt : array
        Input filter.

    Returns
    -------
    a : matrix
        Row-augmented A matrix.

    """
    Identity = np.eye(n_gates)
    filter_length = len(filt)
    M_matrix_middle = np.diag(np.ones(n_gates - filter_length + 1), k=0) * 0.0
    posn = np.linspace(-1.0 * (filter_length - 1) / 2, (filter_length - 1)/2,
                       filter_length)
    for diag in range(filter_length):
        M_matrix_middle = M_matrix_middle + np.diag(
            np.ones(int(n_gates - filter_length + 1 - np.abs(posn[diag]))),
            k=int(posn[diag])) * filt[diag]
    side_pad = (filter_length - 1) // 2
    M_matrix = np.bmat(
        [np.zeros([n_gates-filter_length + 1, side_pad], dtype=float),
         M_matrix_middle, np.zeros(
             [n_gates-filter_length+1, side_pad], dtype=float)])
    Z_matrix = np.zeros([n_gates - filter_length + 1, n_gates])
    return np.bmat([[Identity, -1.0 * Identity], [Identity, Identity],
                    [Z_matrix, M_matrix]])


def construct_B_vectors(phidp_mod, z_mod, filt, coef=0.914, dweight=60000.0):
    """
    Construct B vectors. See Giangrande et al, 2012.

    Parameters
    ----------
    phidp_mod : 2D array
        Phi differential phases.
    z_mod : 2D array.
       Reflectivity, modified as needed.
    filt : array
        Input filter.
    coef : float, optional.
        Cost coefficients.
    dweight : float, optional.
        Weights.

    Returns
    -------
    b : matrix
        Matrix containing B vectors.

    """
    n_gates = phidp_mod.shape[1]
    n_rays = phidp_mod.shape[0]
    filter_length = len(filt)
    side_pad = (filter_length - 1) // 2
    top_of_B_vectors = np.bmat([[-phidp_mod, phidp_mod]])
    data_edges = np.bmat([phidp_mod[:, 0:side_pad],
                          np.zeros([n_rays, n_gates-filter_length+1]),
                          phidp_mod[:, -side_pad:]])
    ii = filter_length - 1
    jj = data_edges.shape[1] - 1
    list_corrl = np.zeros([n_rays, jj - ii + 1])
    for count in range(list_corrl.shape[1]):
        list_corrl[:, count] = -1.0 * (
            np.array(filt) * (np.asarray(
                data_edges))[:, count:count+ii+1]).sum(axis=1)

    sct = (((10.0 ** (0.1 * z_mod)) ** coef / dweight))[:, side_pad: -side_pad]
    sct[np.where(sct < 0.0)] = 0.0
    sct[:, 0:side_pad] = list_corrl[:, 0:side_pad]
    sct[:, -side_pad:] = list_corrl[:, -side_pad:]
    B_vectors = np.bmat([[top_of_B_vectors, sct]])
    return B_vectors


def LP_solver_cvxopt(A_Matrix, B_vectors, weights, solver='glpk'):
    """
    Solve the Linear Programming problem given in Giangrande et al, 2012 using
    the CVXOPT module.

    Parameters
    ----------
    A_Matrix : matrix
        Row augmented A matrix, see :py:func:`construct_A_matrix`
    B_vectors : matrix
        Matrix containing B vectors, see :py:func:`construct_B_vectors`
    weights : array
        Weights.
    solver : str or None
        LP solver backend to use, choices are 'glpk', 'mosek' or None to use
        the conelp function in CVXOPT.  'glpk' and 'mosek' are only available
        if they are installed and CVXOPT was build with the correct bindings.

    Returns
    -------
    soln : array
        Solution to LP problem.

    See Also
    --------
    LP_solver_pyglpk : Solve LP problem using the PyGLPK module.
    LP_solver_cylp : Solve LP problem using the cylp module.
    LP_solver_cylp_mp : Solve LP problem using the cylp module
                        using multi processes.

    """
    from cvxopt import matrix, solvers
    n_gates = weights.shape[1] // 2
    n_rays = B_vectors.shape[0]
    mysoln = np.zeros([n_rays, n_gates])

    G = matrix(np.bmat([[-A_Matrix], [-np.eye(2 * n_gates)]]))
    h_array = np.zeros(5 * n_gates - 4)
    for raynum in range(n_rays):
        c = matrix(weights[raynum]).T
        h_array[:3 * n_gates - 4] = -B_vectors[raynum]
        h = matrix(h_array)
        sol = solvers.lp(c, G, h, solver=solver)
        # XXX when a solution is not found sol is None, need to check and
        # deal with this...

        if sol['x'] is None:
            continue

        # extract the solution
        this_soln = np.zeros(n_gates)
        for i in range(n_gates):
            this_soln[i] = sol['x'][i + n_gates]

        # apply smoothing filter and record in output array
        mysoln[raynum, :] = smooth_and_trim(this_soln, window_len=5,
                                            window='sg_smooth')
    return mysoln


def LP_solver_pyglpk(A_Matrix, B_vectors, weights, it_lim=7000, presolve=True,
                     really_verbose=False):
    """
    Solve the Linear Programming problem given in Giangrande et al, 2012 using
    the PyGLPK module.

    Parameters
    ----------
    A_Matrix : matrix
        Row augmented A matrix, see :py:func:`construct_A_matrix`
    B_vectors : matrix
        Matrix containing B vectors, see :py:func:`construct_B_vectors`
    weights : array
        Weights.
    it_lim : int, optional
        Simplex iteration limit.
    presolve : bool, optional
        True to use the LP presolver.
    really_verbose : bool, optional
        True to print LPX messaging. False to suppress.

    Returns
    -------
    soln : array
        Solution to LP problem.

    See Also
    --------
    LP_solver_cvxopt : Solve LP problem using the CVXOPT module.
    LP_solver_cylp : Solve LP problem using the cylp module.
    LP_solver_cylp_mp : Solve LP problem using the cylp module
                        using multi processes.

    """
    import glpk

    if really_verbose:
        message_state = glpk.LPX.MSG_ON
    else:
        message_state = glpk.LPX.MSG_OFF
    n_gates = weights.shape[1] // 2
    n_rays = B_vectors.shape[0]
    mysoln = np.zeros([n_rays, n_gates])
    lp = glpk.LPX()  # Create empty problem instance
    lp.name = 'LP_MIN'  # Assign symbolic name to problem
    lp.obj.maximize = False  # Set this as a maximization problem
    lp.rows.add(2 * n_gates + n_gates - 4)  # Append rows
    lp.cols.add(2 * n_gates)
    glpk.env.term_on = True
    for cur_row in range(2 * n_gates + n_gates - 4):
        lp.rows[cur_row].matrix = list(np.squeeze(np.asarray(
            A_Matrix[cur_row, :])))
    for i in range(2 * n_gates):
        lp.cols[i].bounds = 0.0, None
    for raynum in range(n_rays):
        this_soln = np.zeros(n_gates)
        for i in range(2 * n_gates + n_gates - 4):
            lp.rows[i].bounds = B_vectors[raynum, i], None
        for i in range(2 * n_gates):
            lp.obj[i] = weights[raynum, i]
        lp.simplex(msg_lev=message_state, meth=glpk.LPX.PRIMAL,
                   it_lim=it_lim, presolve=presolve)
        for i in range(n_gates):
            this_soln[i] = lp.cols[i+n_gates].primal
        mysoln[raynum, :] = smooth_and_trim(this_soln, window_len=5,
                                            window='sg_smooth')
    return mysoln


def solve_cylp(model, B_vectors, weights, ray, chunksize):
    """
    Worker process for LP_solver_cylp_mp.

    Parameters
    ----------
    model : CyClpModel
        Model of the LP Problem, see :py:func:`LP_solver_cylp_mp`
    B_vectors : matrix
        Matrix containing B vectors, see :py:func:`construct_B_vectors`
    weights : array
        Weights.
    ray : int
        Starting ray.
    chunksize : int
        Number of rays to process.

    Returns
    -------
    soln : array
        Solution to LP problem.

    See Also
    --------
    LP_solver_cylp_mp : Parent function.
    LP_solver_cylp : Single Process Solver.

    """
    from cylp.cy.CyClpSimplex import CyClpSimplex
    from cylp.py.modeling.CyLPModel import CyLPModel, CyLPArray

    n_gates = weights.shape[1] // 2
    soln = np.zeros([chunksize, n_gates])

    # import LP model in solver
    s = CyClpSimplex(model)

    # disable logging in multiprocessing anyway
    s.logLevel = 0

    i = 0
    for raynum in range(ray, ray + chunksize):
        # set new B_vector values for actual ray
        s.setRowLowerArray(np.squeeze(np.asarray(B_vectors[raynum])))
        # set new weights (objectives) for actual ray
        s.setObjectiveArray(np.squeeze(np.asarray(weights[raynum])))
        # solve with dual method, it is faster
        s.dual()
        # extract primal solution
        soln[i, :] = s.primalVariableSolution['x'][n_gates: 2 * n_gates]
        i = i + 1

    return soln


def LP_solver_cylp_mp(A_Matrix, B_vectors, weights, really_verbose=False,
                      proc=1):
    """
    Solve the Linear Programming problem given in Giangrande et al, 2012 using
    the CyLP module using multiple processes.

    Parameters
    ----------
    A_Matrix : matrix
        Row augmented A matrix, see :py:func:`construct_A_matrix`
    B_vectors : matrix
        Matrix containing B vectors, see :py:func:`construct_B_vectors`
    weights : array
        Weights.
    really_verbose : bool, optional
        True to print CLP messaging. False to suppress.
    proc : int, optional
        Number of worker processes.

    Returns
    -------
    soln : array
        Solution to LP problem.

    See Also
    --------
    LP_solver_cvxopt : Solve LP problem using the CVXOPT module.
    LP_solver_pyglpk : Solve LP problem using the PyGLPK module.
    LP_solver_cylp : Solve LP problem using the CyLP module using single
                     process.

    """
    from cylp.cy.CyClpSimplex import CyClpSimplex
    from cylp.py.modeling.CyLPModel import CyLPModel, CyLPArray
    import multiprocessing as mp

    n_gates = weights.shape[1] // 2
    n_rays = B_vectors.shape[0]
    soln = np.zeros([n_rays, n_gates])

    # Create CyLPModel and initialize it
    model = CyLPModel()
    G = np.matrix(A_Matrix)
    h = CyLPArray(np.empty(B_vectors.shape[1]))
    x = model.addVariable('x', G.shape[1])
    model.addConstraint(G * x >= h)
    c = CyLPArray(np.empty(weights.shape[1]))
    model.objective = c * x

    chunksize = int(n_rays/proc)
    # check if equal sized chunks can be distributed to worker processes
    if n_rays % chunksize != 0:
        print("Problem of %d rays cannot be split to %d worker processes!\n\r"
              "Fallback to 1 process!" % (n_rays, proc))
        chunksize = n_rays  # fall back to one process
        proc = 1

    print("Calculating with %d processes, %d rays per chunk" %
          (proc, chunksize))

    def worker(model, B_vectors, weights, ray, chunksize, out_q):
        """
        The worker function, invoked in a process.
        The results are placed in a dictionary that's pushed to a queue.
        """
        outdict = {}
        iray = int(ray/chunksize)
        outdict[iray] = solve_cylp(model, B_vectors, weights, ray, chunksize)
        out_q.put(outdict)

    # Queue for LP solutions
    out_q = mp.Queue()
    procs = []

    # fire off worker processes
    for raynum in range(0, n_rays, chunksize):
        p = mp.Process(target=worker, args=(
            model, B_vectors, weights, raynum, chunksize, out_q))
        procs.append(p)
        p.start()

    # collecting results
    resultdict = {}
    for raynum in range(0, n_rays, chunksize):
        resultdict.update(out_q.get())

    # Wait for all worker processes to finish
    for p in procs:
        p.join()

    # copy results in output array
    for raynum in range(0, int(n_rays / chunksize)):
        soln[raynum * chunksize:raynum * chunksize + chunksize, :] = (
            resultdict[raynum])

    # apply smoothing filter to output array
    soln = smooth_and_trim_scan(soln, window_len=5, window='sg_smooth')

    return soln


def LP_solver_cylp(A_Matrix, B_vectors, weights, really_verbose=False):
    """
    Solve the Linear Programming problem given in Giangrande et al, 2012 using
    the CyLP module.

    Parameters
    ----------
    A_Matrix : matrix
        Row augmented A matrix, see :py:func:`construct_A_matrix`
    B_vectors : matrix
        Matrix containing B vectors, see :py:func:`construct_B_vectors`
    weights : array
        Weights.
    really_verbose : bool, optional
        True to print CLP messaging. False to suppress.

    Returns
    -------
    soln : array
        Solution to LP problem.

    See Also
    --------
    LP_solver_cvxopt : Solve LP problem using the CVXOPT module.
    LP_solver_pyglpk : Solve LP problem using the PyGLPK module.

    """
    from cylp.cy.CyClpSimplex import CyClpSimplex
    from cylp.py.modeling.CyLPModel import CyLPModel, CyLPArray

    n_gates = weights.shape[1] // 2
    n_rays = B_vectors.shape[0]
    soln = np.zeros([n_rays, n_gates])

    # Create CyLPModel and initialize it
    model = CyLPModel()
    G = np.matrix(A_Matrix)
    h = CyLPArray(np.empty(B_vectors.shape[1]))
    x = model.addVariable('x', G.shape[1])
    model.addConstraint(G * x >= h)
    c = CyLPArray(np.squeeze(weights[0]))
    model.objective = c * x

    # import model in solver
    s = CyClpSimplex(model)
    # disable logging
    if not really_verbose:
        s.logLevel = 0

    for raynum in range(n_rays):

        # set new B_vector values for actual ray
        s.setRowLowerArray(np.squeeze(np.asarray(B_vectors[raynum])))
        # set new weights (objectives) for actual ray
        # solve with dual method, it is faster
        s.dual()
        # extract primal solution
        soln[raynum, :] = s.primalVariableSolution['x'][n_gates: 2 * n_gates]

    # apply smoothing filter on a per scan basis
    soln = smooth_and_trim_scan(soln, window_len=5, window='sg_smooth')
    return soln


def phase_proc_lp(radar, offset, debug=False, self_const=60000.0,
                  low_z=10.0, high_z=53.0, min_phidp=0.01, min_ncp=0.5,
                  min_rhv=0.8, fzl=4000.0, sys_phase=0.0,
                  overide_sys_phase=False, nowrap=None, really_verbose=False,
                  LP_solver='cylp', refl_field=None, ncp_field=None,
                  rhv_field=None, phidp_field=None, kdp_field=None,
                  unf_field=None, window_len=35, proc=1, coef=0.914):
    """
    Phase process using a LP method [1].

    Parameters
    ----------
    radar : Radar
        Input radar.
    offset : float
        Reflectivity offset in dBz.
    debug : bool, optional
        True to print debugging information.
    self_const : float, optional
        Self consistency factor.
    low_z : float, optional
        Low limit for reflectivity. Reflectivity below this value is set to
        this limit.
    high_z : float, optional
        High limit for reflectivity. Reflectivity above this value is set to
        this limit.
    min_phidp : float, optional
        Minimum Phi differential phase.
    min_ncp : float, optional
        Minimum normal coherent power.
    min_rhv : float, optional
        Minimum copolar coefficient.
    fzl : float, optional
        Maximum altitude.
    sys_phase : float, optional
        System phase in degrees.
    overide_sys_phase : bool, optional
        True to use `sys_phase` as the system phase. False will calculate a
        value automatically.
    nowrap : int or None, optional
        Gate number to begin phase unwrapping. None will unwrap all phases.
    really_verbose : bool, optional
        True to print LPX messaging. False to suppress.
    LP_solver : 'pyglpk' or 'cvxopt', 'cylp', or 'cylp_mp', optional
        Module to use to solve LP problem. Default is 'cylp'.
    refl_field, ncp_field, rhv_field, phidp_field, kdp_field : str, optional
        Name of field in radar which contains the horizonal reflectivity,
        normal coherent power, copolar coefficient, differential phase shift,
        and differential phase. A value of None for any of these parameters
        will use the default field name as defined in the Py-ART configuration
        file.
    unf_field : str, optional
        Name of field which will be added to the radar object which will
        contain the unfolded differential phase. Metadata for this field
        will be taken from the phidp_field. A value of None will use
        the default field name as defined in the Py-ART configuration file.
    window_len : int, optional
        Length of Sobel window applied to PhiDP field when prior to
        calculating KDP.
    proc : int, optional
        Number of worker processes, only used when `LP_solver` is 'cylp_mp'.
    coef : float, optional
        Exponent linking Z to KDP in self consistency. kdp=(10**(0.1z))*coef

    Returns
    -------
    reproc_phase : dict
        Field dictionary containing processed differential phase shifts.
    sob_kdp : dict
        Field dictionary containing recalculated differential phases.

    References
    ----------
    [1] Giangrande, S.E., R. McGraw, and L. Lei. An Application of
    Linear Programming to Polarimetric Radar Differential Phase Processing.
    J. Atmos. and Oceanic Tech, 2013, 30, 1716.

    """
    # parse the field parameters
    if refl_field is None:
        refl_field = get_field_name('reflectivity')
    if ncp_field is None:
        ncp_field = get_field_name('normalized_coherent_power')
    if rhv_field is None:
        rhv_field = get_field_name('cross_correlation_ratio')
    if phidp_field is None:
        phidp_field = get_field_name('differential_phase')
    if kdp_field is None:
        kdp_field = get_field_name('specific_differential_phase')
    if unf_field is None:
        unf_field = get_field_name('unfolded_differential_phase')

    # prepare reflectivity field
    refl = deepcopy(radar.fields[refl_field]['data']) + offset
    is_low_z = (refl) < low_z
    is_high_z = (refl) > high_z
    refl[np.where(is_high_z)] = high_z
    refl[np.where(is_low_z)] = low_z
    z_mod = refl

    # unfold Phi_DP
    if debug:
        print('Unfolding')
    my_unf = get_phidp_unf(radar, ncp_lev=min_ncp, rhohv_lev=min_rhv,
                           debug=debug, ncpts=2, doc=None,
                           sys_phase=sys_phase, nowrap=nowrap,
                           overide_sys_phase=overide_sys_phase,
                           refl_field=refl_field, ncp_field=ncp_field,
                           rhv_field=rhv_field, phidp_field=phidp_field)
    my_new_ph = deepcopy(radar.fields[phidp_field])
    my_unf[:, -1] = my_unf[:, -2]
    my_new_ph['data'] = my_unf
    radar.fields.update({unf_field: my_new_ph})

    phidp_mod = deepcopy(radar.fields[unf_field]['data'])
    phidp_neg = phidp_mod < min_phidp
    phidp_mod[np.where(phidp_neg)] = min_phidp

    # process
    proc_ph = deepcopy(radar.fields[phidp_field])
    proc_ph['data'] = phidp_mod
    St_Gorlv_differential_5pts = [-.2, -.1, 0, .1, .2]
    for sweep in range(len(radar.sweep_start_ray_index['data'])):
        if debug:
            print("Doing ", sweep)
        end_gate, start_ray, end_ray = det_process_range(
            radar, sweep, fzl, doc=15)
        start_gate = 0

        if end_gate < 0:
            continue

        A_Matrix = construct_A_matrix(
            len(radar.range['data'][start_gate:end_gate]),
            St_Gorlv_differential_5pts)

        B_vectors = construct_B_vectors(
            phidp_mod[start_ray:end_ray, start_gate:end_gate],
            z_mod[start_ray:end_ray, start_gate:end_gate],
            St_Gorlv_differential_5pts, dweight=self_const,
            coef=coef)

        weights = np.ones(
            phidp_mod[start_ray:end_ray, start_gate:end_gate].shape)

        nw = np.bmat([weights, np.zeros(weights.shape)])

        if LP_solver == 'pyglpk':
            mysoln = LP_solver_pyglpk(A_Matrix, B_vectors, nw,
                                      really_verbose=really_verbose)
        elif LP_solver == 'cvxopt':
            mysoln = LP_solver_cvxopt(A_Matrix, B_vectors, nw)
        elif LP_solver == 'cylp':
            mysoln = LP_solver_cylp(A_Matrix, B_vectors, nw,
                                    really_verbose=really_verbose)
        elif LP_solver == 'cylp_mp':
            mysoln = LP_solver_cylp_mp(A_Matrix, B_vectors, nw,
                                       really_verbose=really_verbose,
                                       proc=proc)
        else:
            raise ValueError('unknown LP_solver:' + LP_solver)

        proc_ph['data'][start_ray:end_ray, start_gate:end_gate] = mysoln

    last_gates = proc_ph['data'][start_ray:end_ray, -16]
    proc_ph['data'][start_ray:end_ray, -16:] = \
        np.meshgrid(np.ones([16]), last_gates)[1]
    proc_ph['valid_min'] = 0.0          # XXX is this correct?
    proc_ph['valid_max'] = 400.0        # XXX is this correct?

    # prepare output
    sobel = 2. * np.arange(window_len)/(window_len - 1.0) - 1.0
    sobel = sobel/(abs(sobel).sum())
    sobel = sobel[::-1]
    gate_spacing = (radar.range['data'][1] - radar.range['data'][0]) / 1000.
    kdp = (scipy.ndimage.filters.convolve1d(proc_ph['data'], sobel, axis=1) /
           ((window_len / 3.0) * 2.0 * gate_spacing))

    # copy the KDP metadata from existing field or create anew
    if kdp_field in radar.fields:
        sob_kdp = deepcopy(radar.fields[kdp_field])
    else:
        sob_kdp = get_metadata(kdp_field)

    sob_kdp['data'] = kdp
    sob_kdp['_FillValue'] = get_fillvalue()

    return proc_ph, sob_kdp


def _det_sys_phase(ncp, rhv, phidp, last_ray_idx, ncp_lev=0.4,
                   rhv_lev=0.6):
    """ Determine the system phase, see :py:func:`det_sys_phase`. """
    good = False
    phases = []
    for radial in range(last_ray_idx + 1):
        meteo = np.logical_and(ncp[radial, :] > ncp_lev,
                               rhv[radial, :] > rhv_lev)
        mpts = np.where(meteo)
        if len(mpts[0]) > 25:
            good = True
            msmth_phidp = smooth_and_trim(phidp[radial, mpts[0]], 9)
            phases.append(msmth_phidp[0:25].min())
    if not good:
        return None
    return np.median(phases)


def _det_sys_phase_ray(phidp, refl, nrays, ngates, ind_rmin=10, ind_rmax=500,
                       min_rcons=11, zmin=20., zmax=40.):
    """
    Private method
    Alternative determination of the system phase.
    Assumes that the valid gates of phidp are only precipitation.
    A system phase value is found for each ray.

    Parameters
    ----------
    phidp : masked array
        the phidp data
    refl : masked array
        the reflectivity data
    nrays : int
        number of rays in phidp
    ngates : int
        number of gates per ray
    ind_rmin, ind_rmax : int
        Min and max range index where to look for continuous precipitation
    min_rcons : int
        The minimum number of consecutive gates to consider it a rain cell.
    zmin, zmax : float

    Returns
    -------
    phidp0 : array of floats
        Estimate of the system phase at each ray
    first_gates : array of ints
        The first gate where PhiDP is valid

    """
    # initialize output
    phidp0 = np.ma.zeros((nrays, ngates), dtype='float64')
    phidp0[:] = np.ma.masked
    first_gates = np.zeros((nrays, ngates), dtype=int)-1

    # select data to analyse
    phidp_aux = np.ma.masked_where(
        np.logical_or(refl < zmin, refl > zmax), phidp)
    phidp_aux = phidp_aux[:, ind_rmin:ind_rmax]

    deg2rad = np.pi/180.
    # we want an odd window
    if min_rcons % 2 == 0:
        min_rcons += 1
    half_rcons = int((min_rcons-1)/2)
    for ray in range(nrays):
        # split ray in consecutive valid range bins
        isprec = np.ma.getmaskarray(phidp_aux[ray, :]) == 0
        ind_prec = np.where(isprec)[0]
        cons_list = np.split(ind_prec, np.where(np.diff(ind_prec) != 1)[0]+1)

        # check if there is a cell long enough
        found_cell = False
        for ind_prec_cell in cons_list:
            if len(ind_prec_cell) >= min_rcons:
                found_cell = True
                ind_prec_cell = ind_prec_cell[0:min_rcons-1]
                break
        # compute phidp0 as the average in sine and cosine
        if found_cell:
            first_gates[ray, :] = ind_prec_cell[0]+half_rcons+ind_rmin
            phidp0[ray, :] = np.arctan2(
                np.sum(np.sin(phidp_aux[ray, ind_prec_cell]*deg2rad)),
                np.sum(np.cos(phidp_aux[ray, ind_prec_cell]*deg2rad)))/deg2rad

    return phidp0, first_gates


def _correct_sys_phase(phidp, refl, nsweeps, nrays, ngates, start_sweep,
                       end_sweep, ind_rmin=10, ind_rmax=500, min_rcons=11,
                       zmin=20., zmax=40.):
    """
    correction of the system offset. Private method

    Parameters
    ----------
    phidp : masked array
        the phidp field to correct
    refl : masked array
        the reflectivity field
    nsweeps, nrays, ngates : int
        number of sweeps, total rays and gates per ray
    start_sweep, end_sweep : int array
        index of the starting and ending ray of each sweep
    ind_rmin, ind_rmax : int
        the minimum and maximum range indexes to use in the estimation
    min_rcons : int
        the number of consecutive range bins to consider a precipitation cell
        valid

    Returns
    -------
    corr_phidp : masked array
        The corrected phidp field

    """
    mask = np.ma.getmaskarray(phidp)

    # estimate system phase at each ray
    phidp0, first_gates = _det_sys_phase_ray(
        phidp, refl, nrays, ngates, ind_rmin=ind_rmin,
        ind_rmax=ind_rmax, min_rcons=min_rcons, zmin=zmin, zmax=zmax)

    # check if there are invalid Phidp0
    mask_phidp0 = np.ma.getmaskarray(phidp0[:, 0])
    ind_invalid = np.where(mask_phidp0)
    ninvalid = np.size(ind_invalid)

    # if there are gaps in the data we can
    # extract information from the neighbours
    if ninvalid > 0:
        for sweep in range(nsweeps):
            # check if there are gaps in the sweep
            start = start_sweep[sweep]
            end = end_sweep[sweep]

            ind_invalid_sweep = np.where(mask_phidp0[start:end])+start
            ninvalid_sweep = np.size(ind_invalid_sweep)
            if ninvalid_sweep > 0:
                # check if there are valid estimations in sweep
                ind_valid_sweep = (
                    np.where(mask_phidp0[start:end] == 0) + start)
                nvalid_sweep = np.size(ind_valid_sweep)

                if nvalid_sweep > 0:
                    # if there are valid estimations compute the median
                    phidp0[ind_invalid_sweep, :] = np.median(
                        phidp0[ind_valid_sweep, 0])
                    first_gates[ind_invalid_sweep, :] = ind_rmin
                else:
                    # if not compute the median of the valid phidp.
                    # if the median is valid set phidp0 to the median.
                    # Otherwise set to 0
                    phidp_median = np.ma.asarray(np.ma.median(
                        phidp[start:end, :]))
                    if phidp_median.mask is False:
                        phidp0[ind_invalid_sweep, :] = phidp_median
                        first_gates[ind_invalid_sweep] = ind_rmin
                    else:
                        phidp0[ind_invalid_sweep, :] = 0.
                        first_gates[ind_invalid_sweep] = ind_rmin

    # correct phidp of system offset
    corr_phidp = deepcopy(phidp)
    corr_phidp = phidp-phidp0

    for ray in range(nrays):
        corr_phidp[ray, 0:first_gates[ray, 0]] = 0.

    corr_phidp = np.ma.masked_where(mask, corr_phidp)

    return corr_phidp


def phase_proc_lp_gf(radar, gatefilter=None, debug=False, self_const=60000.0,
                     low_z=10.0, high_z=53.0, min_phidp=0.01, fzl=4000.0,
                     system_phase=None, nowrap=None, really_verbose=False,
                     LP_solver='cylp', refl_field=None, phidp_field=None,
                     kdp_field=None, unf_field=None, window_len=35, proc=1,
                     coef=0.914, ncpts=None, first_gate_sysp=None, offset=0.0,
                     doc=0):
    """
    Phase process using a LP method [1] using Py-ART's Gatefilter.

    Parameters
    ----------
    radar : Radar
        Input radar.
    gatefilter : Gatefilter, optional
        Py-ART gatefilter object indicating where processing should be
        carried out
    debug : bool, optional
        True to print debugging information.
    self_const : float, optional
        Self consistency factor.
    low_z : float, optional
        Low limit for reflectivity. Reflectivity below this value is set to
        this limit.
    high_z : float, optional
        High limit for reflectivity. Reflectivity above this value is set to
        this limit.
    fzl : float, optional
        Maximum altitude.
    system_phase : float, optional
        System phase in degrees.
    nowrap : int or None, optional
        Gate number to begin phase unwrapping. None will unwrap all phases.
    really_verbose : bool, optional
        True to print LPX messaging. False to suppress.
    LP_solver : 'pyglpk' or 'cvxopt', 'cylp', or 'cylp_mp', optional
        Module to use to solve LP problem. Default is 'cylp'.
    refl_field, ncp_field, rhv_field, phidp_field, kdp_field : str, optional
        Name of field in radar which contains the horizonal reflectivity,
        normal coherent power, copolar coefficient, differential phase shift,
        and differential phase. A value of None for any of these parameters
        will use the default field name as defined in the Py-ART configuration
        file.
    unf_field : str, optional
        Name of field which will be added to the radar object which will
        contain the unfolded differential phase. Metadata for this field
        will be taken from the phidp_field. A value of None will use
        the default field name as defined in the Py-ART configuration file.
    window_len : int, optional
        Length of Sobel window applied to PhiDP field when prior to
        calculating KDP.
    proc : int, optional
        Number of worker processes, only used when `LP_solver` is 'cylp_mp'.
    coef : float, optional
        Exponent linking Z to KDP in self consistency. kdp=(10**(0.1z))*coef
    ncpts : int, optional
        Minimum number of points in a ray. Regions within a ray smaller than
        this or beginning before this gate number are excluded from unfolding.
    offset : float, optional
        Reflectivity offset to add in dBz.
    doc : int, optional
        Number of gates to "doc" off the end of a ray.

    Returns
    -------
    reproc_phase : dict
        Field dictionary containing processed differential phase shifts.
    sob_kdp : dict
        Field dictionary containing recalculated differential phases.

    References
    ----------
    [1] Giangrande, S.E., R. McGraw, and L. Lei. An Application of
    Linear Programming to Polarimetric Radar Differential Phase Processing.
    J. Atmos. and Oceanic Tech, 2013, 30, 1716.

    """
    # parse the field parameters
    if refl_field is None:
        refl_field = get_field_name('reflectivity')
    if phidp_field is None:
        phidp_field = get_field_name('differential_phase')
    if kdp_field is None:
        kdp_field = get_field_name('specific_differential_phase')
    if unf_field is None:
        unf_field = get_field_name('unfolded_differential_phase')

    # if there is no gatefilter included, include all

    if gatefilter is None:
        gatefilter = GateFilter(radar)
        gatefilter.include_all()

    # prepare reflectivity field
    refl = deepcopy(radar.fields[refl_field]['data']) + offset
    is_low_z = (refl) < low_z
    is_high_z = (refl) > high_z
    refl[np.where(is_high_z)] = high_z
    refl[np.where(is_low_z)] = low_z
    z_mod = refl

    # unfold Phi_DP
    if debug:
        print('Unfolding')

    my_unf = get_phidp_unf_gf(radar, gatefilter,
                              debug=debug, ncpts=ncpts,
                              sys_phase=system_phase, nowrap=nowrap,
                              phidp_field=phidp_field,
                              first_gate_sysp=first_gate_sysp)

    my_new_ph = deepcopy(radar.fields[phidp_field])
    my_unf[:, -1] = my_unf[:, -2]
    my_new_ph['data'] = my_unf
    radar.fields.update({unf_field: my_new_ph})

    phidp_mod = deepcopy(radar.fields[unf_field]['data'])
    phidp_neg = phidp_mod < min_phidp
    phidp_mod[np.where(phidp_neg)] = min_phidp

    # process
    proc_ph = deepcopy(radar.fields[phidp_field])
    proc_ph['data'] = phidp_mod
    St_Gorlv_differential_5pts = [-.2, -.1, 0, .1, .2]

    for sweep in range(len(radar.sweep_start_ray_index['data'])):
        if debug:
            print("Doing ", sweep)

        end_gate, start_ray, end_ray = det_process_range(
            radar, sweep, fzl, doc=doc)

        start_gate = 0

        A_Matrix = construct_A_matrix(
            len(radar.range['data'][start_gate:end_gate]),
            St_Gorlv_differential_5pts)

        B_vectors = construct_B_vectors(
            phidp_mod[start_ray:end_ray, start_gate:end_gate],
            z_mod[start_ray:end_ray, start_gate:end_gate],
            St_Gorlv_differential_5pts, dweight=self_const,
            coef=coef)

        weights = np.ones(
            phidp_mod[start_ray:end_ray, start_gate:end_gate].shape)

        nw = np.bmat([weights, np.zeros(weights.shape)])

        if LP_solver == 'pyglpk':
            mysoln = LP_solver_pyglpk(A_Matrix, B_vectors, nw,
                                      really_verbose=really_verbose)
        elif LP_solver == 'cvxopt':
            mysoln = LP_solver_cvxopt(A_Matrix, B_vectors, nw)
        elif LP_solver == 'cylp':
            mysoln = LP_solver_cylp(A_Matrix, B_vectors, nw,
                                    really_verbose=really_verbose)
        elif LP_solver == 'cylp_mp':
            mysoln = LP_solver_cylp_mp(A_Matrix, B_vectors, nw,
                                       really_verbose=really_verbose,
                                       proc=proc)
        else:
            raise ValueError('unknown LP_solver:' + LP_solver)

        proc_ph['data'][start_ray:end_ray, start_gate:end_gate] = mysoln

    last_gates = proc_ph['data'][start_ray:end_ray, -16]
    proc_ph['data'][start_ray:end_ray, -16:] = \
        np.meshgrid(np.ones([16]), last_gates)[1]
    proc_ph['valid_min'] = 0.0  # XXX is this correct?
    proc_ph['valid_max'] = 400.0  # XXX is this correct?

    # prepare output
    sobel = 2. * np.arange(window_len) / (window_len - 1.0) - 1.0
    sobel = sobel / (abs(sobel).sum())
    sobel = sobel[::-1]
    gate_spacing = (radar.range['data'][1] - radar.range['data'][0]) / 1000.
    kdp = (scipy.ndimage.filters.convolve1d(proc_ph['data'], sobel, axis=1) /
           ((window_len / 3.0) * 2.0 * gate_spacing))

    # copy the KDP metadata from existing field or create anew
    if kdp_field in radar.fields:
        sob_kdp = deepcopy(radar.fields[kdp_field])
    else:
        sob_kdp = get_metadata(kdp_field)

    sob_kdp['data'] = kdp
    sob_kdp['_FillValue'] = get_fillvalue()

    return proc_ph, sob_kdp


def get_phidp_unf_gf(radar, gatefilter, debug=False, ncpts=2, sys_phase=None,
                     nowrap=None, phidp_field=None, first_gate_sysp=None):
    """
    Get Unfolded Phi differential phase in areas not gatefiltered.

    Parameters
    ----------
    radar : Radar
        The input radar.
    gatefilter : GateFilter
        Only apply on areas incuded in the gatefilter
    debug : bool, optioanl
        True to print debugging information, False to supress printing.
    ncpts : int, optional
        Minimum number of points in a ray. Regions within a ray smaller than
        this or beginning before this gate number are excluded from
        calculations.
    doc : int or None, optional
        Index of first gate not to include in field data, None include all.
    sys_phase : float, optional
        System phase overide.
    nowrap : int or None, optional
        Gate number where unwrapping should begin. `None` will unwrap all
        gates.
    refl_field ncp_field, rhv_field, phidp_field : str, optional
        Field names within the radar object which represent the horizontal
        reflectivity, normal coherent power, the copolar coefficient, and the
        differential phase shift. A value of None for any of these parameters
        will use the default field name as defined in the Py-ART
        configuration file.

    Returns
    -------
    cordata : array
        Unwrapped phi differential phase.

    """
    # parse the field parameters
    if phidp_field is None:
        phidp_field = get_field_name('differential_phase')

    t = time()
    my_phidp = radar.fields[phidp_field]['data']
    if sys_phase is not None:
        system_zero = sys_phase
    else:
        system_zero = det_sys_phase_gf(radar, gatefilter,
                                       phidp_field=phidp_field,
                                       first_gate=first_gate_sysp)
        if system_zero is None:
            system_zero = -135
    cordata = np.zeros(my_phidp.shape, dtype=float)
    all_non_meteo = gatefilter.gate_excluded
    for radial in range(my_phidp.shape[0]):

        # deterimine non meteo from gatefilter CHANGED
        notmeteo = all_non_meteo[radial, :]
        x_ma = ma.masked_where(notmeteo, my_phidp[radial, :])
        try:
            ma.notmasked_contiguous(x_ma)
            for slc in ma.notmasked_contiguous(x_ma):
                # so trying to get rid of clutter and small things that
                # should not add to phidp anyway
                if slc.stop - slc.start < ncpts or slc.start < ncpts:
                    x_ma.mask[slc.start - 1:slc.stop + 1] = True
            c = 0
        except TypeError:  # non sequence, no valid regions
            c = 1  # ie do nothing
            x_ma.mask[:] = True
        except AttributeError:
            c = 1  # also do nothing
            x_ma.mask = True
        if 'nowrap' is not None:
            # Start the unfolding a bit later in order to avoid false
            # jumps based on clutter
            unwrapped = deepcopy(x_ma)
            end_unwrap = unwrap_masked(x_ma[nowrap::], centered=False)
            unwrapped[nowrap::] = end_unwrap
        else:
            unwrapped = unwrap_masked(x_ma, centered=False)

        # end so no clutter expected
        system_max = unwrapped[np.where(np.logical_not(
            notmeteo))][-10:-1].mean() - system_zero
        unwrapped_fixed = np.zeros(len(x_ma), dtype=float)
        based = unwrapped-system_zero
        based[0] = 0.0
        notmeteo[0] = False
        based[-1] = system_max
        notmeteo[-1] = False
        unwrapped_fixed[np.where(np.logical_not(based.mask))[0]] = \
            based[np.where(np.logical_not(based.mask))[0]]
        if len(based[np.where(np.logical_not(based.mask))[0]]) > 11:
            unwrapped_fixed[np.where(based.mask)[0]] = \
                np.interp(np.where(based.mask)[0],
                          np.where(np.logical_not(based.mask))[0],
                          smooth_and_trim(based[np.where(
                              np.logical_not(based.mask))[0]]))
        else:
            unwrapped_fixed[np.where(based.mask)[0]] = \
                np.interp(np.where(based.mask)[0],
                          np.where(np.logical_not(based.mask))[0],
                          based[np.where(np.logical_not(based.mask))[0]])
        if c != 1:
            cordata[radial, :] = unwrapped_fixed
        else:
            cordata[radial, :] = np.zeros(my_phidp.shape[1])
    if debug:
        print("Exec time: ", time() - t)
    return cordata


def det_sys_phase_gf(radar, gatefilter, phidp_field=None, first_gate=30.):
    """
    Determine the system phase.

    Parameters
    ----------
    radar : Radar
        Radar object for which to determine the system phase.
    gatefilter : Gatefilter
        Gatefilter object highlighting valid gates.
    phidp_field : str, optional
        Field name within the radar object which represents
        differential phase shift. A value of None will use the default
        field name as defined in the Py-ART configuration file.
    first_gate : int, optional
        Gate index for where to being applying the gatefilter.

    Returns
    -------
    sys_phase : float or None
        Estimate of the system phase. None is not estimate can be made.

    """
    # parse the field parameters
    if phidp_field is None:
        phidp_field = get_field_name('differential_phase')
    phidp = radar.fields[phidp_field]['data'][:, first_gate:]
    last_ray_idx = radar.sweep_end_ray_index['data'][0]
    is_meteo = gatefilter.gate_included[:, first_gate:]
    return _det_sys_phase_gf(phidp, last_ray_idx, is_meteo)


def _det_sys_phase_gf(phidp, last_ray_idx, radar_meteo):
    """ Determine the system phase, see :py:func:`det_sys_phase`. """
    good = False
    phases = []
    for radial in range(last_ray_idx + 1):
        meteo = radar_meteo[radial, :]
        mpts = np.where(meteo)
        if len(mpts[0]) > 25:
            good = True
            msmth_phidp = smooth_and_trim(phidp[radial, mpts[0]], 9)
            phases.append(msmth_phidp[0:25].min())
    if not good:
        return None
    return np.median(phases)
