pyqt_fit.plot_fit — PyQt-Fit 1.2.4 documentation

### Navigation

- index
- modules |
- PyQt-Fit 1.2.4 documentation »
- Module code »

# Source code for pyqt_fit.plot_fit

    """
    :Author: Pierre Barbier de Reuille <pierre.barbierdereuille@gmail.com>

    This modules implement functions to test and plot parametric regression.
    """

    from __future__ import division, print_function, absolute_import
    from numpy import argsort, std, abs, sqrt, arange, pi, c_
    from pylab import figure, title, legend, plot, xlabel, ylabel, subplot, ylim, hist, suptitle, gca
    from .compat import izip
    from itertools import chain
    from scipy.special import erfinv, gamma
    from scipy import stats
    from .kernel_smoothing import LocalLinearKernel1D
    from .compat import unicode_csv_writer as csv_writer
    from collections import namedtuple

    smoothing = LocalLinearKernel1D

    import sys
    if sys.version_info >= (3,):
        CSV_WRITE_FLAGS = "wt"
    else:
        CSV_WRITE_FLAGS = "wb"

    def plot_dist_residuals(res):
        """
        Plot the distribution of the residuals.

        :returns: the handle toward the histogram and the plot of the fitted normal distribution
        """
        ph = hist(res, normed=True)
        xr = arange(res.min(), res.max(), (res.max() - res.min()) / 1024)
        yr = stats.norm(0, res.std()).pdf(xr)
        pn = plot(xr, yr, 'r--')
        xlabel('Residuals')
        ylabel('Frequency')
        title('Distributions of the residuals')
        return ph, pn

    def plot_residuals(xname, xdata, res_desc, res):
        """
        Plot the residuals against the X axis

        :param str     xname:    Name of the X axis
        :param ndarray xdata:    1D array with the X data
        :param str     res_desc: Name of the Y axis
        :param ndarray res: 1D   array with the residuals

        The shapes of ``xdata`` and ``res`` must be the same

        :returns: The handles of the the plots of the residuals and of the smoothed residuals.
        """
        p_res = plot(xdata, res, '+', label='residuals')[0]
        plot([xdata.min(), xdata.max()], [0, 0], 'r--')
        av = LocalLinearKernel1D(xdata, res)
        xr = arange(xdata.min(), xdata.max(), (xdata.max() - xdata.min()) / 1024)
        rr = av(xr)
        p_smooth = plot(xr, rr, 'g', label='smoothed residuals')
        xlabel(xname)
        ylabel("Residuals")
        ymin, ymax = ylim()
        ymax = max(ymax, -ymin)
        ylim(-ymax, ymax)
        title("Residuals (%s) vs. fitted" % (res_desc,))
        return p_res, p_smooth

    def scaled_location_plot(yname, yopt, scaled_res):
        """
        Plot the scaled location, given the dependant values and scaled residuals.

        :param str     yname:      Name of the Y axis
        :param ndarray yopt:       Estimated values
        :param ndarray scaled_res: Scaled residuals

        :returns: the handles for the data and the smoothed curve
        """

        scr = sqrt(abs(scaled_res))
        p_scaled = plot(yopt, scr, '+')[0]
        av = LocalLinearKernel1D(yopt, scr)
        xr = arange(yopt.min(), yopt.max(), (yopt.max() - yopt.min()) / 1024)
        rr = av(xr)
        p_smooth = plot(xr, rr, 'g')[0]
        expected_mean = 2 ** (1 / 4) * gamma(3 / 4) / sqrt(pi)
        plot([yopt.min(), yopt.max()], [expected_mean, expected_mean], 'r--')
        title('Scale-location')
        xlabel(yname)
        ylabel('$|$Normalized residuals$|^{1/2}$')
        gca().set_yticks([0, 1, 2])
        return [p_scaled, p_smooth]

    def qqplot(scaled_res, normq):
        """
        Draw a Q-Q Plot from the sorted, scaled residuals (i.e. residuals sorted
        and normalized by their standard deviation)

        :param ndarray scaled_res: Scaled residuals
        :param ndarray normq:      Expected value for each scaled residual, based on its quantile.

        :returns: handle to the data plot
        """
        qqp = []
        qqp += plot(normq, scaled_res, '+')
        qqp += plot(normq, normq, 'r--')
        xlabel('Theoretical quantiles')
        ylabel('Normalized residuals')
        title('Normal Q-Q plot')
        return qqp

    ResultStruct = namedtuple('ResultStruct', "fct fct_desc param_names xdata ydata xname yname "
                              "res_name residuals popt res yopts eval_points interpolation "
                              "sorted_yopts scaled_res normq CI CIs CIresults")

    def fit_evaluation(fit, xdata, ydata, eval_points=None,
                       CI=(), CIresults = None, xname="X", yname="Y",
                       fct_desc=None, param_names=(), residuals=None, res_name='Standard'):
        """
        This function takes the output of a curve fitting experiment and store all the relevant
        information for evaluating its success in the result.

        :type  fit: fitting object
        :param fit: object configured for the fitting

        :type  xdata: ndarray of shape (N,) or (k,N) for function with k prefictors
        :param xdata: The independent variable where the data is measured

        :type  ydata: ndarray
        :param ydata: The dependant data

        :type  eval_points: ndarray or None
        :param eval_points: Contain the list of points on which the result must be expressed. It is
            used both for plotting and for the bootstrapping.

        :type  CI: tuple of int
        :param CI: List of confidence intervals to calculate. If empty, none are calculated.

        :type  xname: string
        :param xname: Name of the X axis

        :type  yname: string
        :param yname: Name of the Y axis

        :type  fct_desc: string
        :param fct_desc: Formula of the function

        :type  param_names: tuple of strings
        :param param_names: Name of the various parameters

        :type  residuals: callable
        :param residuals: Residual function

        :type  res_desc: string
        :param res_desc: Description of the residuals

        :rtype: :py:class:`ResultStruct`
        :returns: Data structure summarising the fitting and its evaluation
        """
        popt = fit.popt
        res = fit.res

        if CI:
            CIs = CIresults.CIs
        else:
            CIs = []

        yopts = fit(xdata)
        if eval_points is None:
            yvals = yopts
            eval_points = xdata
        else:
            yvals = fit(eval_points)

        scaled_res, res_IX, prob, normq = residual_measures(res)
        sorted_yopts = yopts[res_IX]

        result = {}
        result["fct"] = fit
        result["fct_desc"] = fct_desc
        result["param_names"] = param_names
        result["xdata"] = xdata
        result["ydata"] = ydata
        result["xname"] = xname
        result["yname"] = yname
        result["res_name"] = res_name
        result["residuals"] = residuals
        #result["args"] = fit.args
        result["popt"] = popt
        result["res"] = res
        result["yopts"] = yopts
        result["eval_points"] = eval_points
        result["interpolation"] = yvals
        result["sorted_yopts"] = sorted_yopts
        result["scaled_res"] = scaled_res
        result["normq"] = normq
        result["CI"] = CI
        result["CIs"] = CIs
        #result["CIparams"] = CIparams
        result["CIresults"] = CIresults
        #print("estimate jacobian = %s" % result["extra_output"][-1]["est_jacobian"])
        return ResultStruct(**result)

    ResidualMeasures = namedtuple("ResidualMeasures", "scaled_res res_IX prob normq")

    def residual_measures(res):
        """
        Compute quantities needed to evaluate the quality of the estimation, based solely
        on the residuals.

        :rtype: :py:class:`ResidualMeasures`
        :returns: the scaled residuals, their ordering, the theoretical quantile for each residuals,
            and the expected value for each quantile.
        """
        IX = argsort(res)
        scaled_res = res[IX] / std(res)

        prob = (arange(len(scaled_res)) + 0.5) / len(scaled_res)
        normq = sqrt(2) * erfinv(2 * prob - 1)

        return ResidualMeasures(scaled_res, IX, prob, normq)

    _restestfields = "res_figure residuals scaled_residuals qqplot dist_residuals"
    ResTestResult = namedtuple("ResTestResult", _restestfields)
    Plot1dResult = namedtuple("Plot1dResult", "figure estimate data CIs " + _restestfields)

    def plot1d(result, loc=0, fig=None, res_fig=None):
        """
        Use matplotlib to display the result of a fit, and return the list of plots used

        :rtype: :py:class:`Plot1dResult`
        :returns: hangles to the various figures and plots
        """
        if fig is None:
            fig = figure()
        else:
            try:
                figure(fig)
            except TypeError:
                figure(fig.number)

        p_est = plot(result.eval_points, result.interpolation, label='estimated')[0]
        p_data = plot(result.xdata, result.ydata, '+', label='data')[0]
        p_CIs = []
        if result.CI:
            for p, (low, high) in izip(result.CI, result.CIs[0]):
                l = plot(result.eval_points, low, '--', label='%g%% CI' % (p,))[0]
                h = plot(result.eval_points, high, l.get_color() + '--')[0]
                p_CIs += [l, h]
        if result.param_names:
            param_strs = ", ".join("%s=%g" % (n, v) for n, v in izip(result.param_names, result.popt))
        else:
            param_strs = ", ".join("%g" % v for v in result.popt)
        param_strs = "$%s$" % (param_strs,)

        title("Estimated function %s with params %s" % (result.fct_desc, param_strs))

        xlabel(result.xname)
        ylabel(result.yname)
        legend(loc=loc)

        plots = {"figure": fig, "estimate": p_est, "data": p_data, "CIs": p_CIs}

        prt = plot_residual_tests(result.xdata, result.yopts, result.res,
                                  "{0} with params {1}".format(result.fct_desc, param_strs),
                                  result.xname, result.yname, result.res_name, result.sorted_yopts,
                                  result.scaled_res,
                                  result.normq, res_fig)

        plots.update(prt._asdict())

        return Plot1dResult(**plots)

    def plot_residual_tests(xdata, yopts, res, fct_name, xname="X", yname='Y', res_name="residuals",
                            sorted_yopts=None, scaled_res=None, normq=None, fig=None):
        """
        Plot, in a single figure, all four residuals evaluation plots: :py:func:`plot_residuals`,
        :py:func:`plot_dist_residuals`, :py:func:`scaled_location_plot` and :py:func:`qqplot`.

        :param ndarray xdata:        Explaining variables
        :param ndarray yopt:         Optimized explained variables
        :param str     fct_name:     Name of the fitted function
        :param str     xname:        Name of the explaining variables
        :param str     yname:        Name of the dependant variables
        :param str     res_name:     Name of the residuals
        :param ndarray sorted_yopts: ``yopt``, sorted to match the scaled residuals
        :param ndarray scaled_res:   Scaled residuals
        :param ndarray normq:        Estimated value of the quantiles for a normal distribution

        :type  fig: handle or None
        :param fig: Handle of the figure to put the plots in, or None to create a new figure

        :rtype: :py:class:`ResTestResult`
        :returns: The handles to all the plots
        """
        if fig is None:
            fig = figure()
        else:
            try:
                figure(fig)
            except TypeError:
                figure(fig.number)

        subplot(2, 2, 1)
    # First subplot is the residuals
        if len(xdata.shape) == 1 or xdata.shape[1] == 1:
            p_res = plot_residuals(xname, xdata.squeeze(), res_name, res)
        else:
            p_res = plot_residuals(yname, yopts, res_name, res)

        if scaled_res is None or sorted_yopts is None or normq is None:
            scaled_res, res_IX, _, normq = residual_measures(res)
            sorted_yopts = yopts[res_IX]

        subplot(2, 2, 2)
        p_scaled = scaled_location_plot(yname, sorted_yopts, scaled_res)

        subplot(2, 2, 3)
    # Q-Q plot
        qqp = qqplot(scaled_res, normq)

        subplot(2, 2, 4)
    # Distribution of residuals
        drp = plot_dist_residuals(res)

        suptitle("Residual Test for {}".format(fct_name))

        return ResTestResult(fig, p_res, p_scaled, qqp, drp)

    def write1d(outfile, result, res_desc, CImethod):
        """
        Write the result of a fitting and its evaluation to a CSV file.

        :param str          outfile:  Name of the file to write to
        :param ResultStruct result:   Result of the fitting evaluation
            (e.g. output of :py:func:`fit_evaluation`)
        :param str          res_desc: Description of the residuals
            (in more details than just the name of the residuals)
        :param str          CImethod: Description of the confidence interval estimation method
        """
        with open(outfile, CSV_WRITE_FLAGS) as f:
            w = csv_writer(f)
            w.writerow(["Function", result.fct.fct.description])
            w.writerow(["Residuals", result.res_name, res_desc])
            w.writerow(["Parameter", "Value"])
            for pn, pv in izip(result.param_names, result.popt):
                w.writerow([pn, "%.20g" % pv])
            #TODO w.writerow(["Regression Evaluation"])
            w.writerow([])
            w.writerow(["Data"])
            w.writerow([result.xname, result.yname, result.fct_desc, "Residuals: %s" % result.res_name])
            w.writerows(c_[result.xdata, result.ydata, result.yopts, result.res])
            w.writerow([])
            w.writerow(['Model validation'])
            w.writerow([result.yname, 'Normalized residuals', 'Theoretical quantiles'])
            w.writerows(c_[result.sorted_yopts, result.scaled_res, result.normq])
            if result.eval_points is not result.xdata:
                w.writerow([])
                w.writerow(["Interpolated data"])
                w.writerow([result.xname, result.yname])
                w.writerows(c_[result.eval_points, result.interpolation])
            if result.CI:
                w.writerow([])
                w.writerow(["Confidence interval"])
                w.writerow(["Method", CImethod])
                head = ["Parameters"] + \
                    list(chain(*[["%g%% - low" % v, "%g%% - high" % v] for v in result.CI]))
                w.writerow(head)
                #print(result.CIs[1])
                for cis in izip(result.param_names, *chain(*result.CIs[1])):
                    cistr = [cis[0]] + ["%.20g" % v for v in cis[1:]]
                    w.writerow(cistr)
                w.writerow([result.yname])
                head[0] = result.xname
                w.writerow(head)
                w.writerows(c_[tuple(chain([result.eval_points], *result.CIs[0]))])

    # /home/barbier/prog/python/curve_fitting/test.csv

### Navigation

- index
- modules |
- PyQt-Fit 1.2.4 documentation »
- Module code »