% Generated by Sphinx.
\def\sphinxdocclass{report}
\documentclass[letterpaper,10pt,english]{sphinxmanual}
\usepackage[utf8]{inputenc}
\DeclareUnicodeCharacter{00A0}{\nobreakspace}
\usepackage[T1]{fontenc}
\usepackage{babel}
\usepackage{times}
\usepackage[Bjarne]{fncychap}
\usepackage{longtable}
\usepackage{sphinx}
\usepackage{multirow}


\title{Least-Squares Minimization with Constraints for Python}
\date{September 19, 2012}
\release{0.7}
\author{Matthew Newville}
\newcommand{\sphinxlogo}{}
\renewcommand{\releasename}{Release}
\makeindex

\makeatletter
\def\PYG@reset{\let\PYG@it=\relax \let\PYG@bf=\relax%
    \let\PYG@ul=\relax \let\PYG@tc=\relax%
    \let\PYG@bc=\relax \let\PYG@ff=\relax}
\def\PYG@tok#1{\csname PYG@tok@#1\endcsname}
\def\PYG@toks#1+{\ifx\relax#1\empty\else%
    \PYG@tok{#1}\expandafter\PYG@toks\fi}
\def\PYG@do#1{\PYG@bc{\PYG@tc{\PYG@ul{%
    \PYG@it{\PYG@bf{\PYG@ff{#1}}}}}}}
\def\PYG#1#2{\PYG@reset\PYG@toks#1+\relax+\PYG@do{#2}}

\expandafter\def\csname PYG@tok@gd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}}
\expandafter\def\csname PYG@tok@gu\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}}
\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.25,0.82}{##1}}}
\expandafter\def\csname PYG@tok@gs\endcsname{\let\PYG@bf=\textbf}
\expandafter\def\csname PYG@tok@gr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}}
\expandafter\def\csname PYG@tok@cm\endcsname{\let\PYG@it=\textit\def\PYG@tc##1{\textcolor[rgb]{0.25,0.50,0.56}{##1}}}
\expandafter\def\csname PYG@tok@vg\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.38,0.84}{##1}}}
\expandafter\def\csname PYG@tok@m\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.13,0.50,0.31}{##1}}}
\expandafter\def\csname PYG@tok@mh\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.13,0.50,0.31}{##1}}}
\expandafter\def\csname PYG@tok@cs\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.25,0.50,0.56}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
\expandafter\def\csname PYG@tok@ge\endcsname{\let\PYG@it=\textit}
\expandafter\def\csname PYG@tok@vc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.38,0.84}{##1}}}
\expandafter\def\csname PYG@tok@il\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.13,0.50,0.31}{##1}}}
\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.19,0.19,0.19}{##1}}}
\expandafter\def\csname PYG@tok@cp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}}
\expandafter\def\csname PYG@tok@gh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}}
\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.84,0.33,0.22}{##1}}}
\expandafter\def\csname PYG@tok@nl\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.13,0.44}{##1}}}
\expandafter\def\csname PYG@tok@nn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}}
\expandafter\def\csname PYG@tok@no\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.38,0.68,0.84}{##1}}}
\expandafter\def\csname PYG@tok@na\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.25,0.44,0.63}{##1}}}
\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
\expandafter\def\csname PYG@tok@nc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}}
\expandafter\def\csname PYG@tok@nd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.33,0.33,0.33}{##1}}}
\expandafter\def\csname PYG@tok@ne\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
\expandafter\def\csname PYG@tok@nf\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.02,0.16,0.49}{##1}}}
\expandafter\def\csname PYG@tok@si\endcsname{\let\PYG@it=\textit\def\PYG@tc##1{\textcolor[rgb]{0.44,0.63,0.82}{##1}}}
\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.25,0.44,0.63}{##1}}}
\expandafter\def\csname PYG@tok@vi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.38,0.84}{##1}}}
\expandafter\def\csname PYG@tok@nt\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.02,0.16,0.45}{##1}}}
\expandafter\def\csname PYG@tok@nv\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.38,0.84}{##1}}}
\expandafter\def\csname PYG@tok@s1\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.25,0.44,0.63}{##1}}}
\expandafter\def\csname PYG@tok@gp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}}
\expandafter\def\csname PYG@tok@sh\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.25,0.44,0.63}{##1}}}
\expandafter\def\csname PYG@tok@ow\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}}
\expandafter\def\csname PYG@tok@bp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
\expandafter\def\csname PYG@tok@c1\endcsname{\let\PYG@it=\textit\def\PYG@tc##1{\textcolor[rgb]{0.25,0.50,0.56}{##1}}}
\expandafter\def\csname PYG@tok@kc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
\expandafter\def\csname PYG@tok@c\endcsname{\let\PYG@it=\textit\def\PYG@tc##1{\textcolor[rgb]{0.25,0.50,0.56}{##1}}}
\expandafter\def\csname PYG@tok@mf\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.13,0.50,0.31}{##1}}}
\expandafter\def\csname PYG@tok@err\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\fcolorbox[rgb]{1.00,0.00,0.00}{1,1,1}{\strut ##1}}}
\expandafter\def\csname PYG@tok@kd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.32,0.47,0.09}{##1}}}
\expandafter\def\csname PYG@tok@sr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.14,0.33,0.53}{##1}}}
\expandafter\def\csname PYG@tok@mo\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.13,0.50,0.31}{##1}}}
\expandafter\def\csname PYG@tok@mi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.13,0.50,0.31}{##1}}}
\expandafter\def\csname PYG@tok@kn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
\expandafter\def\csname PYG@tok@o\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
\expandafter\def\csname PYG@tok@kr\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
\expandafter\def\csname PYG@tok@s\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.25,0.44,0.63}{##1}}}
\expandafter\def\csname PYG@tok@kp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
\expandafter\def\csname PYG@tok@w\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}}
\expandafter\def\csname PYG@tok@kt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.56,0.13,0.00}{##1}}}
\expandafter\def\csname PYG@tok@sc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.25,0.44,0.63}{##1}}}
\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.25,0.44,0.63}{##1}}}
\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
\expandafter\def\csname PYG@tok@se\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.25,0.44,0.63}{##1}}}
\expandafter\def\csname PYG@tok@sd\endcsname{\let\PYG@it=\textit\def\PYG@tc##1{\textcolor[rgb]{0.25,0.44,0.63}{##1}}}

\def\PYGZbs{\char`\\}
\def\PYGZus{\char`\_}
\def\PYGZob{\char`\{}
\def\PYGZcb{\char`\}}
\def\PYGZca{\char`\^}
\def\PYGZam{\char`\&}
\def\PYGZlt{\char`\<}
\def\PYGZgt{\char`\>}
\def\PYGZsh{\char`\#}
\def\PYGZpc{\char`\%}
\def\PYGZdl{\char`\$}
\def\PYGZti{\char`\~}
% for compatibility with earlier versions
\def\PYGZat{@}
\def\PYGZlb{[}
\def\PYGZrb{]}
\makeatother

\begin{document}

\maketitle
\tableofcontents
\phantomsection\label{index::doc}


The lmfit Python package provides a simple, flexible interface to
non-linear optimization or curve fitting problems.  The default
optimization method used is the \href{http://en.wikipedia.org/wiki/Levenberg-Marquardt\_algorithm}{Levenberg-Marquardt} algorithm from
\href{http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.leastsq.html}{scipy.optimize.leastsq}, but other optimization procedures such as
\href{http://en.wikipedia.org/wiki/Nelder-Mead\_method}{Nelder-Mead} downhill simplex, Powell's method, COBYLA, Sequential Least
Squares, and a few other optimization approaches available from
\href{http://docs.scipy.org/doc/scipy/reference/optimize.html}{scipy.optimize} can also be used.

For any optimization problem, the programmer must write an objective
function that takes a set of values for the variables in the fit, and
produces either a scalar value to be minimized or a residual array to be
minimized in the least-squares sense.  Generally, the set of variables can
all be varied freely by the algorithm calling the objective functions,
though some algorithms allow some sorts of constraints or bounds to be
placed on the values.

With the lmfit package, one writes an objective function in terms of a set
of Parameter objects, where a Parameter has the following properties:
\begin{itemize}
\item {} 
Parameters have values that can be fixed or floated in the fit.

\item {} 
Upper and/or lower bounds can be placed on Parameter values.

\item {} 
Parameters can be written as algebraic expressions of other Parameters.
These values will be re-evaluated at each step in the fit, giving a
a simple and very flexible approach to constraining fit variables.

\end{itemize}

The principle advantage of using Parameters instead of fit variables is
that the objective function does not have to be rewritten for a change in
what is varied or what constraints are placed on the fit.  A programmer can
write a general model that encapsulates the phenomenon to be optimized, and
then allow a user of the model to change what is varied and what
constraints are placed on the model.  The ability to easily change whether
a Parameter is floated or fixed also allows one to easily test the
significance of certain Parameters to the fitting model.  A second
advantage is that Parameters can be used and objective function using them
can be given to a number of fitting algorithms without any change.

By default, lmfit uses and the \href{http://en.wikipedia.org/wiki/Levenberg-Marquardt\_algorithm}{Levenberg-Marquardt} minimization
algorithm from \href{http://en.wikipedia.org/wiki/MINPACK}{MINPACK-1} as implemented in \href{http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.leastsq.html}{scipy.optimize.leastsq}.
This method is by far the most tested and best support method in lmfit, and
much of this document assumes this algorithm is used unless explicitly
stated. An important point for many scientific analysis is that this is
only method that automatically estimates uncertainties and correlations
between fitted variables from the covariance matrix used in the fit.

A few other optimization routines are also supported, including
\href{http://en.wikipedia.org/wiki/Nelder-Mead\_method}{Nelder-Mead} simplex downhill method as implemented in
\href{http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.fmin.html}{scipy.optimize.fmin}, and several others from \href{http://docs.scipy.org/doc/scipy/reference/optimize.html}{scipy.optimize}. Some
methods, including the \href{http://en.wikipedia.org/wiki/Limited-memory\_BFGS}{L-BFGS} (limited memory
Broyden-Fletcher-Goldfarb-Shanno) algorithm as implemented in
\href{http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.fmin\_l\_bfgs\_b.html}{scipy.optimize.l\_bfgs\_b} and the \href{http://en.wikipedia.org/wiki/Simulated\_annealing}{simulated annealing} algorithm as
implemented in \href{http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.anneal.html}{scipy.optimize.anneal} are implemented, but appear to not
work very well.  In their native form, some of these methods setting upper
or lower bounds on parameters, or adding constraints on fitted variables.
By using Parameter objects, lmfit allows bounds and constraints for all of
these methods, and makes it easy to swap between methods.

Finally, because this approach of using the covariance matrix to determine
uncertainties is sometimes questioned (and sometimes rightly so), lmfit
supports methods to do a brute force search of the confidence intervals and
correlations for sets of parameters.

lmfit and this document are a work in progress.


\chapter{Downloading and Installation}
\label{installation::doc}\label{installation:non-linear-least-square-minimization-for-python}\label{installation:downloading-and-installation}

\section{Prerequisites}
\label{installation:prerequisites}
The lmfit package requires Python, Numpy, and Scipy.  Scipy version 0.11 or
higher is recommended, but extensive testing on version compatibility has
not been done.  Initial tests do work with Python 3.2, but little testing
with Python 3 has yet been done.  No testing has been done with 64-bit
architectures, but as this package is pure Python, no significant troubles
are expected.

If installed, the \href{http://packages.python.org/uncertainties/}{uncertainties} package will be used for propogation of
uncertainties to constrained parameters.


\section{Downloads}
\label{installation:downloads}
The latest stable version is available from PyPI or CARS (Univ of Chicago):

\begin{tabular}{|p{0.317\linewidth}|p{0.317\linewidth}|p{0.317\linewidth}|}
\hline
\textbf{
Download Option
} & \textbf{
Python Versions
} & \textbf{
Location
}\\\hline

Source Kit
 & 
2.6, 2.7, 3.2
 & \begin{itemize}
\item {} 
\href{http://pypi.python.org/packages/source/l/lmfit/lmfit-0.7.tar.gz}{lmfit-0.7.tar.gz (PyPI)}

\item {} 
\href{http://cars9.uchicago.edu/software/python/lmfit/src/lmfit-0.7.tar.gz}{lmfit-0.7.tar.gz (CARS)}

\end{itemize}
\\\hline

Win32 Installer
 & 
2.6
 & \begin{itemize}
\item {} 
\href{http://pypi.python.org/packages/2.6/l/lmfit/lmfit-0.7.win32-py2.6.exe}{lmfit-0.7.win32-py2.6.exe (PyPI)}

\item {} 
\href{http://cars9.uchicago.edu/software/python/lmfit/src/lmfit-0.7.win32-py2.6.exe}{lmfit-0.7.win32-py2.6.exe (CARS)}

\end{itemize}
\\\hline

Win32 Installer
 & 
2.7
 & \begin{itemize}
\item {} 
\href{http://pypi.python.org/packages/2.7/l/lmfit/lmfit-0.7.win32-py2.7.exe}{lmfit-0.7.win32-py2.7.exe (PyPI)}

\item {} 
\href{http://cars9.uchicago.edu/software/python/lmfit/src/lmfit-0.7.win32-py2.7.exe}{lmfit-0.7.win32-py2.7.exe (CARS)}

\end{itemize}
\\\hline

Win32 Installer
 & 
3.2
 & \begin{itemize}
\item {} 
\href{http://pypi.python.org/packages/3.2/l/lmfit/lmfit-0.7.win32-py3.2.exe}{lmfit-0.7.win32-py3.2.exe (PyPI)}

\item {} 
\href{http://cars9.uchicago.edu/software/python/lmfit/src/lmfit-0.7.win32-py3.2.exe}{lmfit-0.7.win32-py3.2.exe (CARS)}

\end{itemize}
\\\hline

Development Version
 & 
all
 & 
use \href{http://github.com/newville/lmfit-py}{lmfit github repository}
\\\hline
\end{tabular}


if you have \href{http://pypi.python.org/pypi/setuptools}{Python Setup Tools}  installed, you can download and install
the lmfit-py Package simply with:

\begin{Verbatim}[commandchars=\\\{\}]
easy\_install -U lmfit
\end{Verbatim}


\section{Development Version}
\label{installation:development-version}
To get the latest development version, use:

\begin{Verbatim}[commandchars=\\\{\}]
git clone http://github.com/newville/lmfit-py.git
\end{Verbatim}


\section{Installation}
\label{installation:installation}
Installation from source on any platform is:

\begin{Verbatim}[commandchars=\\\{\}]
python setup.py install
\end{Verbatim}


\section{Acknowledgements}
\label{installation:acknowledgements}
LMFIT was originally written by Matthew Newville.  Substantial code and
documentation improvements, especially for improved estimates of confidence
intervals was provided by Till Stensitzki.  The implemenation of parameter
bounds as described in the MINUIT documentation is taken from Jonathan
J. Helmus' leastsqbound code, with permission. Many valuable suggestions
for improvements have come from Christoph Deil.  The code obviously depends
on, and owes a very large debt to the code in scipy.optimize.  Several
discussions on the scipy mailing lists have also led to improvements in
this code.


\section{License}
\label{installation:license}
The LMFIT-py code is distribution under the following license:
\begin{quote}
\begin{description}
\item[{Copyright (c) 2012 Matthew Newville, The University of Chicago}] \leavevmode
Till Stensitzki, Freie Universitat Berlin

\end{description}

Permission to use and redistribute the source code or binary forms of this
software and its documentation, with or without modification is hereby
granted provided that the above notice of copyright, these terms of use,
and the disclaimer of warranty below appear in the source code and
documentation, and that none of the names of above institutions or
authors appear in advertising or endorsement of works derived from this
software without specific prior written permission from all parties.

THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THIS SOFTWARE.
\end{quote}


\chapter{Getting started with Non-Linear Least-Squares Fitting}
\label{parameters::doc}\label{parameters:getting-started-with-non-linear-least-squares-fitting}
The lmfit package is designed to provide a simple way to build complex
fitting models and apply them to real data.  This chapter describes how to
set up and perform simple fits.  Some basic knowledge of Python, Numpy, and
modeling data are assumed.

To do a least-squares fit of a model to data, or for a host of other
optimization problems, the main task is to write an \emph{objective function}
that takes the values of the fitting variables and calculates either a
scalar value to be minimized or an array of values that is to be minimized
in the least-squares sense.  For many data fitting processes, the
least-squaers approach is used, and the the objective function should
return an array of (data-model), perhaps scaled by some weighting factor
such as the inverse of the uncertainty in the data.  For such a problem,
the chi-square ($\chi^2$) statistic is often defined as:
\begin{gather}
\begin{split}\chi^2 =  \sum_i^{N} \frac{[y^{\rm meas}_i - y_i^{\rm model}({\bf{v}})]^2}{\epsilon_i^2}\end{split}\notag
\end{gather}
where $y_i^{\rm meas}$ is the set of measured data, $y_i^{\rm
model}({\bf{v}})$ is the model calculation, ${\bf{v}}$ is the set of
variables in the model to be optimized in the fit, and $\epsilon_i$
is the estimated uncertainty in the data.

In a traditional non-linear fit, one writes a function that takes the
variable values and calculates the residual $y^{\rm meas}_i -
y_i^{\rm model}({\bf{v}})$, perhaps something like:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{k}{def} \PYG{n+nf}{residual}\PYG{p}{(}\PYG{n+nb}{vars}\PYG{p}{,} \PYG{n}{x}\PYG{p}{,} \PYG{n}{data}\PYG{p}{)}\PYG{p}{:}
    \PYG{n}{amp} \PYG{o}{=} \PYG{n+nb}{vars}\PYG{p}{[}\PYG{l+m+mi}{0}\PYG{p}{]}
    \PYG{n}{phaseshift} \PYG{o}{=} \PYG{n+nb}{vars}\PYG{p}{[}\PYG{l+m+mi}{1}\PYG{p}{]}
    \PYG{n}{freq} \PYG{o}{=} \PYG{n+nb}{vars}\PYG{p}{[}\PYG{l+m+mi}{2}\PYG{p}{]}
    \PYG{n}{decay} \PYG{o}{=} \PYG{n+nb}{vars}\PYG{p}{[}\PYG{l+m+mi}{3}\PYG{p}{]}

    \PYG{n}{model} \PYG{o}{=} \PYG{n}{amp} \PYG{o}{*} \PYG{n}{sin}\PYG{p}{(}\PYG{n}{x} \PYG{o}{*} \PYG{n}{freq}  \PYG{o}{+} \PYG{n}{phaseshift}\PYG{p}{)} \PYG{o}{*} \PYG{n}{exp}\PYG{p}{(}\PYG{o}{-}\PYG{n}{x}\PYG{o}{*}\PYG{n}{x}\PYG{o}{*}\PYG{n}{decay}\PYG{p}{)}

    \PYG{k}{return} \PYG{p}{(}\PYG{n}{data}\PYG{o}{-}\PYG{n}{model}\PYG{p}{)}
\end{Verbatim}

To perform the minimization with scipy, one would do:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{k+kn}{from} \PYG{n+nn}{scipy.optimize} \PYG{k+kn}{import} \PYG{n}{leastsq}
\PYG{n+nb}{vars} \PYG{o}{=} \PYG{p}{[}\PYG{l+m+mf}{10.0}\PYG{p}{,} \PYG{l+m+mf}{0.2}\PYG{p}{,} \PYG{l+m+mf}{3.0}\PYG{p}{,} \PYG{l+m+mf}{0.007}\PYG{p}{]}
\PYG{n}{out} \PYG{o}{=} \PYG{n}{leastsq}\PYG{p}{(}\PYG{n}{residual}\PYG{p}{,} \PYG{n+nb}{vars}\PYG{p}{,} \PYG{n}{args}\PYG{o}{=}\PYG{p}{(}\PYG{n}{x}\PYG{p}{,} \PYG{n}{data}\PYG{p}{)}\PYG{p}{)}
\end{Verbatim}

Though it is wonderful to be able to use python for such optimization
problems, and the scipy library is fairly easy to use, the approach here is
not terribly different from how one would do the same fit in C or Fortran.


\section{Using \texttt{Parameters} instead of Variables}
\label{parameters:parameters-label}\label{parameters:using-parameters-instead-of-variables}
As described above, there are several practical challenges in doing
least-squares fits and other optimizations with the traditional
implementation (Fortran, scipy.optimize.leastsq, and most other) in which a
list of fitting variables to the function to be minimized.  These
challenges include:
\begin{enumerate}
\item {} 
The user has to keep track of the order of the variables, and their
meaning -- vars{[}2{]} is the frequency, and so on.

\item {} 
If the user wants to fix a particular variable (\emph{not} vary it in the fit),
the residual function has to be altered.  While reasonable for simple
cases, this quickly becomes significant work for more complex models,
and greatly complicates modeling for people not intimately familiar
with the code.

\item {} 
There is no simple, robust way to put bounds on values for the
variables, or enforce mathematical relationships between the
variables.

\end{enumerate}

The lmfit module is designed to void these shortcomings.

The main idea of lmfit is to expand a numerical variable with a
{\hyperref[parameters:Parameter]{\code{Parameter}}}, which have more attributes than simply their value.
Instead of a pass a list of numbers to the function to minimize, you create
a {\hyperref[parameters:Parameters]{\code{Parameters}}} object, add parameters to this object, and pass along
this object to your function to be minimized.  With this transformation,
the above example would be translated to look like:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{k+kn}{from} \PYG{n+nn}{lmfit} \PYG{k+kn}{import} \PYG{n}{minimize}\PYG{p}{,} \PYG{n}{Parameters}

\PYG{k}{def} \PYG{n+nf}{residual}\PYG{p}{(}\PYG{n}{params}\PYG{p}{,} \PYG{n}{x}\PYG{p}{,} \PYG{n}{data}\PYG{p}{)}\PYG{p}{:}
    \PYG{n}{amp} \PYG{o}{=} \PYG{n}{params}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{amp}\PYG{l+s}{'}\PYG{p}{]}\PYG{o}{.}\PYG{n}{value}
    \PYG{n}{pshift} \PYG{o}{=} \PYG{n}{params}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{phase}\PYG{l+s}{'}\PYG{p}{]}\PYG{o}{.}\PYG{n}{value}
    \PYG{n}{freq} \PYG{o}{=} \PYG{n}{params}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{frequency}\PYG{l+s}{'}\PYG{p}{]}\PYG{o}{.}\PYG{n}{value}
    \PYG{n}{decay} \PYG{o}{=} \PYG{n}{params}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{decay}\PYG{l+s}{'}\PYG{p}{]}\PYG{o}{.}\PYG{n}{value}

    \PYG{n}{model} \PYG{o}{=} \PYG{n}{amp} \PYG{o}{*} \PYG{n}{sin}\PYG{p}{(}\PYG{n}{x} \PYG{o}{*} \PYG{n}{freq}  \PYG{o}{+} \PYG{n}{pshift}\PYG{p}{)} \PYG{o}{*} \PYG{n}{exp}\PYG{p}{(}\PYG{o}{-}\PYG{n}{x}\PYG{o}{*}\PYG{n}{x}\PYG{o}{*}\PYG{n}{decay}\PYG{p}{)}

    \PYG{k}{return} \PYG{p}{(}\PYG{n}{data}\PYG{o}{-}\PYG{n}{model}\PYG{p}{)}

\PYG{n}{params} \PYG{o}{=} \PYG{n}{Parameters}\PYG{p}{(}\PYG{p}{)}
\PYG{n}{params}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{amp}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value}\PYG{o}{=}\PYG{l+m+mi}{10}\PYG{p}{)}
\PYG{n}{params}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{decay}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value}\PYG{o}{=}\PYG{l+m+mf}{0.007}\PYG{p}{)}
\PYG{n}{params}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{phase}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value}\PYG{o}{=}\PYG{l+m+mf}{0.2}\PYG{p}{)}
\PYG{n}{params}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{frequency}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value}\PYG{o}{=}\PYG{l+m+mf}{3.0}\PYG{p}{)}

\PYG{n}{out} \PYG{o}{=} \PYG{n}{minimize}\PYG{p}{(}\PYG{n}{residual}\PYG{p}{,} \PYG{n}{params}\PYG{p}{,} \PYG{n}{args}\PYG{o}{=}\PYG{p}{(}\PYG{n}{x}\PYG{p}{,} \PYG{n}{data}\PYG{p}{)}\PYG{p}{)}
\end{Verbatim}

So far, this simply looks like it replaced a list of values with a
dictionary, accessed by name.  But each of the named {\hyperref[parameters:Parameter]{\code{Parameter}}} in
the {\hyperref[parameters:Parameters]{\code{Parameters}}} object hold additional attributes to modify the
value during the fit.  For example, Parameters can be fixed or bounded, and
this can be done when being defined:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{n}{params} \PYG{o}{=} \PYG{n}{Parameters}\PYG{p}{(}\PYG{p}{)}
\PYG{n}{params}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{amp}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value}\PYG{o}{=}\PYG{l+m+mi}{10}\PYG{p}{,} \PYG{n}{vary}\PYG{o}{=}\PYG{n+nb+bp}{False}\PYG{p}{)}
\PYG{n}{params}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{decay}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value}\PYG{o}{=}\PYG{l+m+mf}{0.007}\PYG{p}{,} \PYG{n+nb}{min}\PYG{o}{=}\PYG{l+m+mf}{0.0}\PYG{p}{)}
\PYG{n}{params}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{phase}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value}\PYG{o}{=}\PYG{l+m+mf}{0.2}\PYG{p}{)}
\PYG{n}{params}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{frequency}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value}\PYG{o}{=}\PYG{l+m+mf}{3.0}\PYG{p}{,} \PYG{n+nb}{max}\PYG{o}{=}\PYG{l+m+mi}{10}\PYG{p}{)}
\end{Verbatim}

or later:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{n}{params}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{amp}\PYG{l+s}{'}\PYG{p}{]}\PYG{o}{.}\PYG{n}{vary} \PYG{o}{=} \PYG{n+nb+bp}{True}
\PYG{n}{params}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{decay}\PYG{l+s}{'}\PYG{p}{]}\PYG{o}{.}\PYG{n}{max} \PYG{o}{=} \PYG{l+m+mf}{0.10}
\end{Verbatim}

Now the fit will \emph{not} vary the amplitude parameter, and will also impose a
lower bound on the decay factor and an upper bound on the frequency.
Importantly, our function to be minimized remains unchanged.

An important point here is that the \emph{params} object can be copied and
modified to make many user-level changes to the model and fitting process.
Of course, most of the information about how your data is modeled goes into
the fitting function, but the approach here allows some external control as
well.


\section{The \texttt{Parameter} class}
\label{parameters:the-parameter-class}\index{Parameter (built-in class)}

\begin{fulllineitems}
\phantomsection\label{parameters:Parameter}\pysiglinewithargsret{\strong{class }\bfcode{Parameter}}{\emph{value=None}\optional{, \emph{vary=True}\optional{, \emph{min=None}\optional{, \emph{max=None}\optional{, \emph{name=None}\optional{, \emph{expr=None}}}}}}}{}
create a Parameter object.  These are the fundamental extension of a fit
variable within lmfit, but you will probably create most of these with
the {\hyperref[parameters:Parameters]{\code{Parameters}}} class.
\begin{quote}\begin{description}
\item[{Parameters}] \leavevmode\begin{itemize}
\item {} 
\textbf{value} -- the numerical value for the parameter

\item {} 
\textbf{vary} (boolean (\code{True}/\code{False})) -- whether to vary the parameter or not.

\item {} 
\textbf{min} -- lower bound for value (\code{None} = no lower bound).

\item {} 
\textbf{max} -- upper bound for value (\code{None} = no upper bound).

\item {} 
\textbf{name} (\code{None} or string -- will be overwritten during fit if \code{None}.) -- parameter name

\item {} 
\textbf{expr} (\code{None} or string) -- mathematical expression to use to evaluate value during fit.

\end{itemize}

\end{description}\end{quote}

\end{fulllineitems}


Each of these inputs is turned into an attribute of the same name.   As
above, one hands a dictionary of Parameters to the fitting routines.   The
name for the Parameter will be set to be consistent

After a fit, a Parameter for a fitted variable (ie with vary = \code{True})
will have the \code{value} attribute holding the best-fit value, and may
(depending on the success of the fit) have obtain additional attributes.
\index{stderr}

\begin{fulllineitems}
\phantomsection\label{parameters:stderr}\pysigline{\bfcode{stderr}}
the estimated standard error for the best-fit value.

\end{fulllineitems}

\index{correl}

\begin{fulllineitems}
\phantomsection\label{parameters:correl}\pysigline{\bfcode{correl}}
a dictionary of the correlation with the other fitted variables in the
fit, of the form:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{p}{\PYGZob{}}\PYG{l+s}{'}\PYG{l+s}{decay}\PYG{l+s}{'}\PYG{p}{:} \PYG{l+m+mf}{0.404}\PYG{p}{,} \PYG{l+s}{'}\PYG{l+s}{phase}\PYG{l+s}{'}\PYG{p}{:} \PYG{o}{-}\PYG{l+m+mf}{0.020}\PYG{p}{,} \PYG{l+s}{'}\PYG{l+s}{frequency}\PYG{l+s}{'}\PYG{p}{:} \PYG{l+m+mf}{0.102}\PYG{p}{\PYGZcb{}}
\end{Verbatim}

\end{fulllineitems}


For details of the use of the bounds \code{min} and \code{max},
see {\hyperref[bounds:parameter-bounds-label]{\emph{Bounds Implementation}}}.

The \code{expr} attribute can contain a mathematical expression that will
be used to compute the value for the Parameter at each step in the fit.
See {\hyperref[constraints:math-constraints-label]{\emph{Using Mathematical Constraints}}} for more details and examples of this
feature.


\section{The \texttt{Parameters} class}
\label{parameters:the-parameters-class}\index{Parameters (built-in class)}

\begin{fulllineitems}
\phantomsection\label{parameters:Parameters}\pysigline{\strong{class }\bfcode{Parameters}}
create a Parameters object.  This is little more than a fancy
dictionary, with the restrictions that

1. keys must be valid Python symbol names (so that they can be used in
expressions of mathematical constraints).  This means the names must
match \code{{[}a-z\_{]}{[}a-z0-9\_{]}*}  and cannot be a Python reserved word.
\begin{enumerate}
\setcounter{enumi}{1}
\item {} 
values must be valid {\hyperref[parameters:Parameter]{\code{Parameter}}} objects.

\end{enumerate}

Two methods for provided for convenience of initializing Parameters.

\end{fulllineitems}

\index{add()}

\begin{fulllineitems}
\phantomsection\label{parameters:add}\pysiglinewithargsret{\bfcode{add}}{\emph{name}\optional{, \emph{value=None}\optional{, \emph{vary=True}\optional{, \emph{min=None}\optional{, \emph{max=None}\optional{, \emph{expr=None}}}}}}}{}
add a named parameter.  This simply creates a {\hyperref[parameters:Parameter]{\code{Parameter}}}
object associated with the key \emph{name}, with optional arguments
passed to {\hyperref[parameters:Parameter]{\code{Parameter}}}:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{n}{p} \PYG{o}{=} \PYG{n}{Parameters}\PYG{p}{(}\PYG{p}{)}
\PYG{n}{p}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{myvar}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value}\PYG{o}{=}\PYG{l+m+mi}{1}\PYG{p}{,} \PYG{n}{vary}\PYG{o}{=}\PYG{n+nb+bp}{True}\PYG{p}{)}
\end{Verbatim}

\end{fulllineitems}

\index{add\_many()}

\begin{fulllineitems}
\phantomsection\label{parameters:add_many}\pysiglinewithargsret{\bfcode{add\_many}}{\emph{self}, \emph{paramlist}}{}
add a list of named parameters.  Each entry must be a tuple
with the following entries:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{n}{name}\PYG{p}{,} \PYG{n}{value}\PYG{p}{,} \PYG{n}{vary}\PYG{p}{,} \PYG{n+nb}{min}\PYG{p}{,} \PYG{n+nb}{max}\PYG{p}{,} \PYG{n}{expr}
\end{Verbatim}

That is, this method is somewhat rigid and verbose (no default values),
but can be useful when initially defining a parameter list so that it
looks table-like:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{n}{p} \PYG{o}{=} \PYG{n}{Parameters}\PYG{p}{(}\PYG{p}{)}
\PYG{c}{\PYGZsh{}           (Name,  Value,  Vary,   Min,  Max,  Expr)}
\PYG{n}{p}\PYG{o}{.}\PYG{n}{add\PYGZus{}many}\PYG{p}{(}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{amp1}\PYG{l+s}{'}\PYG{p}{,}    \PYG{l+m+mi}{10}\PYG{p}{,}  \PYG{n+nb+bp}{True}\PYG{p}{,} \PYG{n+nb+bp}{None}\PYG{p}{,} \PYG{n+nb+bp}{None}\PYG{p}{,}  \PYG{n+nb+bp}{None}\PYG{p}{)}\PYG{p}{,}
           \PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{cen1}\PYG{l+s}{'}\PYG{p}{,}   \PYG{l+m+mf}{1.2}\PYG{p}{,}  \PYG{n+nb+bp}{True}\PYG{p}{,}  \PYG{l+m+mf}{0.5}\PYG{p}{,}  \PYG{l+m+mf}{2.0}\PYG{p}{,}  \PYG{n+nb+bp}{None}\PYG{p}{)}\PYG{p}{,}
           \PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{wid1}\PYG{l+s}{'}\PYG{p}{,}   \PYG{l+m+mf}{0.8}\PYG{p}{,}  \PYG{n+nb+bp}{True}\PYG{p}{,}  \PYG{l+m+mf}{0.1}\PYG{p}{,} \PYG{n+nb+bp}{None}\PYG{p}{,}  \PYG{n+nb+bp}{None}\PYG{p}{)}\PYG{p}{,}
           \PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{amp2}\PYG{l+s}{'}\PYG{p}{,}   \PYG{l+m+mf}{7.5}\PYG{p}{,}  \PYG{n+nb+bp}{True}\PYG{p}{,} \PYG{n+nb+bp}{None}\PYG{p}{,} \PYG{n+nb+bp}{None}\PYG{p}{,}  \PYG{n+nb+bp}{None}\PYG{p}{)}\PYG{p}{,}
           \PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{cen2}\PYG{l+s}{'}\PYG{p}{,}   \PYG{l+m+mf}{1.9}\PYG{p}{,}  \PYG{n+nb+bp}{True}\PYG{p}{,}  \PYG{l+m+mf}{1.0}\PYG{p}{,}  \PYG{l+m+mf}{3.0}\PYG{p}{,}  \PYG{n+nb+bp}{None}\PYG{p}{)}\PYG{p}{,}
           \PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{wid2}\PYG{l+s}{'}\PYG{p}{,}  \PYG{n+nb+bp}{None}\PYG{p}{,} \PYG{n+nb+bp}{False}\PYG{p}{,} \PYG{n+nb+bp}{None}\PYG{p}{,} \PYG{n+nb+bp}{None}\PYG{p}{,} \PYG{l+s}{'}\PYG{l+s}{2*wid1/3}\PYG{l+s}{'}\PYG{p}{)}\PYG{p}{)}
\end{Verbatim}

\end{fulllineitems}



\section{Simple Example}
\label{parameters:simple-example}
Putting it all together, a simple example of using a dictionary of
{\hyperref[parameters:Parameter]{\code{Parameter}}} objects and {\hyperref[fitting:minimize]{\code{minimize()}}} might look like this:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{k+kn}{from} \PYG{n+nn}{lmfit} \PYG{k+kn}{import} \PYG{n}{minimize}\PYG{p}{,} \PYG{n}{Parameters}\PYG{p}{,} \PYG{n}{Parameter}\PYG{p}{,} \PYG{n}{report\PYGZus{}errors}
\PYG{k+kn}{import} \PYG{n+nn}{numpy} \PYG{k+kn}{as} \PYG{n+nn}{np}

\PYG{c}{\PYGZsh{} create data to be fitted}
\PYG{n}{x} \PYG{o}{=} \PYG{n}{np}\PYG{o}{.}\PYG{n}{linspace}\PYG{p}{(}\PYG{l+m+mi}{0}\PYG{p}{,} \PYG{l+m+mi}{15}\PYG{p}{,} \PYG{l+m+mi}{301}\PYG{p}{)}
\PYG{n}{data} \PYG{o}{=} \PYG{p}{(}\PYG{l+m+mf}{5.} \PYG{o}{*} \PYG{n}{np}\PYG{o}{.}\PYG{n}{sin}\PYG{p}{(}\PYG{l+m+mi}{2} \PYG{o}{*} \PYG{n}{x} \PYG{o}{-} \PYG{l+m+mf}{0.1}\PYG{p}{)} \PYG{o}{*} \PYG{n}{np}\PYG{o}{.}\PYG{n}{exp}\PYG{p}{(}\PYG{o}{-}\PYG{n}{x}\PYG{o}{*}\PYG{n}{x}\PYG{o}{*}\PYG{l+m+mf}{0.025}\PYG{p}{)} \PYG{o}{+}
        \PYG{n}{np}\PYG{o}{.}\PYG{n}{random}\PYG{o}{.}\PYG{n}{normal}\PYG{p}{(}\PYG{n}{size}\PYG{o}{=}\PYG{n+nb}{len}\PYG{p}{(}\PYG{n}{x}\PYG{p}{)}\PYG{p}{,} \PYG{n}{scale}\PYG{o}{=}\PYG{l+m+mf}{0.2}\PYG{p}{)} \PYG{p}{)}

\PYG{c}{\PYGZsh{} define objective function: returns the array to be minimized}
\PYG{k}{def} \PYG{n+nf}{fcn2min}\PYG{p}{(}\PYG{n}{params}\PYG{p}{,} \PYG{n}{x}\PYG{p}{,} \PYG{n}{data}\PYG{p}{)}\PYG{p}{:}
    \PYG{l+s+sd}{""" model decaying sine wave, subtract data"""}
    \PYG{n}{amp} \PYG{o}{=} \PYG{n}{params}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{amp}\PYG{l+s}{'}\PYG{p}{]}\PYG{o}{.}\PYG{n}{value}
    \PYG{n}{shift} \PYG{o}{=} \PYG{n}{params}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{shift}\PYG{l+s}{'}\PYG{p}{]}\PYG{o}{.}\PYG{n}{value}
    \PYG{n}{omega} \PYG{o}{=} \PYG{n}{params}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{omega}\PYG{l+s}{'}\PYG{p}{]}\PYG{o}{.}\PYG{n}{value}
    \PYG{n}{decay} \PYG{o}{=} \PYG{n}{params}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{decay}\PYG{l+s}{'}\PYG{p}{]}\PYG{o}{.}\PYG{n}{value}
    
    \PYG{n}{model} \PYG{o}{=} \PYG{n}{amp} \PYG{o}{*} \PYG{n}{np}\PYG{o}{.}\PYG{n}{sin}\PYG{p}{(}\PYG{n}{x} \PYG{o}{*} \PYG{n}{omega} \PYG{o}{+} \PYG{n}{shift}\PYG{p}{)} \PYG{o}{*} \PYG{n}{np}\PYG{o}{.}\PYG{n}{exp}\PYG{p}{(}\PYG{o}{-}\PYG{n}{x}\PYG{o}{*}\PYG{n}{x}\PYG{o}{*}\PYG{n}{decay}\PYG{p}{)}
    \PYG{k}{return} \PYG{n}{model} \PYG{o}{-} \PYG{n}{data}

\PYG{c}{\PYGZsh{} create a set of Parameters}
\PYG{n}{params} \PYG{o}{=} \PYG{n}{Parameters}\PYG{p}{(}\PYG{p}{)}
\PYG{n}{params}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{amp}\PYG{l+s}{'}\PYG{p}{,}   \PYG{n}{value}\PYG{o}{=} \PYG{l+m+mi}{10}\PYG{p}{,}  \PYG{n+nb}{min}\PYG{o}{=}\PYG{l+m+mi}{0}\PYG{p}{)}
\PYG{n}{params}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{decay}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value}\PYG{o}{=} \PYG{l+m+mf}{0.1}\PYG{p}{)} 
\PYG{n}{params}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{shift}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value}\PYG{o}{=} \PYG{l+m+mf}{0.0}\PYG{p}{,} \PYG{n+nb}{min}\PYG{o}{=}\PYG{o}{-}\PYG{n}{np}\PYG{o}{.}\PYG{n}{pi}\PYG{o}{/}\PYG{l+m+mf}{2.}\PYG{p}{,} \PYG{n+nb}{max}\PYG{o}{=}\PYG{n}{np}\PYG{o}{.}\PYG{n}{pi}\PYG{o}{/}\PYG{l+m+mi}{2}\PYG{p}{)}
\PYG{n}{params}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{omega}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value}\PYG{o}{=} \PYG{l+m+mf}{3.0}\PYG{p}{)}


\PYG{c}{\PYGZsh{} do fit, here with leastsq model}
\PYG{n}{result} \PYG{o}{=} \PYG{n}{minimize}\PYG{p}{(}\PYG{n}{fcn2min}\PYG{p}{,} \PYG{n}{params}\PYG{p}{,} \PYG{n}{args}\PYG{o}{=}\PYG{p}{(}\PYG{n}{x}\PYG{p}{,} \PYG{n}{data}\PYG{p}{)}\PYG{p}{)}

\PYG{c}{\PYGZsh{} calculate final result}
\PYG{n}{final} \PYG{o}{=} \PYG{n}{data} \PYG{o}{+} \PYG{n}{result}\PYG{o}{.}\PYG{n}{residual}

\PYG{c}{\PYGZsh{} write error report}
\PYG{n}{report\PYGZus{}errors}\PYG{p}{(}\PYG{n}{params}\PYG{p}{)}

\PYG{c}{\PYGZsh{} try to plot results}
\PYG{k}{try}\PYG{p}{:}
    \PYG{k+kn}{import} \PYG{n+nn}{pylab}
    \PYG{n}{pylab}\PYG{o}{.}\PYG{n}{plot}\PYG{p}{(}\PYG{n}{x}\PYG{p}{,} \PYG{n}{data}\PYG{p}{,} \PYG{l+s}{'}\PYG{l+s}{k+}\PYG{l+s}{'}\PYG{p}{)}
    \PYG{n}{pylab}\PYG{o}{.}\PYG{n}{plot}\PYG{p}{(}\PYG{n}{x}\PYG{p}{,} \PYG{n}{final}\PYG{p}{,} \PYG{l+s}{'}\PYG{l+s}{r}\PYG{l+s}{'}\PYG{p}{)}
    \PYG{n}{pylab}\PYG{o}{.}\PYG{n}{show}\PYG{p}{(}\PYG{p}{)}
\PYG{k}{except}\PYG{p}{:}
    \PYG{k}{pass}
\end{Verbatim}


\chapter{Performing Fits, Analyzing Outputs}
\label{fitting:performing-fits-analyzing-outputs}\label{fitting::doc}
As shown in the previous sections, a simple fit can be performed with
the {\hyperref[fitting:minimize]{\code{minimize()}}} function.    For more sophisticated modeling,
the {\hyperref[fitting:Minimizer]{\code{Minimizer}}} class can be used to gain a bit more control,
especially when using complicated constraints.


\section{The \texttt{minimize()} function}
\label{fitting:the-minimize-function}
The minimize function takes a function to minimize, a dictionary of
{\hyperref[parameters:Parameter]{\code{Parameter}}} , and several optional arguments.    See
{\hyperref[fitting:fit-func-label]{\emph{Writing a Fitting Function}}} for details on writing the function to minimize.
\index{minimize() (built-in function)}

\begin{fulllineitems}
\phantomsection\label{fitting:minimize}\pysiglinewithargsret{\bfcode{minimize}}{\emph{function}, \emph{params}, \emph{args=None}, \emph{kws=None}, \emph{method='leastsq'}, \emph{**leastsq\_kws}}{}
find values for the params so that the sum-of-squares of the returned array
from function is minimized.
\begin{quote}\begin{description}
\item[{Parameters}] \leavevmode\begin{itemize}
\item {} 
\textbf{function} (\emph{callable.}) -- function to return fit residual.  See {\hyperref[fitting:fit-func-label]{\emph{Writing a Fitting Function}}} for details.

\item {} 
\textbf{params} (\emph{dict}) -- a dictionary of Parameters.  Keywords must be strings
that match \code{{[}a-z\_{]}{[}a-z0-9\_{]}*} and is not a python
reserved word.  Each value must be {\hyperref[parameters:Parameter]{\code{Parameter}}}.

\item {} 
\textbf{args} (\emph{tuple}) -- arguments tuple to pass to the residual function as  positional arguments.

\item {} 
\textbf{kws} (\emph{dict}) -- dictionary to pass to the residual function as keyword arguments.

\item {} 
\textbf{method} (\emph{string}) -- name of fitting method to use. See  {\hyperref[fitting:fit-engines-label]{\emph{Choosing Different Fitting Methods}}} for details

\item {} 
\textbf{leastsq\_kws} (\emph{dict}) -- dictionary to pass to scipy.optimize.leastsq

\end{itemize}

\item[{Returns}] \leavevmode
Minimizer object, which can be used to inspect goodness-of-fit
statistics, or to re-run fit.

\end{description}\end{quote}

For backward compatibility, the keyword \emph{engine} is retained as a synonym for \emph{method},
but this should be considered depracated.

On output, the params will be updated with best-fit values and, where
appropriate, estimated uncertainties and correlations.  See
{\hyperref[fitting:fit-results-label]{\emph{Goodness-of-Fit and estimated uncertainty and correlations}}} for further details.

\end{fulllineitems}



\section{Writing a Fitting Function}
\label{fitting:fit-func-label}\label{fitting:writing-a-fitting-function}
An important component of a fit is writing a function to be minimized in
the least-squares sense.   Since this function will be called by other
routines, there are fairly stringent requirements for its call signature
and return value.   In principle, your function can be any python callable,
but it must look like this:


\begin{fulllineitems}
\pysigline{\bfcode{func(params,~*args,~**kws):}}
calculate objective residual to be minimized from parameters.
\begin{quote}\begin{description}
\item[{Parameters}] \leavevmode\begin{itemize}
\item {} 
\textbf{params} (\emph{dict}) -- parameters.

\item {} 
\textbf{args} -- positional arguments.  Must match \code{args} argument to {\hyperref[fitting:minimize]{\code{minimize()}}}

\item {} 
\textbf{kws} -- keyword arguments.  Must match \code{kws} argument to {\hyperref[fitting:minimize]{\code{minimize()}}}

\end{itemize}

\item[{Returns}] \leavevmode
residual array (generally data-model) to be minimized in the least-squares sense.

\item[{Return type}] \leavevmode
numpy array.  The length of this array cannot change between calls.

\end{description}\end{quote}

\end{fulllineitems}


A common use for the positional and keyword arguments would be to pass in other
data needed to calculate the residual, including such things as the data array,
dependent variable, uncertainties in the data, and other data structures for the
model calculation.

The objective function should return the value to be minimized.  For the
Levenberg-Marquardt algorithm from {\hyperref[fitting:leastsq]{\code{leastsq()}}}, this returned value \textbf{must} be an
array, with a length greater than or equal to the number of fitting variables in the
model.  For the other methods, the return value can either be a scalar or an array.  If an
array is returned, the sum of squares of the array will be sent to the underlying fitting
method, effectively doing a least-squares optimization of the return values.

Since the function will be passed in a dictionary of {\hyperref[parameters:Parameters]{\code{Parameters}}}, it is advisable
to unpack these to get numerical values at the top of the function.  A simple example
would look like:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{k}{def} \PYG{n+nf}{residual}\PYG{p}{(}\PYG{n}{pars}\PYG{p}{,} \PYG{n}{x}\PYG{p}{,} \PYG{n}{data}\PYG{o}{=}\PYG{n+nb+bp}{None}\PYG{p}{)}\PYG{p}{:}
    \PYG{c}{\PYGZsh{} unpack parameters:}
    \PYG{c}{\PYGZsh{}  extract .value attribute for each parameter}
    \PYG{n}{amp} \PYG{o}{=} \PYG{n}{pars}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{amp}\PYG{l+s}{'}\PYG{p}{]}\PYG{o}{.}\PYG{n}{value}
    \PYG{n}{period} \PYG{o}{=} \PYG{n}{pars}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{period}\PYG{l+s}{'}\PYG{p}{]}\PYG{o}{.}\PYG{n}{value}
    \PYG{n}{shift} \PYG{o}{=} \PYG{n}{pars}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{shift}\PYG{l+s}{'}\PYG{p}{]}\PYG{o}{.}\PYG{n}{value}
    \PYG{n}{decay} \PYG{o}{=} \PYG{n}{pars}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{decay}\PYG{l+s}{'}\PYG{p}{]}\PYG{o}{.}\PYG{n}{value}

    \PYG{k}{if} \PYG{n+nb}{abs}\PYG{p}{(}\PYG{n}{shift}\PYG{p}{)} \PYG{o}{\PYGZgt{}} \PYG{n}{pi}\PYG{o}{/}\PYG{l+m+mi}{2}\PYG{p}{:}
        \PYG{n}{shift} \PYG{o}{=} \PYG{n}{shift} \PYG{o}{-} \PYG{n}{sign}\PYG{p}{(}\PYG{n}{shift}\PYG{p}{)}\PYG{o}{*}\PYG{n}{pi}

    \PYG{k}{if} \PYG{n+nb}{abs}\PYG{p}{(}\PYG{n}{period}\PYG{p}{)} \PYG{o}{\PYGZlt{}} \PYG{l+m+mf}{1.e-10}\PYG{p}{:}
        \PYG{n}{period} \PYG{o}{=} \PYG{n}{sign}\PYG{p}{(}\PYG{n}{period}\PYG{p}{)}\PYG{o}{*}\PYG{l+m+mf}{1.e-10}

    \PYG{n}{model} \PYG{o}{=} \PYG{n}{amp} \PYG{o}{*} \PYG{n}{sin}\PYG{p}{(}\PYG{n}{shift} \PYG{o}{+} \PYG{n}{x}\PYG{o}{/}\PYG{n}{period}\PYG{p}{)} \PYG{o}{*} \PYG{n}{exp}\PYG{p}{(}\PYG{o}{-}\PYG{n}{x}\PYG{o}{*}\PYG{n}{x}\PYG{o}{*}\PYG{n}{decay}\PYG{o}{*}\PYG{n}{decay}\PYG{p}{)}

    \PYG{k}{if} \PYG{n}{data} \PYG{o+ow}{is} \PYG{n+nb+bp}{None}\PYG{p}{:}
        \PYG{k}{return} \PYG{n}{model}
    \PYG{k}{return} \PYG{p}{(}\PYG{n}{model} \PYG{o}{-} \PYG{n}{data}\PYG{p}{)}
\end{Verbatim}

In this example, \code{x} is a positional (required) argument, while the \code{data}
array is actually optional (so that the function returns the model calculation
if the data is neglected).   Also note that the model calculation will divide
\code{x} by the varied value of the `period' Parameter.  It might be wise to
make sure this parameter cannot be 0.   It would be possible to use the bounds
on the {\hyperref[parameters:Parameter]{\code{Parameter}}} to do this:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{n}{params}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{period}\PYG{l+s}{'}\PYG{p}{]} \PYG{o}{=} \PYG{n}{Parameter}\PYG{p}{(}\PYG{n}{value}\PYG{o}{=}\PYG{l+m+mi}{2}\PYG{p}{,} \PYG{n+nb}{min}\PYG{o}{=}\PYG{l+m+mf}{1.e-10}\PYG{p}{)}
\end{Verbatim}

but might be wiser to put this directly in the function with:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{k}{if} \PYG{n+nb}{abs}\PYG{p}{(}\PYG{n}{period}\PYG{p}{)} \PYG{o}{\PYGZlt{}} \PYG{l+m+mf}{1.e-10}\PYG{p}{:}
    \PYG{n}{period} \PYG{o}{=} \PYG{n}{sign}\PYG{p}{(}\PYG{n}{period}\PYG{p}{)}\PYG{o}{*}\PYG{l+m+mf}{1.e-10}
\end{Verbatim}


\section{Choosing Different Fitting Methods}
\label{fitting:fit-engines-label}\label{fitting:choosing-different-fitting-methods}
By default, the \href{http://en.wikipedia.org/wiki/Levenberg-Marquardt\_algorithm}{Levenberg-Marquardt} algorithm is
used for fitting.  While often criticized, including the fact it finds a
\emph{local} minima, this approach has some distinct advantages.  These include
being fast, and well-behaved for most curve-fitting needs, and making it
easy to estimate uncertainties for and correlations between pairs of fit
variables, as discussed in {\hyperref[fitting:fit-results-label]{\emph{Goodness-of-Fit and estimated uncertainty and correlations}}}.

Alternative algorithms can also be used. These include \href{http://en.wikipedia.org/wiki/Simulated\_annealing}{simulated annealing} which promises a
better ability to avoid local minima, and \href{http://en.wikipedia.org/wiki/Limited-memory\_BFGS}{BFGS}, which is a
modification of the quasi-Newton method.

To select which of these algorithms to use, use the \code{method} keyword to the
{\hyperref[fitting:minimize]{\code{minimize()}}} function or use the corresponding method name from the
{\hyperref[fitting:Minimizer]{\code{Minimizer}}} class as listed in the
{\hyperref[fitting:fit-engine-table]{\emph{Table of Supported Fitting Methods}}}.
\phantomsection\label{fitting:fit-engine-table}\begin{quote}

Table of Supported Fitting Methods:

\begin{tabulary}{\linewidth}{|L|L|L|L|}
\hline
\textbf{
Fitting
Method
} & \textbf{
\code{method} arg to
{\hyperref[fitting:minimize]{\code{minimize()}}}
} & \textbf{
{\hyperref[fitting:Minimizer]{\code{Minimizer}}}
method
} & \textbf{
\code{method} arg to
{\hyperref[fitting:scalar_minimize]{\code{scalar\_minimize()}}}
}\\\hline

Levenberg-Marquardt
 & 
\code{leastsq}
 & 
{\hyperref[fitting:leastsq]{\code{leastsq()}}}
 & 
Not available
\\\hline

Nelder-Mead
 & 
\code{nelder}
 & 
{\hyperref[fitting:fmin]{\code{fmin()}}}
 & 
\code{Nelder-Mead}
\\\hline

L-BFGS-B
 & 
\code{lbfgsb}
 & 
{\hyperref[fitting:lbfgsb]{\code{lbfgsb()}}}
 & 
\code{L-BFGS-B}
\\\hline

Simulated Annealing
 & 
\code{anneal}
 & 
{\hyperref[fitting:anneal]{\code{anneal()}}}
 & 
\code{Anneal}
\\\hline

Powell
 & 
\code{powell}
 &  & 
\code{Powell}
\\\hline

Conjugate Gradient
 & 
\code{cg}
 &  & 
\code{CG}
\\\hline

Newtown-CG
 & 
\code{newton}
 &  & 
\code{Newton-CG}
\\\hline

COBYLA
 & 
\code{cobyla}
 &  & 
\code{COBYLA}
\\\hline

Sequential Linear
Squares Programming
 & 
\code{slsqp}
 &  & 
\code{SLSQP}
\\\hline
\end{tabulary}

\end{quote}

\begin{notice}{note}{Note:}
Use of \code{scipy.optimize.minimize()} requires scipy 0.11 or higher.
\end{notice}

\begin{notice}{note}{Note:}
The objective function for the Levenberg-Marquardt method \textbf{must} return an array,
with more elements than variables.  All other methods can return either a scalar value
or an array.
\end{notice}

\begin{notice}{warning}{Warning:}
The Levenberg-Marquardt method is \emph{by far} the most tested fit method,
and much of this documentation assumes that this is the method used.  For
example, many of the fit statistics and estimates for uncertainties in
parameters discussed in {\hyperref[fitting:fit-results-label]{\emph{Goodness-of-Fit and estimated uncertainty and correlations}}} are done only for the
\code{leastsq} method.
\end{notice}

In particular, the simulated annealing method appears to not work
correctly.... understanding this is on the ToDo list.


\section{Goodness-of-Fit and estimated uncertainty and correlations}
\label{fitting:fit-results-label}\label{fitting:goodness-of-fit-and-estimated-uncertainty-and-correlations}
On a successful fit using the \emph{leastsq} method, several goodness-of-fit
statistics and values related to the uncertainty in the fitted variables will be
calculated.  These are all encapsulated in the {\hyperref[fitting:Minimizer]{\code{Minimizer}}} object for the
fit, as returned by {\hyperref[fitting:minimize]{\code{minimize()}}}.  The values related to the entire fit are
stored in attributes of the {\hyperref[fitting:Minimizer]{\code{Minimizer}}} object, as shown in {\hyperref[fitting:goodfit-table]{\emph{Table
of Fit Results}}} while those related to each fitted variables are
stored as attributes of the corresponding {\hyperref[parameters:Parameter]{\code{Parameter}}}.
\phantomsection\label{fitting:goodfit-table}\begin{quote}

Table of Fit Results:  These values, including the standard Goodness-of-Fit statistics,
are all attributes of the {\hyperref[fitting:Minimizer]{\code{Minimizer}}} object returned by {\hyperref[fitting:minimize]{\code{minimize()}}}.
\end{quote}

\begin{tabulary}{\linewidth}{|L|L|}
\hline
\textbf{
{\hyperref[fitting:Minimizer]{\code{Minimizer}}}
Attribute
} & \textbf{
Description / Formula
}\\\hline

nfev
 & 
number of function evaluations
\\\hline

success
 & 
boolean (\code{True}/\code{False}) for whether fit succeeded.
\\\hline

errorbars
 & 
boolean (\code{True}/\code{False}) for whether uncertainties were estimated.
\\\hline

message
 & 
message about fit success.
\\\hline

ier
 & 
integer error value from scipy.optimize.leastsq
\\\hline

lmdif\_message
 & 
message from scipy.optimize.leastsq
\\\hline

nvarys
 & 
number of variables in fit  $N_{\rm varys}$
\\\hline

ndata
 & 
number of data points:  $N$
\\\hline

nfree {}`
 & 
degrees of freedom in fit:  $N - N_{\rm varys}$
\\\hline

residual
 & 
residual array (return of \code{func()}:  ${\rm Resid}$
\\\hline

chisqr
 & 
chi-square: $\chi^2 = \sum_i^N [{\rm Resid}_i]^2$
\\\hline

redchi
 & 
reduced chi-square: $\chi^2_{\nu}= {\chi^2} / {(N - N_{\rm varys})}$
\\\hline
\end{tabulary}


Note that the calculation of chi-square and reduced chi-square assume that the
returned residual function is scaled properly to the uncertainties in the data.
For these statistics to be meaningful, the person writing the function to
be minimized must scale them properly.

After a fit using using the {\hyperref[fitting:leastsq]{\code{leastsq()}}} method has completed succsessfully,
standard errors for the fitted variables and correlations between pairs of
fitted variables are automatically calculated from the covariance matrix.
The standard error (estimated $1\sigma$ error-bar) go into the
{\hyperref[parameters:stderr]{\code{stderr}}} attribute of the Parameter.  The correlations with all other
variables will be put into the {\hyperref[parameters:correl]{\code{correl}}} attribute of the Parameter --
a dictionary with keys for all other Parameters and values of the
corresponding correlation.

In some cases, it may not be possible to estimate the errors and
correlations.  For example, if a variable actually has no practical effect
on the fit, it will likely cause the covariance matrix to be singular,
making standard errors impossible to estimate.  Placing bounds on varied
Parameters makes it more likely that errors cannot be estimated, as being
near the maximum or minimum value makes the covariance matrix singular.  In
these cases, the \code{errorbars} attribute of the fit result
({\hyperref[fitting:Minimizer]{\code{Minimizer}}} object) will be \code{False}.


\section{Using the \texttt{Minimizer} class}
\label{fitting:fit-minimizer-label}\label{fitting:using-the-minimizer-class}
For full control of the fitting process, you'll want to create a
{\hyperref[fitting:Minimizer]{\code{Minimizer}}} object, or at least use the one returned from the
{\hyperref[fitting:minimize]{\code{minimize()}}} function.
\index{Minimizer (built-in class)}

\begin{fulllineitems}
\phantomsection\label{fitting:Minimizer}\pysiglinewithargsret{\strong{class }\bfcode{Minimizer}}{\emph{function}, \emph{params}, \emph{fcn\_args=None}, \emph{fcn\_kws=None}, \emph{iter\_cb=None}, \emph{scale\_covar=True}, \emph{**kws}}{}
creates a Minimizer, for fine-grain access to fitting methods and attributes.
\begin{quote}\begin{description}
\item[{Parameters}] \leavevmode\begin{itemize}
\item {} 
\textbf{function} (\emph{callable.}) -- objective function to return fit residual.  See {\hyperref[fitting:fit-func-label]{\emph{Writing a Fitting Function}}} for details.

\item {} 
\textbf{params} (\emph{dict}) -- a dictionary of Parameters.  Keywords must be strings
that match \code{{[}a-z\_{]}{[}a-z0-9\_{]}*} and is not a python
reserved word.  Each value must be {\hyperref[parameters:Parameter]{\code{Parameter}}}.

\item {} 
\textbf{fcn\_args} (\emph{tuple}) -- arguments tuple to pass to the residual function as  positional arguments.

\item {} 
\textbf{fcn\_kws} (\emph{dict}) -- dictionary to pass to the residual function as keyword arguments.

\item {} 
\textbf{iter\_cb} (callable or \code{None}) -- function to be called at each fit iteration

\item {} 
\textbf{scale\_covar} -- flag for scaling covariance matrix and uncertainties to reduced chi-square (\code{leastsq} only)

\item {} 
\textbf{kws} (\emph{dict}) -- dictionary to pass as keywords to the underlying scipy.optimize method.

\end{itemize}

\item[{Returns}] \leavevmode
Minimizer object, which can be used to inspect goodness-of-fit
statistics, or to re-run fit.

\end{description}\end{quote}

\end{fulllineitems}


The Minimizer object has a few public methods:
\index{leastsq()}

\begin{fulllineitems}
\phantomsection\label{fitting:leastsq}\pysiglinewithargsret{\bfcode{leastsq}}{\emph{scale\_covar=True}, \emph{**kws}}{}
perform fit with Levenberg-Marquardt algorithm.  Keywords will be passed directly to
\href{http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.leastsq.html}{scipy.optimize.leastsq}.
By default, numerical derivatives are used, and the following arguments are set:
\begin{quote}

\begin{tabulary}{\linewidth}{|L|L|L|}
\hline
\textbf{
{\hyperref[fitting:leastsq]{\code{leastsq()}}}
arg
} & \textbf{
Default Value
} & \textbf{
Description
}\\\hline

xtol
 & 
1.e-7
 & 
Relative error in the approximate solution
\\\hline

ftol
 & 
1.e-7
 & 
Relative error in the desired sum of squares
\\\hline

maxfev
 & 
2000*(nvar+1)
 & 
maximum number of function calls (nvar= \# of variables)
\\\hline

Dfun
 & 
\code{None}
 & 
function to call for Jacobian calculation
\\\hline
\end{tabulary}

\end{quote}

\end{fulllineitems}

\index{anneal()}

\begin{fulllineitems}
\phantomsection\label{fitting:anneal}\pysiglinewithargsret{\bfcode{anneal}}{\emph{**kws}}{}
perform fit with Simulated Annealing.  Keywords will be passed directly to
\href{http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.anneal.html}{scipy.optimize.anneal}.
\begin{quote}

\begin{tabulary}{\linewidth}{|L|L|L|}
\hline
\textbf{
{\hyperref[fitting:anneal]{\code{anneal()}}}
arg
} & \textbf{
Default Value
} & \textbf{
Description
}\\\hline

schedule
 & 
\code{cauchy}
 & 
annealing schedule
\\\hline

maxiter
 & 
2000*(nvar+1)
 & 
maximum number of iterations
\\\hline
\end{tabulary}


For me, this Simulated Annealing appears to never work.
\end{quote}

\end{fulllineitems}

\index{lbfgsb()}

\begin{fulllineitems}
\phantomsection\label{fitting:lbfgsb}\pysiglinewithargsret{\bfcode{lbfgsb}}{\emph{**kws}}{}
perform fit with L-BFGS-B algorithm.  Keywords will be passed directly to
\href{http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.fmin\_l\_bfgs\_b.html}{scipy.optimize.fmin\_l\_bfgs\_b}.
\begin{quote}

\begin{tabulary}{\linewidth}{|L|L|L|}
\hline
\textbf{
{\hyperref[fitting:lbfgsb]{\code{lbfgsb()}}}
arg
} & \textbf{
Default Value
} & \textbf{
Description
}\\\hline

factr
 & 
1000.0
 & \\\hline

approx\_grad
 & 
\code{True}
 & 
calculate approximations of gradient
\\\hline

maxfun
 & 
2000*(nvar+1)
 & 
maximum number of function calls (nvar= \# of variables)
\\\hline
\end{tabulary}

\end{quote}

\end{fulllineitems}

\index{fmin()}

\begin{fulllineitems}
\phantomsection\label{fitting:fmin}\pysiglinewithargsret{\bfcode{fmin}}{\emph{**kws}}{}
perform fit with Nelder-Mead downhill simplex algorithm.  Keywords will be passed directly to
\href{http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.fmin.html}{scipy.optimize.fmin}.
\begin{quote}

\begin{tabulary}{\linewidth}{|L|L|L|}
\hline
\textbf{
{\hyperref[fitting:fmin]{\code{fmin()}}}
arg
} & \textbf{
Default Value
} & \textbf{
Description
}\\\hline

ftol
 & 
1.e-4
 & 
function tolerance
\\\hline

xtol
 & 
1.e-4
 & 
parameter tolerance
\\\hline

maxfun
 & 
5000*(nvar+1)
 & 
maximum number of function calls (nvar= \# of variables)
\\\hline
\end{tabulary}

\end{quote}

\end{fulllineitems}

\index{scalar\_minimize()}

\begin{fulllineitems}
\phantomsection\label{fitting:scalar_minimize}\pysiglinewithargsret{\bfcode{scalar\_minimize}}{\emph{method='Nelder-Mead'}, \emph{hess=None}, \emph{tol=None}, \emph{**kws}}{}
perform fit with any of the scalar minimimization algorithms supported by
\href{http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.minimize.html}{scipy.optimize.minimize}.
\begin{quote}

\begin{tabulary}{\linewidth}{|L|L|L|}
\hline
\textbf{
{\hyperref[fitting:scalar_minimize]{\code{scalar\_minimize()}}}
arg
} & \textbf{
Default Value
} & \textbf{
Description
}\\\hline

method
 & 
\code{Nelder-Mead}
 & 
fitting method
\\\hline

tol
 & 
1.e-7
 & 
fitting and parameter tolerance
\\\hline

hess
 & 
None
 & 
Hessian of objective function
\\\hline
\end{tabulary}

\end{quote}

\end{fulllineitems}

\index{prepare\_fit()}

\begin{fulllineitems}
\phantomsection\label{fitting:prepare_fit}\pysiglinewithargsret{\bfcode{prepare\_fit}}{\emph{**kws}}{}
prepares and initializes model and Parameters for subsequent
fitting. This routine prepares the conversion of {\hyperref[parameters:Parameters]{\code{Parameters}}}
into fit variables, organizes parameter bounds, and parses, checks and
``compiles'' constrain expressions.

This is called directly by the fitting methods, and it is generally not
necessary to call this function explicitly.  An exception is when you
would like to call your function to minimize prior to running one of the
minimization routines, for example, to calculate the initial residual
function.  In that case, you might want to do something like:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{n}{myfit} \PYG{o}{=} \PYG{n}{Minimizer}\PYG{p}{(}\PYG{n}{my\PYGZus{}residual}\PYG{p}{,} \PYG{n}{params}\PYG{p}{,}  \PYG{n}{fcn\PYGZus{}args}\PYG{o}{=}\PYG{p}{(}\PYG{n}{x}\PYG{p}{,}\PYG{p}{)}\PYG{p}{,} \PYG{n}{fcn\PYGZus{}kws}\PYG{o}{=}\PYG{p}{\PYGZob{}}\PYG{l+s}{'}\PYG{l+s}{data}\PYG{l+s}{'}\PYG{p}{:}\PYG{n}{data}\PYG{p}{\PYGZcb{}}\PYG{p}{)}

\PYG{n}{myfit}\PYG{o}{.}\PYG{n}{prepare\PYGZus{}fit}\PYG{p}{(}\PYG{p}{)}
\PYG{n}{init} \PYG{o}{=} \PYG{n}{my\PYGZus{}residual}\PYG{p}{(}\PYG{n}{p\PYGZus{}fit}\PYG{p}{,} \PYG{n}{x}\PYG{p}{)}
\PYG{n}{pylab}\PYG{o}{.}\PYG{n}{plot}\PYG{p}{(}\PYG{n}{x}\PYG{p}{,} \PYG{n}{init}\PYG{p}{,} \PYG{l+s}{'}\PYG{l+s}{b--}\PYG{l+s}{'}\PYG{p}{)}

\PYG{n}{myfit}\PYG{o}{.}\PYG{n}{leastsq}\PYG{p}{(}\PYG{p}{)}
\end{Verbatim}

That is, this method should be called prior to your fitting function being called.

\end{fulllineitems}



\chapter{Calculation of confidence intervals}
\label{confidence:module-confidence}\label{confidence::doc}\label{confidence:calculation-of-confidence-intervals}\index{confidence (module)}
Since version \emph{0.5}, lmfit is also capable of calculating the confidence
intervals directly. For most models, it is not necessary: the estimation
of the standard error from the estimated covariance matrix is normally quite
good.

But for some models, e.g. a sum of two exponentials, the approximation
begins to fail. For this case, lmfit has the function {\hyperref[confidence:confidence.conf_interval]{\code{conf\_interval()}}}
to calculate confidence inverals directly.  This is substantially slower
than using the errors estimated from the covariance matrix, but the results
are more robust.


\section{Method used for calculating confidence intervals}
\label{confidence:method-used-for-calculating-confidence-intervals}
The F-test is used to compare our null model, which is the best fit we have
found, with an alternate model, where one of the parameters is fixed to a
specific value. The value is changed until the differnce between $\chi^2_0$
and $\chi^2_{f}$ can't be explained by the loss of a degree of freedom
within a certain confidence.
\begin{gather}
\begin{split}F(P_{fix},N-P) = \left(\frac{\chi^2_f}{\chi^2_{0}}-1\right)\frac{N-P}{P_{fix}}\end{split}\notag
\end{gather}
N is the number of data-points, P the number of parameter of the null model.
$P_{fix}$ is the number of fixed parameters (or to be more clear, the
difference of number of parameters betweeen our null model and the alternate
model).

A log-likelihood method will be added soon.


\section{A basic example}
\label{confidence:a-basic-example}
First we create a toy problem:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{g+gp}{In [1]: }\PYG{k+kn}{import} \PYG{n+nn}{lmfit}

\PYG{g+gp}{In [2]: }\PYG{k+kn}{import} \PYG{n+nn}{numpy} \PYG{k+kn}{as} \PYG{n+nn}{np}

\PYG{g+gp}{In [3]: }\PYG{n}{x}\PYG{o}{=}\PYG{n}{np}\PYG{o}{.}\PYG{n}{linspace}\PYG{p}{(}\PYG{l+m+mf}{0.3}\PYG{p}{,}\PYG{l+m+mi}{10}\PYG{p}{,}\PYG{l+m+mi}{100}\PYG{p}{)}

\PYG{g+gp}{In [4]: }\PYG{n}{y}\PYG{o}{=}\PYG{l+m+mi}{1}\PYG{o}{/}\PYG{p}{(}\PYG{l+m+mf}{0.1}\PYG{o}{*}\PYG{n}{x}\PYG{p}{)}\PYG{o}{+}\PYG{l+m+mi}{2}\PYG{o}{+}\PYG{l+m+mf}{0.1}\PYG{o}{*}\PYG{n}{np}\PYG{o}{.}\PYG{n}{random}\PYG{o}{.}\PYG{n}{randn}\PYG{p}{(}\PYG{n}{x}\PYG{o}{.}\PYG{n}{size}\PYG{p}{)}

\PYG{g+gp}{In [5]: }\PYG{n}{p}\PYG{o}{=}\PYG{n}{lmfit}\PYG{o}{.}\PYG{n}{Parameters}\PYG{p}{(}\PYG{p}{)}

\PYG{g+gp}{In [6]: }\PYG{n}{p}\PYG{o}{.}\PYG{n}{add\PYGZus{}many}\PYG{p}{(}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{a}\PYG{l+s}{'}\PYG{p}{,}\PYG{l+m+mf}{0.1}\PYG{p}{)}\PYG{p}{,}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{b}\PYG{l+s}{'}\PYG{p}{,}\PYG{l+m+mi}{1}\PYG{p}{)}\PYG{p}{)}

\PYG{g+gp}{In [7]: }\PYG{k}{def} \PYG{n+nf}{residual}\PYG{p}{(}\PYG{n}{p}\PYG{p}{)}\PYG{p}{:}
\PYG{g+gp}{   ...:}      \PYG{n}{a}\PYG{o}{=}\PYG{n}{p}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{a}\PYG{l+s}{'}\PYG{p}{]}\PYG{o}{.}\PYG{n}{value}
\PYG{g+gp}{   ...:}      \PYG{n}{b}\PYG{o}{=}\PYG{n}{p}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{b}\PYG{l+s}{'}\PYG{p}{]}\PYG{o}{.}\PYG{n}{value}
\PYG{g+gp}{   ...:}      \PYG{k}{return} \PYG{l+m+mi}{1}\PYG{o}{/}\PYG{p}{(}\PYG{n}{a}\PYG{o}{*}\PYG{n}{x}\PYG{p}{)}\PYG{o}{+}\PYG{n}{b}\PYG{o}{-}\PYG{n}{y}
\PYG{g+gp}{   ...:}
\end{Verbatim}

We have to fit it, before we can generate the confidence intervals.

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{g+gp}{In [8]: }\PYG{n}{mi}\PYG{o}{=}\PYG{n}{lmfit}\PYG{o}{.}\PYG{n}{minimize}\PYG{p}{(}\PYG{n}{residual}\PYG{p}{,} \PYG{n}{p}\PYG{p}{)}

\PYG{g+gp}{In [9]: }\PYG{n}{mi}\PYG{o}{.}\PYG{n}{leastsq}\PYG{p}{(}\PYG{p}{)}
\PYG{g+gr}{Out[9]: }\PYG{n+nb+bp}{True}

\PYG{g+gp}{In [10]: }\PYG{n}{lmfit}\PYG{o}{.}\PYG{n}{printfuncs}\PYG{o}{.}\PYG{n}{report\PYGZus{}errors}\PYG{p}{(}\PYG{n}{mi}\PYG{o}{.}\PYG{n}{params}\PYG{p}{)}
\PYG{g+go}{  a:     0.100149 +/- 0.000192 (0.19\PYGZpc{}) initial =  0.100000}
\PYG{g+go}{  b:     2.005144 +/- 0.011988 (0.60\PYGZpc{}) initial =  1.000000}
\PYG{g+go}{Correlations:}
\PYG{g+go}{    C(a, b)                      =  0.601 }
\end{Verbatim}

Now it just a simple function call to start the calculation:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{g+gp}{In [11]: }\PYG{n}{ci}\PYG{o}{=}\PYG{n}{lmfit}\PYG{o}{.}\PYG{n}{conf\PYGZus{}interval}\PYG{p}{(}\PYG{n}{mi}\PYG{p}{)}

\PYG{g+gp}{In [12]: }\PYG{n}{lmfit}\PYG{o}{.}\PYG{n}{printfuncs}\PYG{o}{.}\PYG{n}{report\PYGZus{}ci}\PYG{p}{(}\PYG{n}{ci}\PYG{p}{)}
\PYG{g+go}{     99.70\PYGZpc{}    95.00\PYGZpc{}    67.40\PYGZpc{}     0.00\PYGZpc{}    67.40\PYGZpc{}    95.00\PYGZpc{}    99.70\PYGZpc{}}
\PYG{g+go}{a   0.09938   0.09938   0.09938   0.10015   0.10092   0.10092   0.10092}
\PYG{g+go}{b   1.96908   1.98150   1.99364   2.00514   2.01665   2.02878   2.04121}
\end{Verbatim}

As we can see, the estimated error is almost the same:
it is not necessary to caclulate ci's for this problem.


\section{An advanced example}
\label{confidence:an-advanced-example}
Now we look at a problem, where calculating the error from approximated
covariance can lead to wrong results:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{g+gp}{In [14]: }\PYG{n}{y}\PYG{o}{=}\PYG{l+m+mi}{3}\PYG{o}{*}\PYG{n}{np}\PYG{o}{.}\PYG{n}{exp}\PYG{p}{(}\PYG{o}{-}\PYG{n}{x}\PYG{o}{/}\PYG{l+m+mf}{2.}\PYG{p}{)}\PYG{o}{-}\PYG{l+m+mi}{5}\PYG{o}{*}\PYG{n}{np}\PYG{o}{.}\PYG{n}{exp}\PYG{p}{(}\PYG{o}{-}\PYG{n}{x}\PYG{o}{/}\PYG{l+m+mf}{10.}\PYG{p}{)}\PYG{o}{+}\PYG{l+m+mf}{0.2}\PYG{o}{*}\PYG{n}{np}\PYG{o}{.}\PYG{n}{random}\PYG{o}{.}\PYG{n}{randn}\PYG{p}{(}\PYG{n}{x}\PYG{o}{.}\PYG{n}{size}\PYG{p}{)}

\PYG{g+gp}{In [15]: }\PYG{n}{p}\PYG{o}{=}\PYG{n}{lmfit}\PYG{o}{.}\PYG{n}{Parameters}\PYG{p}{(}\PYG{p}{)}

\PYG{g+gp}{In [16]: }\PYG{n}{p}\PYG{o}{.}\PYG{n}{add\PYGZus{}many}\PYG{p}{(}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{a1}\PYG{l+s}{'}\PYG{p}{,}\PYG{l+m+mi}{5}\PYG{p}{)}\PYG{p}{,}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{a2}\PYG{l+s}{'}\PYG{p}{,}\PYG{o}{-}\PYG{l+m+mi}{5}\PYG{p}{)}\PYG{p}{,}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{t1}\PYG{l+s}{'}\PYG{p}{,}\PYG{l+m+mi}{2}\PYG{p}{)}\PYG{p}{,}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{t2}\PYG{l+s}{'}\PYG{p}{,}\PYG{l+m+mi}{5}\PYG{p}{)}\PYG{p}{)}

\PYG{g+gp}{In [17]: }\PYG{k}{def} \PYG{n+nf}{residual}\PYG{p}{(}\PYG{n}{p}\PYG{p}{)}\PYG{p}{:}
\PYG{g+gp}{   ....:}      \PYG{n}{a1}\PYG{p}{,}\PYG{n}{a2}\PYG{p}{,}\PYG{n}{t1}\PYG{p}{,}\PYG{n}{t2}\PYG{o}{=}\PYG{p}{[}\PYG{n}{i}\PYG{o}{.}\PYG{n}{value} \PYG{k}{for} \PYG{n}{i} \PYG{o+ow}{in} \PYG{n}{p}\PYG{o}{.}\PYG{n}{values}\PYG{p}{(}\PYG{p}{)}\PYG{p}{]}
\PYG{g+gp}{   ....:}      \PYG{k}{return} \PYG{n}{a1}\PYG{o}{*}\PYG{n}{np}\PYG{o}{.}\PYG{n}{exp}\PYG{p}{(}\PYG{o}{-}\PYG{n}{x}\PYG{o}{/}\PYG{n}{t1}\PYG{p}{)}\PYG{o}{+}\PYG{n}{a2}\PYG{o}{*}\PYG{n}{np}\PYG{o}{.}\PYG{n}{exp}\PYG{p}{(}\PYG{o}{-}\PYG{n}{x}\PYG{o}{/}\PYG{n}{t2}\PYG{p}{)}\PYG{o}{-}\PYG{n}{y}
\PYG{g+gp}{   ....:}
\end{Verbatim}

Now lets fit it:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{g+gp}{In [18]: }\PYG{n}{mi}\PYG{o}{=}\PYG{n}{lmfit}\PYG{o}{.}\PYG{n}{minimize}\PYG{p}{(}\PYG{n}{residual}\PYG{p}{,} \PYG{n}{p}\PYG{p}{)}

\PYG{g+gp}{In [19]: }\PYG{n}{mi}\PYG{o}{.}\PYG{n}{leastsq}\PYG{p}{(}\PYG{p}{)}
\PYG{g+gr}{Out[19]: }\PYG{n+nb+bp}{True}

\PYG{g+gp}{In [20]: }\PYG{n}{lmfit}\PYG{o}{.}\PYG{n}{printfuncs}\PYG{o}{.}\PYG{n}{report\PYGZus{}errors}\PYG{p}{(}\PYG{n}{mi}\PYG{o}{.}\PYG{n}{params}\PYG{p}{,} \PYG{n}{show\PYGZus{}correl}\PYG{o}{=}\PYG{n+nb+bp}{False}\PYG{p}{)}
\PYG{g+go}{  a1:     2.611014 +/- 0.327959 (12.56\PYGZpc{}) initial =  5.000000}
\PYG{g+go}{  a2:    -4.512928 +/- 0.399194 (8.85\PYGZpc{}) initial = -5.000000}
\PYG{g+go}{  t1:     1.569477 +/- 0.334505 (21.31\PYGZpc{}) initial =  2.000000}
\PYG{g+go}{  t2:     10.961366 +/- 1.263868 (11.53\PYGZpc{}) initial =  5.000000}
\end{Verbatim}

Again we call {\hyperref[confidence:confidence.conf_interval]{\code{conf\_interval()}}}, this time with tracing and only for 1-
and 2-sigma:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{g+gp}{In [21]: }\PYG{n}{ci}\PYG{p}{,} \PYG{n}{trace} \PYG{o}{=} \PYG{n}{lmfit}\PYG{o}{.}\PYG{n}{conf\PYGZus{}interval}\PYG{p}{(}\PYG{n}{mi}\PYG{p}{,}\PYG{n}{sigmas}\PYG{o}{=}\PYG{p}{[}\PYG{l+m+mf}{0.68}\PYG{p}{,}\PYG{l+m+mf}{0.95}\PYG{p}{]}\PYG{p}{,}\PYG{n}{trace}\PYG{o}{=}\PYG{n+nb+bp}{True}\PYG{p}{,} \PYG{n}{verbose}\PYG{o}{=}\PYG{l+m+mi}{0}\PYG{p}{)}

\PYG{g+gp}{In [22]: }\PYG{n}{lmfit}\PYG{o}{.}\PYG{n}{printfuncs}\PYG{o}{.}\PYG{n}{report\PYGZus{}ci}\PYG{p}{(}\PYG{n}{ci}\PYG{p}{)}
\PYG{g+go}{      95.00\PYGZpc{}    68.00\PYGZpc{}     0.00\PYGZpc{}    68.00\PYGZpc{}    95.00\PYGZpc{}}
\PYG{g+go}{a1   2.11696   2.33696   2.61101   3.06631   4.28728}
\PYG{g+go}{a2  -6.39492  -5.05982  -4.51293  -4.19528  -3.97850}
\PYG{g+go}{t2   8.00414   9.62688  10.96137  12.17947  13.34824}
\PYG{g+go}{t1   1.07036   1.28482   1.56948   1.97534   2.64341}
\end{Verbatim}

If you compare the calculated error estimates, you will see that the
regular estimate is too small. Now let's plot a confidence region:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{g+gp}{In [23]: }\PYG{k+kn}{import} \PYG{n+nn}{matplotlib.pylab} \PYG{k+kn}{as} \PYG{n+nn}{plt}

\PYG{g+gp}{In [24]: }\PYG{n}{x}\PYG{p}{,} \PYG{n}{y}\PYG{p}{,} \PYG{n}{grid}\PYG{o}{=}\PYG{n}{lmfit}\PYG{o}{.}\PYG{n}{conf\PYGZus{}interval2d}\PYG{p}{(}\PYG{n}{mi}\PYG{p}{,}\PYG{l+s}{'}\PYG{l+s}{a1}\PYG{l+s}{'}\PYG{p}{,}\PYG{l+s}{'}\PYG{l+s}{t2}\PYG{l+s}{'}\PYG{p}{,}\PYG{l+m+mi}{30}\PYG{p}{,}\PYG{l+m+mi}{30}\PYG{p}{)}

\PYG{g+gp}{In [25]: }\PYG{n}{plt}\PYG{o}{.}\PYG{n}{contourf}\PYG{p}{(}\PYG{n}{x}\PYG{p}{,}\PYG{n}{y}\PYG{p}{,}\PYG{n}{grid}\PYG{p}{,}\PYG{n}{np}\PYG{o}{.}\PYG{n}{linspace}\PYG{p}{(}\PYG{l+m+mi}{0}\PYG{p}{,}\PYG{l+m+mi}{1}\PYG{p}{,}\PYG{l+m+mi}{11}\PYG{p}{)}\PYG{p}{)}
\PYG{g+gr}{Out[25]: }\PYG{o}{\PYGZlt{}}\PYG{n}{matplotlib}\PYG{o}{.}\PYG{n}{contour}\PYG{o}{.}\PYG{n}{QuadContourSet} \PYG{n}{instance} \PYG{n}{at} \PYG{l+m+mh}{0xb7ae6ac}\PYG{o}{\PYGZgt{}}

\PYG{g+gp}{In [26]: }\PYG{n}{plt}\PYG{o}{.}\PYG{n}{xlabel}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{a1}\PYG{l+s}{'}\PYG{p}{)}\PYG{p}{;}

\PYG{g+gp}{In [27]: }\PYG{n}{plt}\PYG{o}{.}\PYG{n}{colorbar}\PYG{p}{(}\PYG{p}{)}\PYG{p}{;}

\PYG{g+gp}{In [28]: }\PYG{n}{plt}\PYG{o}{.}\PYG{n}{ylabel}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{t2}\PYG{l+s}{'}\PYG{p}{)}\PYG{p}{;}
\end{Verbatim}

\includegraphics[width=7in]{conf_interval.png}

Remember the trace? It shows the dependence between two parameters.

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{g+gp}{In [33]: }\PYG{n}{x}\PYG{p}{,}\PYG{n}{y}\PYG{p}{,}\PYG{n}{prob}\PYG{o}{=}\PYG{n}{trace}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{a1}\PYG{l+s}{'}\PYG{p}{]}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{a1}\PYG{l+s}{'}\PYG{p}{]}\PYG{p}{,} \PYG{n}{trace}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{a1}\PYG{l+s}{'}\PYG{p}{]}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{t2}\PYG{l+s}{'}\PYG{p}{]}\PYG{p}{,}\PYG{n}{trace}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{a1}\PYG{l+s}{'}\PYG{p}{]}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{prob}\PYG{l+s}{'}\PYG{p}{]}

\PYG{g+gp}{In [34]: }\PYG{n}{x2}\PYG{p}{,}\PYG{n}{y2}\PYG{p}{,}\PYG{n}{prob2}\PYG{o}{=}\PYG{n}{trace}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{t2}\PYG{l+s}{'}\PYG{p}{]}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{t2}\PYG{l+s}{'}\PYG{p}{]}\PYG{p}{,} \PYG{n}{trace}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{t2}\PYG{l+s}{'}\PYG{p}{]}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{a1}\PYG{l+s}{'}\PYG{p}{]}\PYG{p}{,}\PYG{n}{trace}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{t2}\PYG{l+s}{'}\PYG{p}{]}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{prob}\PYG{l+s}{'}\PYG{p}{]}

\PYG{g+gp}{In [35]: }\PYG{n}{plt}\PYG{o}{.}\PYG{n}{scatter}\PYG{p}{(}\PYG{n}{x}\PYG{p}{,}\PYG{n}{y}\PYG{p}{,}\PYG{n}{c}\PYG{o}{=}\PYG{n}{prob}\PYG{p}{,}\PYG{n}{s}\PYG{o}{=}\PYG{l+m+mi}{30}\PYG{p}{)}
\PYG{g+gr}{Out[35]: }\PYG{o}{\PYGZlt{}}\PYG{n}{matplotlib}\PYG{o}{.}\PYG{n}{collections}\PYG{o}{.}\PYG{n}{PathCollection} \PYG{n}{at} \PYG{l+m+mh}{0xba91fac}\PYG{o}{\PYGZgt{}}

\PYG{g+gp}{In [36]: }\PYG{n}{plt}\PYG{o}{.}\PYG{n}{scatter}\PYG{p}{(}\PYG{n}{x2}\PYG{p}{,}\PYG{n}{y2}\PYG{p}{,}\PYG{n}{c}\PYG{o}{=}\PYG{n}{prob2}\PYG{p}{,}\PYG{n}{s}\PYG{o}{=}\PYG{l+m+mi}{30}\PYG{p}{)}
\PYG{g+gr}{Out[36]: }\PYG{o}{\PYGZlt{}}\PYG{n}{matplotlib}\PYG{o}{.}\PYG{n}{collections}\PYG{o}{.}\PYG{n}{PathCollection} \PYG{n}{at} \PYG{l+m+mh}{0xbaa84ac}\PYG{o}{\PYGZgt{}}
\end{Verbatim}

\includegraphics[width=7in]{conf_interval2.png}


\section{Documentation of methods}
\label{confidence:documentation-of-methods}\index{conf\_interval() (in module confidence)}

\begin{fulllineitems}
\phantomsection\label{confidence:confidence.conf_interval}\pysiglinewithargsret{\bfcode{conf\_interval}}{\emph{minimizer}, \emph{p\_names=None}, \emph{sigmas=(0.67400000000000004}, \emph{0.94999999999999996}, \emph{0.997)}, \emph{trace=False}, \emph{maxiter=200}, \emph{verbose=False}, \emph{prob\_func=None}}{}
Calculates the confidence interval for parameters
from the given minimizer.

The parameter for which the ci is calculated will be varied, while
the remaining parameters are reoptimized for minimizing chi-square.
The resulting chi-square is used  to calculate the probability with 
a given statistic e.g. F-statistic. This function uses a 1d-rootfinder
from scipy to find the values resulting in the searched confidence
region.
\begin{quote}\begin{description}
\item[{Parameters }] \leavevmode
\textbf{minimizer} : Minimizer
\begin{quote}

The minimizer to use, should be already fitted via leastsq.
\end{quote}

\textbf{p\_names} : list, optional
\begin{quote}

Names of the parameters for which the ci is calculated. If None,
the ci is calculated for every parameter.
\end{quote}

\textbf{sigmas} : list, optional
\begin{quote}

The probabilities (1-alpha) to find. Default is 1,2 and 3-sigma.
\end{quote}

\textbf{trace} : bool, optional
\begin{quote}

Defaults to False, if true, each result of a probability calculation 
is saved along with the parameter. This can be used to plot so
called ``profile traces''.
\end{quote}

\item[{Returns }] \leavevmode
\textbf{output} : dict
\begin{quote}

A dict, which contains a list of (sigma, vals)-tuples for each name.
\end{quote}

\textbf{trace\_dict} : dict
\begin{quote}

Only if trace is set true. Is a dict, the key is the parameter which
was fixed.The values are again a dict with the names as keys, but with
an additional key `prob'. Each contains an array of the corresponding 
values.
\end{quote}

\item[{Other Parameters}] \leavevmode
\textbf{maxiter} : int
\begin{quote}

Maximum of iteration to find an upper limit.
\end{quote}

\textbf{prob\_func} : \code{None} or callable
\begin{quote}

Function to calculate the probality from the opimized chi-square.
Default (\code{None}) uses built-in f\_compare (F test).
\end{quote}

\end{description}\end{quote}


\strong{See Also:}


{\hyperref[confidence:confidence.conf_interval2d]{\code{conf\_interval2d}}}


\paragraph{Examples}

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{k+kn}{from} \PYG{n+nn}{lmfit.printfuncs} \PYG{k+kn}{import} \PYG{o}{*}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{mini}\PYG{o}{=}\PYG{n}{minimize}\PYG{p}{(}\PYG{n}{some\PYGZus{}func}\PYG{p}{,} \PYG{n}{params}\PYG{p}{)}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{mini}\PYG{o}{.}\PYG{n}{leastsq}\PYG{p}{(}\PYG{p}{)}
\PYG{g+go}{True}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{report\PYGZus{}errors}\PYG{p}{(}\PYG{n}{params}\PYG{p}{)}
\PYG{g+gp}{... }\PYG{c}{\PYGZsh{}report}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{ci}\PYG{o}{=}\PYG{n}{conf\PYGZus{}interval}\PYG{p}{(}\PYG{n}{mini}\PYG{p}{)}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{report\PYGZus{}ci}\PYG{p}{(}\PYG{n}{ci}\PYG{p}{)}
\PYG{g+gp}{... }\PYG{c}{\PYGZsh{}report}
\end{Verbatim}

Now with quantils for the sigmas and using the trace.

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{ci}\PYG{p}{,} \PYG{n}{trace}\PYG{o}{=}\PYG{n}{conf\PYGZus{}interval}\PYG{p}{(}\PYG{n}{mini}\PYG{p}{,} \PYG{n}{sigmas}\PYG{o}{=}\PYG{p}{(}\PYG{l+m+mf}{0.25}\PYG{p}{,}\PYG{l+m+mf}{0.5}\PYG{p}{,}\PYG{l+m+mf}{0.75}\PYG{p}{,}\PYG{l+m+mf}{0.999}\PYG{p}{)}\PYG{p}{,}\PYG{n}{trace}\PYG{o}{=}\PYG{n+nb+bp}{True}\PYG{p}{)}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{fixed}\PYG{o}{=}\PYG{n}{trace}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{para1}\PYG{l+s}{'}\PYG{p}{]}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{para1}\PYG{l+s}{'}\PYG{p}{]}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{free}\PYG{o}{=}\PYG{n}{trace}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{para1}\PYG{l+s}{'}\PYG{p}{]}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{not\PYGZus{}para1}\PYG{l+s}{'}\PYG{p}{]}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{prob}\PYG{o}{=}\PYG{n}{trace}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{para1}\PYG{l+s}{'}\PYG{p}{]}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{prob}\PYG{l+s}{'}\PYG{p}{]}
\end{Verbatim}

This makes it possible to plot the dependence between free and fixed.

\end{fulllineitems}

\index{conf\_interval2d() (in module confidence)}

\begin{fulllineitems}
\phantomsection\label{confidence:confidence.conf_interval2d}\pysiglinewithargsret{\bfcode{conf\_interval2d}}{\emph{minimizer}, \emph{x\_name}, \emph{y\_name}, \emph{nx=10}, \emph{ny=10}, \emph{limits=None}, \emph{prob\_func=None}}{}
Calculates confidence regions for two fixed parameters.

The method is explained in \emph{conf\_interval}: here we are fixing
two parameters.
\begin{quote}\begin{description}
\item[{Parameters }] \leavevmode
\textbf{minimizer} : minimizer
\begin{quote}

The minimizer to use, should be already fitted via leastsq.
\end{quote}

\textbf{x\_name} : string
\begin{quote}

The name of the parameter which will be the x direction.
\end{quote}

\textbf{y\_name} : string
\begin{quote}

The name of the parameter which will be the y direction.
\end{quote}

\textbf{nx, ny} : ints, optional
\begin{quote}

Number of points.
\end{quote}

\textbf{limits} : tuple: optional
\begin{quote}

Should have the form ((x\_upper, x\_lower),(y\_upper, y\_lower)). If not
given, the default is 5 stderrs in each direction.
\end{quote}

\item[{Returns }] \leavevmode
\textbf{x} : (nx)-array
\begin{quote}

x-coordinates
\end{quote}

\textbf{y} : (ny)-array
\begin{quote}

y-coordinates
\end{quote}

\textbf{grid} : (nx,ny)-array
\begin{quote}

grid contains the calculated probabilities.
\end{quote}

\item[{Other Parameters}] \leavevmode
\textbf{prob\_func} : \code{None} or callable
\begin{quote}

Function to calculate the probality from the opimized chi-square.
Default (\code{None}) uses built-in f\_compare (F test).
\end{quote}

\end{description}\end{quote}
\paragraph{Examples}

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{k+kn}{from} \PYG{n+nn}{lmfit.printfuncs} \PYG{k+kn}{import} \PYG{o}{*}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{mini}\PYG{o}{=}\PYG{n}{minimize}\PYG{p}{(}\PYG{n}{some\PYGZus{}func}\PYG{p}{,} \PYG{n}{params}\PYG{p}{)}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{mini}\PYG{o}{.}\PYG{n}{leastsq}\PYG{p}{(}\PYG{p}{)}
\PYG{g+go}{True}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{x}\PYG{p}{,}\PYG{n}{y}\PYG{p}{,}\PYG{n}{gr}\PYG{o}{=}\PYG{n}{conf\PYGZus{}interval2d}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{para1}\PYG{l+s}{'}\PYG{p}{,}\PYG{l+s}{'}\PYG{l+s}{para2}\PYG{l+s}{'}\PYG{p}{)}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{plt}\PYG{o}{.}\PYG{n}{contour}\PYG{p}{(}\PYG{n}{x}\PYG{p}{,}\PYG{n}{y}\PYG{p}{,}\PYG{n}{gr}\PYG{p}{)}
\end{Verbatim}

\end{fulllineitems}



\chapter{Bounds Implementation}
\label{bounds:parameter-bounds-label}\label{bounds::doc}\label{bounds:bounds-implementation}
This section describes the implementation of {\hyperref[parameters:Parameter]{\code{Parameter}}} bounds.
The \href{http://en.wikipedia.org/wiki/MINPACK}{MINPACK-1} implementation used in \href{http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.leastsq.html}{scipy.optimize.leastsq} for the
Levenberg-Marquardt algorithm does not explicitly support bounds on
parameters, and expects to be able to fully explore the available range of
values for any Parameter.  Simply placing hard constraints (that is,
resetting the value when it exceeds the desired bounds) prevents the
algorithm from determining the partial derivatives, and leads to unstable
results.

Instead of placing such hard constraints, bounded parameters are
mathematically transformed using the formulation devised (and documented)
for \href{http://en.wikipedia.org/wiki/MINUIT}{MINUIT}.  This is implemented following (and borrowing heavily from)
the \href{https://github.com/jjhelmus/leastsqbound-scipy}{leastsqbound} from J. J. Helmus.   Parameter values are mapped from
internally used, freely variable values $P_{\rm internal}$ to bounded
parameters $P_{\rm bounded}$.   When both \code{min} and \code{max} bounds
are specified, the mapping is
\begin{eqnarray*}
     P_{\rm internal} &=& \arcsin\big(\frac{2 (P_{\rm bounded} - {\rm min})}{({\rm max} - {\rm min})} - 1\big) \\
     P_{\rm bounded}  &=& {\rm min} + \big(\sin(P_{\rm internal}) + 1\big) \frac{({\rm max} - {\rm min})}{2}
 \end{eqnarray*}
With only an upper limit \code{max} supplied, but \code{min} left unbounded, the
mapping is:
\begin{eqnarray*}
     P_{\rm internal} &=& \sqrt{({\rm max} - P_{\rm bounded} + 1)^2 - 1} \\
     P_{\rm bounded}  &=& {\rm max} + 1 - \sqrt{P_{\rm internal}^2 + 1}
 \end{eqnarray*}
With only a lower limit \code{min} supplied, but \code{max} left unbounded, the
mapping is:
\begin{eqnarray*}
     P_{\rm internal} &=& \sqrt{(P_{\rm bounded} - {\rm min} + 1)^2 - 1} \\
     P_{\rm bounded}  &=& {\rm min} - 1 + \sqrt{P_{\rm internal}^2 + 1}
\end{eqnarray*}
With these mappings, the value for the bounded Parameter cannot exceed the
specified bounds, though the internally varied value can be freely varied.

It bears repeating that code from \href{https://github.com/jjhelmus/leastsqbound-scipy}{leastsqbound} was adopted to implement
the transformation described above.  The challenging part (Thanks again to
Jonathan J. Helmus!) here is to re-transform the covariance matrix so that
the uncertainties can be estimated for bounded Parameters.  This is
included by using the derivate $dP_{\rm internal}/dP_{\rm bounded}$
from the equations above to re-scale the jacobian matrix before
constructing the covariance matrix from it.  Tests show that this
re-scaling of the covariance matrix works quite well, and that
uncertainties estimated for bounded are quite reasonable.  Of course, if
the best fit value is very close to a boundary, the derivative estimated
uncertainty and correlations for that parameter may not be reliable.

The \href{http://en.wikipedia.org/wiki/MINUIT}{MINUIT} documentation recommends caution in using bounds.  Setting
bounds can certainly increase the number of function evaluations (and so
computation time), and in some cases may cause some instabilities, as the
range of acceptable parameter values is not fully explored.  On the other
hand, prelminary tests suggest that using \code{max} and \code{min} to set
clearly outlandish bounds does not greatly affect performance or results.


\chapter{Using Mathematical Constraints}
\label{constraints:using-mathematical-constraints}\label{constraints:math-constraints-label}\label{constraints::doc}
While being able to fix variables and place upper and lower bounds on their
values are key parts of lmfit, the ability to place mathematical
constraints on parameters is also highly desirable.  This section describes
how to do this, and what sort of parameterizations are possible -- see
the \href{http://newville.github.com/asteval/}{asteval} for further documentation.


\section{Overview}
\label{constraints:overview}
Just as one can place bounds on a Parameter, or keep it fixed during the
fit, so too can one place mathematical constraints on parameters.  The way
this is done with lmfit is to write a Parameter as a mathematical
expression of the other parameters and a set of pre-defined operators and
functions.   The constraint expressions are simple Python statements,
allowing one to place constraints like:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{n}{pars} \PYG{o}{=} \PYG{n}{Parameters}\PYG{p}{(}\PYG{p}{)}
\PYG{n}{pars}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{frac\PYGZus{}curve1}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value}\PYG{o}{=}\PYG{l+m+mf}{0.5}\PYG{p}{,} \PYG{n+nb}{min}\PYG{o}{=}\PYG{l+m+mi}{0}\PYG{p}{,} \PYG{n+nb}{max}\PYG{o}{=}\PYG{l+m+mi}{1}\PYG{p}{)}
\PYG{n}{pars}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{frac\PYGZus{}curve2}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{expr}\PYG{o}{=}\PYG{l+s}{'}\PYG{l+s}{1-frac\PYGZus{}curve1}\PYG{l+s}{'}\PYG{p}{)}
\end{Verbatim}

as the value of the \emph{frac\_curve1} parameter is updated at each step in the
fit, the value of \emph{frac\_curve2} will be updated so that the two values are
constrained to add to 1.0.  Of course, such a constraint could be placed in
the fitting function, but the use of such constraints allows the end-user
to modify the model of a more general-purpose fitting function.

Nearly any valid mathematical expression can be used, and a variety of
built-in functions are available for flexible modeling.


\section{Supported Operators, Functions, and Constants}
\label{constraints:supported-operators-functions-and-constants}
The mathematical expressions used to define constrained Parameters need to
be valid python expressions.  As you'd expect, the operators `+', `-`, `*',
`/', `**', are supported.  In fact, a much more complete set can be used,
including Python's bit- and logical operators:

\begin{Verbatim}[commandchars=\\\{\}]
+, -, *, /, **, \&, \textbar{}, \textasciicircum{}, \textless{}\textless{}, \textgreater{}\textgreater{}, \%, and, or,
==, \textgreater{}, \textgreater{}=, \textless{}, \textless{}=, !=, \textasciitilde{}, not, is, is not, in, not in
\end{Verbatim}

The values for \emph{e} (2.7182818...) and \emph{pi} (3.1415926...) are available, as
are  several supported mathematical and trigonometric function:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{n+nb}{abs}\PYG{p}{,} \PYG{n}{acos}\PYG{p}{,} \PYG{n}{acosh}\PYG{p}{,} \PYG{n}{asin}\PYG{p}{,} \PYG{n}{asinh}\PYG{p}{,} \PYG{n}{atan}\PYG{p}{,} \PYG{n}{atan2}\PYG{p}{,} \PYG{n}{atanh}\PYG{p}{,} \PYG{n}{ceil}\PYG{p}{,}
\PYG{n}{copysign}\PYG{p}{,} \PYG{n}{cos}\PYG{p}{,} \PYG{n}{cosh}\PYG{p}{,} \PYG{n}{degrees}\PYG{p}{,} \PYG{n}{exp}\PYG{p}{,} \PYG{n}{fabs}\PYG{p}{,} \PYG{n}{factorial}\PYG{p}{,}
\PYG{n}{floor}\PYG{p}{,} \PYG{n}{fmod}\PYG{p}{,} \PYG{n}{frexp}\PYG{p}{,} \PYG{n}{fsum}\PYG{p}{,} \PYG{n}{hypot}\PYG{p}{,} \PYG{n}{isinf}\PYG{p}{,} \PYG{n}{isnan}\PYG{p}{,} \PYG{n}{ldexp}\PYG{p}{,}
\PYG{n}{log}\PYG{p}{,} \PYG{n}{log10}\PYG{p}{,} \PYG{n}{log1p}\PYG{p}{,} \PYG{n+nb}{max}\PYG{p}{,} \PYG{n+nb}{min}\PYG{p}{,} \PYG{n}{modf}\PYG{p}{,} \PYG{n+nb}{pow}\PYG{p}{,} \PYG{n}{radians}\PYG{p}{,} \PYG{n}{sin}\PYG{p}{,}
\PYG{n}{sinh}\PYG{p}{,} \PYG{n}{sqrt}\PYG{p}{,} \PYG{n}{tan}\PYG{p}{,} \PYG{n}{tanh}\PYG{p}{,} \PYG{n}{trunc}
\end{Verbatim}

In addition, all Parameter names will be available in the mathematical
expressions.  Thus, with parameters for a few peak-like functions:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{n}{pars} \PYG{o}{=} \PYG{n}{Parameters}\PYG{p}{(}\PYG{p}{)}
\PYG{n}{pars}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{amp\PYGZus{}1}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value}\PYG{o}{=}\PYG{l+m+mf}{0.5}\PYG{p}{,} \PYG{n+nb}{min}\PYG{o}{=}\PYG{l+m+mi}{0}\PYG{p}{,} \PYG{n+nb}{max}\PYG{o}{=}\PYG{l+m+mi}{1}\PYG{p}{)}
\PYG{n}{pars}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{cen\PYGZus{}1}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value}\PYG{o}{=}\PYG{l+m+mf}{2.2}\PYG{p}{)}
\PYG{n}{pars}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{wid\PYGZus{}1}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value}\PYG{o}{=}\PYG{l+m+mf}{0.2}\PYG{p}{)}
\end{Verbatim}

The following expression are all valid:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{n}{pars}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{amp\PYGZus{}2}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{expr}\PYG{o}{=}\PYG{l+s}{'}\PYG{l+s}{(2.0 - amp\PYGZus{}1**2)}\PYG{l+s}{'}\PYG{p}{)}
\PYG{n}{pars}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{cen\PYGZus{}2}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{expr}\PYG{o}{=}\PYG{l+s}{'}\PYG{l+s}{cen\PYGZus{}1 * wid\PYGZus{}2 / max(wid\PYGZus{}1, 0.001)}\PYG{l+s}{'}\PYG{p}{)}
\PYG{n}{pars}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{wid\PYGZus{}2}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{expr}\PYG{o}{=}\PYG{l+s}{'}\PYG{l+s}{sqrt(pi)*wid\PYGZus{}1}\PYG{l+s}{'}\PYG{p}{)}
\end{Verbatim}

In fact, almost any valid Python expression is allowed.  A notable example
is that Python's 1-line \emph{if expression} is supported:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{n}{pars}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{bounded}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{expr}\PYG{o}{=}\PYG{l+s}{'}\PYG{l+s}{param\PYGZus{}a if test\PYGZus{}val/2. \PYGZgt{} 100 else param\PYGZus{}b}\PYG{l+s}{'}\PYG{p}{)}
\end{Verbatim}

which is equivalent to the more familiar:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{k}{if} \PYG{n}{test\PYGZus{}val}\PYG{o}{/}\PYG{l+m+mf}{2.} \PYG{o}{\PYGZgt{}} \PYG{l+m+mi}{100}\PYG{p}{:}
    \PYG{n}{bounded} \PYG{o}{=} \PYG{n}{param\PYGZus{}a}
\PYG{k}{else}\PYG{p}{:}
    \PYG{n}{bounded} \PYG{o}{=} \PYG{n}{param\PYGZus{}b}
\end{Verbatim}


\section{Using Inequality Constraints}
\label{constraints:using-inequality-constraints}
A rather common question about how to set up constraints
that use an inequality, say, $x + y \le 10$.  This
can be done with algebraic constraints by recasting the
problem, as $x + y = \delta$ and $\delta \le
10$.  That is, first, allow $x$ to be held by the
freely varying parameter \emph{x}.  Next, define a parameter
\emph{delta} to be variable with a maximum value of 10, and
define parameter \emph{y} as \emph{delta - x}:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{n}{pars} \PYG{o}{=} \PYG{n}{Parameters}\PYG{p}{(}\PYG{p}{)}
\PYG{n}{pars}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{x}\PYG{l+s}{'}\PYG{p}{,}     \PYG{n}{value} \PYG{o}{=} \PYG{l+m+mi}{5}\PYG{p}{,} \PYG{n}{vary}\PYG{o}{=}\PYG{n+nb+bp}{True}\PYG{p}{)}
\PYG{n}{pars}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{delta}\PYG{l+s}{'}\PYG{p}{,} \PYG{n}{value} \PYG{o}{=} \PYG{l+m+mi}{5}\PYG{p}{,} \PYG{n+nb}{max}\PYG{o}{=}\PYG{l+m+mi}{10}\PYG{p}{,} \PYG{n}{vary}\PYG{o}{=}\PYG{n+nb+bp}{True}\PYG{p}{)}
\PYG{n}{pars}\PYG{o}{.}\PYG{n}{add}\PYG{p}{(}\PYG{l+s}{'}\PYG{l+s}{y}\PYG{l+s}{'}\PYG{p}{,}     \PYG{n}{expr}\PYG{o}{=}\PYG{l+s}{'}\PYG{l+s}{delta-x}\PYG{l+s}{'}\PYG{p}{)}
\end{Verbatim}

The essential point is that an inequality still implies
that a variable (here, \emph{delta}) is needed to describe the
constraint.  The secondary point is that upper and lower
bounds can be used as part of the inequality to make the
definitions more convenient.


\section{Advanced usage of Expressions in lmfit}
\label{constraints:advanced-usage-of-expressions-in-lmfit}
The expression used in a constraint is converted to a
Python \href{http://docs.python.org/library/ast.html}{Abstract Syntax Tree}, which is an
intermediate version of the expression -- a syntax-checked,
partially compiled expression.  Among other things, this
means that Python's own parser is used to parse and convert
the expression into something that can easily be evaluated
within Python.  It also means that the symbols in the
expressions can point to any Python object.

In fact, the use of Python's AST allows a nearly full version of Python to
be supported, without using Python's built-in \code{eval()} function.  The
\href{http://newville.github.com/asteval/}{asteval} module actually supports most Python syntax, including for- and
while-loops, conditional expressions, and user-defined functions.  There
are several unsupported Python constructs, most notably the class
statement, so that new classes cannot be created, and the import statement,
which helps make the \href{http://newville.github.com/asteval/}{asteval} module safe from malicious use.

One important feature of the \href{http://newville.github.com/asteval/}{asteval} module is that you can add
domain-specific functions into the it, for later use in constraint
expressions.  To do this, you would use the \code{asteval} attribute of
the {\hyperref[fitting:Minimizer]{\code{Minimizer}}} class, which contains a complete AST interpreter.
The \href{http://newville.github.com/asteval/}{asteval} interpreter uses a flat namespace, implemented as a single
dictionary. That means you can preload any Python symbol into the namespace
for the constraints:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{k}{def} \PYG{n+nf}{lorenztian}\PYG{p}{(}\PYG{n}{x}\PYG{p}{,} \PYG{n}{amp}\PYG{p}{,} \PYG{n}{cen}\PYG{p}{,} \PYG{n}{wid}\PYG{p}{)}\PYG{p}{:}
    \PYG{l+s}{"}\PYG{l+s}{lorenztian function: wid = half-width at half-max}\PYG{l+s}{"}
    \PYG{k}{return} \PYG{p}{(}\PYG{n}{amp}  \PYG{o}{/} \PYG{p}{(}\PYG{l+m+mi}{1} \PYG{o}{+} \PYG{p}{(}\PYG{p}{(}\PYG{n}{x}\PYG{o}{-}\PYG{n}{cen}\PYG{p}{)}\PYG{o}{/}\PYG{n}{wid}\PYG{p}{)}\PYG{o}{*}\PYG{o}{*}\PYG{l+m+mi}{2}\PYG{p}{)}\PYG{p}{)}

\PYG{n}{fitter} \PYG{o}{=} \PYG{n}{Minimizer}\PYG{p}{(}\PYG{p}{)}
\PYG{n}{fitter}\PYG{o}{.}\PYG{n}{asteval}\PYG{o}{.}\PYG{n}{symtable}\PYG{p}{[}\PYG{l+s}{'}\PYG{l+s}{lorenztian}\PYG{l+s}{'}\PYG{p}{]} \PYG{o}{=} \PYG{n}{lorenztian}
\end{Verbatim}

and this \code{lorenztian()} function can now be used in constraint
expressions.


\renewcommand{\indexname}{Python Module Index}
\begin{theindex}
\def\bigletter#1{{\Large\sffamily#1}\nopagebreak\vspace{1mm}}
\bigletter{c}
\item {\texttt{confidence}}, \pageref{confidence:module-confidence}
\end{theindex}
\renewcommand{\indexname}{Python Module Index}
\begin{theindex}
\def\bigletter#1{{\Large\sffamily#1}\nopagebreak\vspace{1mm}}
\bigletter{c}
\item {\texttt{confidence}}, \pageref{confidence:module-confidence}
\end{theindex}

\renewcommand{\indexname}{Index}
\printindex
\end{document}
