\documentclass[]{article}

% definitions
\def\D{\partial}

\def\solidline{{$\textcolor{blue}{\overline{\hskip 0.5cm}}\ $}}
\def\dashedline{{$\textcolor{red}{\overline{\hskip 0.1cm}\ \overline{\hskip 0.1cm}\ \overline{\hskip 0.1cm}\ }$}}
\def\dashdottedline{$\overline{\hskip 0.1cm}\ \overline{\hskip 0.03cm}\ \overline{\hskip 0.1cm}\ $}
\def\dottedline{$\overline{\hskip 0.04cm}\ \overline{\hskip 0.04cm}\ \overline{\hskip 0.04cm}\ $}

\usepackage{amsmath,amssymb,amsthm}
\newtheorem{thm}{Theorem}
\usepackage{lettrine}
\usepackage{amsfonts}
\usepackage[pdftex]{graphicx}
\usepackage[latin1]{inputenc}
\usepackage{epstopdf}
\usepackage{hyperref}
\usepackage{placeins}
\usepackage{bigints}
\usepackage{gensymb}
\usepackage{float}
\usepackage{color}
\usepackage{graphicx}
\usepackage{graphics}
\usepackage[hmarginratio=1:1,top=25mm,columnsep=20pt]{geometry}
\usepackage[font=it]{caption}
\usepackage{paralist}
\usepackage{multicol}
\usepackage{lettrine}
\usepackage[nodayofweek]{datetime}
\usepackage{centernot}
\usepackage{ mathrsfs }
\usepackage{textcomp}
\usepackage{color}
%\usepackage{listings}
\usepackage{setspace}
\usepackage{xcolor}
\usepackage{subcaption}
\usepackage{color}
\usepackage[procnames]{listings}
\usepackage{setspace}
\usepackage{palatino}
\usepackage{multirow}
\usepackage{algorithm2e}
\usepackage[authoryear]{natbib}

\renewcommand{\lstlistlistingname}{Code Listings}
\renewcommand{\lstlistingname}{Code Listing}
\definecolor{gray}{gray}{0.5}
\definecolor{green}{rgb}{0,0.5,0}
\definecolor{lightgreen}{rgb}{0,0.7,0}
\definecolor{purple}{rgb}{0.5,0,0.5}
\definecolor{darkred}{rgb}{0.5,0,0}
\lstnewenvironment{python}[1][]{
\lstset{
language=python,
basicstyle=\ttfamily\small\setstretch{1},
stringstyle=\color{green},
showstringspaces=false,
alsoletter={1234567890},
otherkeywords={\ , \}, \{},
keywordstyle=\color{blue},
emph={access,and,as,break,class,continue,def,del,elif,else,%
except,exec,finally,for,from,global,if,import,in,is,%
lambda,not,or,pass,print,raise,return,try,while,assert},
emphstyle=\color{orange}\bfseries,
emph={[2]self},
emphstyle=[2]\color{gray},
emph={[4]ArithmeticError,AssertionError,AttributeError,BaseException,%
DeprecationWarning,EOFError,Ellipsis,EnvironmentError,Exception,%
False,FloatingPointError,FutureWarning,GeneratorExit,IOError,%
ImportError,ImportWarning,IndentationError,IndexError,KeyError,%
KeyboardInterrupt,LookupError,MemoryError,NameError,None,%
NotImplemented,NotImplementedError,OSError,OverflowError,%
PendingDeprecationWarning,ReferenceError,RuntimeError,RuntimeWarning,%
StandardError,StopIteration,SyntaxError,SyntaxWarning,SystemError,%
SystemExit,TabError,True,TypeError,UnboundLocalError,UnicodeDecodeError,%
UnicodeEncodeError,UnicodeError,UnicodeTranslateError,UnicodeWarning,%
UserWarning,ValueError,Warning,ZeroDivisionError,abs,all,any,apply,%
basestring,bool,buffer,callable,chr,classmethod,cmp,coerce,compile,%
complex,copyright,credits,delattr,dict,dir,divmod,enumerate,eval,%
execfile,exit,file,filter,float,frozenset,getattr,globals,hasattr,%
hash,help,hex,id,input,int,intern,isinstance,issubclass,iter,len,%
license,list,locals,long,map,max,min,object,oct,open,ord,pow,property,%
quit,range,raw_input,reduce,reload,repr,reversed,round,set,setattr,%
slice,sorted,staticmethod,str,sum,super,tuple,type,unichr,unicode,%
vars,xrange,zip},
emphstyle=[4]\color{purple}\bfseries,
upquote=true,
morecomment=[s][\color{lightgreen}]{"""}{"""},
commentstyle=\color{red}\slshape,
literate={>>>}{\textbf{\textcolor{darkred}{>{>}>}}}3%
         {...}{{\textcolor{gray}{...}}}3,
procnamekeys={def,class},
procnamestyle=\color{blue}\textbf,
framexleftmargin=1mm, framextopmargin=1mm, frame=shadowbox,
rulesepcolor=\color{blue},#1
}}{}


%%%%% Titling
\usepackage[compact]{titlesec}
\newcommand{\overbar}[1]{\mkern 1.5mu\overline{\mkern-1.5mu#1\mkern-1.5mu}\mkern 1.5mu}

\newenvironment{corollary}[1]{\par\noindent\underline{Corollary:}\space#1}{}
\newenvironment{lemma}[1]{\par\noindent\underline{Lemma:}\space#1}{}
\newenvironment{theorem}[1]{\par\noindent\underline{Theorem:}\space#1}{}
\newenvironment{claim}[1]{\par\noindent\underline{Claim:}\space#1}{}
\newenvironment{proposition}[1]{\par\noindent\underline{Proposition:}\space#1}{}
\newenvironment{claimproof}[1]{\par\noindent\underline{Proof:}\space#1}{\hfill\ensuremath{\square}}

\include{pythonlisting}

% ------
% Header/footer
\usepackage{fancyhdr}
	\pagestyle{fancy}
	\fancyhead{}
	\fancyfoot{}
	\fancyhead[C]{Surrogate Optimization Toolbox in Python (pySOT) - 0.1.21 $\bullet$ 
	Tutorial $\bullet$ David Eriksson, David Bindel, Christine Shoemaker $\bullet$ \today}
	\fancyfoot[RO,LE]{\thepage}

%%% ---- MATLAB CODE ---- %%%

\definecolor{Blue}{rgb}{0.1,0.1,0.3}
\hypersetup{colorlinks=true,linkcolor=Blue,citecolor=Blue,urlcolor=Blue}

\definecolor{listinggray}{gray}{0.9} 
\definecolor{lbcolor}{rgb}{0.9,0.9,0.9}
\definecolor{dorange}{rgb}{1.000000,0.549020,0.000000}
\definecolor{lblue}{rgb}{0.529412,0.807843,0.980392}
\lstset{escapeinside={<@}{@>}}


\definecolor{orange}{rgb}{1,0.5,0}
\DeclareMathOperator{\sgn}{sgn}
\DeclareMathOperator{\tr}{tr}
\DeclareMathOperator{\argmax}{argmax}
\DeclareMathOperator{\X}{\mathcal{X}}
\DeclareMathOperator{\Y}{\mathcal{Y}}
\DeclareMathOperator{\Z}{\mathbb{Z}}
\DeclareMathOperator{\Fx}{\mathcal{F}_{\X}}
\DeclareMathOperator{\Fy}{\mathcal{F}_{\Y}}
\DeclareMathOperator{\Fz}{\mathcal{F}_{\Z}}
\DeclareMathOperator{\Nc}{\mathcal{N}}
\DeclareMathOperator{\Rc}{\mathcal{R}}
\DeclareMathOperator{\B}{\mathcal{B}}
\DeclareMathOperator{\C}{\mathbb{C}}
\DeclareMathOperator{\Rb}{\mathbb{R}}
\DeclareMathOperator{\Sb}{\mathcal{S}}
\DeclareMathOperator{\N}{\mathbb{N}}
\DeclareMathOperator{\Q}{\mathbb{Q}}
\DeclareMathOperator{\limn}{\lim\limits_{n \to \infty}}
\DeclareMathOperator{\limm}{\lim\limits_{m \to \infty}}
\DeclareMathOperator{\R}{Re}
\DeclareMathOperator{\I}{Im}
\DeclareMathOperator{\spann}{span}
\newcommand*{\QED}{\hfill\ensuremath{\square}}%

%%%% Title
\title{\vspace{-15mm}%
	\fontsize{18pt}{10pt}\selectfont
	\textbf{Surrogate Optimization Toolbox (pySOT) - 0.1.21 \\ Tutorial}
	}	
\author{%
	\Large\textsc{David Eriksson} \\[2mm]
	\Large\textsc{David Bindel} \\[2mm]
	\Large\textsc{Christine Shoemaker} \\[2mm]
		\normalsize	Cornell University \\
	\normalsize Center for Applied Mathematics \\
	\normalsize	dme65@cornell.edu \\ 
	}
\date{\today}

\begin{document}
\fontsize{12}{14}\rm

\maketitle
\thispagestyle{fancy}
\tableofcontents
\newpage

\section{Change history:}
\begin{itemize}

	\item (\textbf{0.1.21})
	\begin{itemize}
		\item 	Added an option for supplying weights to the candidate point methods
		\item Cleaned up some of the tests by appending attributes to the workers
		\item Extended the MATLAB example to parallel
		\item Added a help function for doing a progress plot
	\end{itemize}

	\item (\textbf{0.1.20})
	\begin{itemize}
		\item Added some basic input checking (evaluations, dimensionality, etc)
		\item Added an example with a MATLAB engine in case the optimization problems is in MATLAB
		\item Fixed a bug in the polynomial regression
		\item Moved the merit function out of sampling\_methods.py
	\end{itemize}

	\item (\textbf{0.1.19})
	\begin{itemize}
		\item 	Too much regularization was added to the RBF surface when the volume of the domain was large.
	\end{itemize}

	\item (\textbf{0.1.18})
	\begin{itemize}
		\item Significant restructuring of the code base
		\item make\_points now takes an argument that specifies the number of new points to be generated
		\item Added Box-Behnken and 2-factorial to the experimental designs
		\item Simplified the penalty method strategy by moving evals and derivs into a surrogate wrapper
	\end{itemize}

	\item (\textbf{0.1.17})
	\begin{itemize}
		\item Added the possibility to input the penalty for the penalty method in the GUI
		\item Added the possibility of making a performance plot using matplotlib that adds new points 
		dynamically as evaluations are finished
		\item Switched from subprocess to subprocess32
	\end{itemize}

	\item (\textbf{0.1.16})
	\begin{itemize}
		\item Added a projection strategy
	\end{itemize}

	\item (\textbf{0.1.15})
	\begin{itemize}
		\item Added an example test\_subprocess\_files that shows how to use pySOT in 
		case the objective function needs to read the input from a textfile
	\end{itemize}

	\item (\textbf{0.1.14}) 
	\begin{itemize}
		\item Updated the Tutorial to reflect the changes for the last few months
		\item Simplified the object creation from strings in the GUI by importing directly from the namespace.
	\end{itemize}

	\item (\textbf{0.1.13}) 
	\begin{itemize}
		\item Allowed to still import the rest of pySOT when PySide is not found. In this case, the 
		GUI will be unavailable.
	\end{itemize}

	\item (\textbf{0.1.12}) 
	\begin{itemize}
		\item The capping can now take in a general transformation that is used to transform the function 
		values. Default is median capping.
		\item The Genetic Algorithm now defaults to initialize the population using a symmetric Latin hypercube
		\item DYCORS uses the remaining evaluation budget to change the probabilities after a restart instead 
		of using the total budget 
	\end{itemize}

	\item (\textbf{0.1.11}) 
	\begin{itemize}
		\item Fixed a bug in the capped response surface
		\item pySOT now internally works on the unit hypercube
		\item The distance can be passed to the RBF after being computed when generating candidate points 
		so it is not computed twice anymore
		\item Fixed some bugs in the candidate functions
		\item GA and Multi-Search gradient perturb the best solution in the case when the best solution is a 
		previously evaluated point
		\item Added an additional test for the multi-search strategy
	\end{itemize}

	\item (\textbf{0.1.10}) 
	\begin{itemize}
		\item README.md not uploaded to pypi which caused the pip install to fail
	\end{itemize}

	\item (\textbf{0.1.9}) 
	\begin{itemize}
		\item Fixed a bug in the merit function and several bugs in the DYCORS strategy
		\item Added a DDS candidate based strategy for searching on the surrogate
	\end{itemize}

	\item (\textbf{0.1.8}) 
	\begin{itemize}
		\item Multi Start Gradient method that uses the L-BFGS-B algorithm to search on the surroagate
	\end{itemize}

	\item (\textbf{0.1.7}) 
	\begin{itemize}
		\item Fixed some parameters (and bugs) to improve the DYCORS results. Using DYCORS together 
		with the genetic algorithm is recommended.
		\item Added polynomial regression (not yet in the GUI)
		\item Changed so that candidate points are generated using truncated normal distribution to avoid 
		projections onto the boundary
		\item Removed some accidental scikit dependencies in the ensemble surrogate
	\end{itemize}

	\item (\textbf{0.1.6}) 
	\begin{itemize}
		\item GUI inactivates all buttons but the stop button while running
		\item Bug fixes
	\end{itemize}

	\item (\textbf{0.1.5}) 
	\begin{itemize}
		\item GUI now has support for multiple search strategies and ensemble surrogates
		\item Reallocation bug in the ensemble surrogates fixed
		\item Genetic algorithm added to search on the surrogate
	\end{itemize}

	\item (\textbf{0.1.4}) 
	\begin{itemize}
		\item GUI now has improved error handling 
		\item Strategies informs the user if they get constraints when not expecting constraints 
		(and the other way) before the run starts
	\end{itemize}

	\item (\textbf{0.1.3}) 
	\begin{itemize}
		\item Experimental (but not documented) GUI added. You need PySide to use it.
		\item Changes in testproblems.py to allow external objective functions that implement 
		ProcessWorkerThread
		\item Added GUI test examples in documentation (Ackley.py, Keane.py, SphereExt.py)
	\end{itemize}

	\item (\textbf{0.1.2})
	\begin{itemize}
		\item 	Changed to using the logging module for all the logging in order to conform to the 
		changes in POAP 0.1.9
		\item The quiet and stream arguments in the strategies were removed and the tests 
		updated accordingly
		\item Turned sleeping of in the sub process test, to avoid platform dependency issues
	\end{itemize}

	\item (\textbf{0.1.1})
	\begin{itemize}
		\item surrogate\_optimizer.py was removed, so the user now has to create his own controller
		\item constraint\_method.py is gone, and the constraint handling is handled in specific 
		strategies instead
		\item 	There are now two strategies, SyncStrategyNoConstraints and SyncStrategyPenalty
		\item The search strategies now take a method for providing surrogate predictions rather 
		than keeping a copy of the response surface
		\item It is now possible for the user to provide additional points to be added to the initial 
		design, in case a 'good starting point' is known.
		\item Ensemble surrogates have been added to the toolbox
		\item 	The strategies takes an additional option 'quiet' so that all of the printing can be 
		avoided if the user wants
		\item There is also an option 'stream' in case the printing should be redirected somewhere 
		else, for example to a text file. Default is printing to stdout.
		\item 	Several examples added to pySOT.test
	\end{itemize}
	
	\item (\textbf{0.1.0})
	\begin{itemize}
		\item 	Initial release
	\end{itemize}
\end{itemize}

\section{Introduction}
This is a tutorial (user guide) for the Surrogate Optimization Toolbox (pySOT) for global deterministic 
optimization problems. The main purpose of the toolbox is for optimization of computationally expensive 
black-box objective functions with continuous and/or integer variables. We support inequality constraints 
of any form through a penalty method approach, but cannot yet efficiently handle equality constraints. 
All variables are assumed to have bound constraints in some form where none of the bounds are infinity. 
The tighter the bounds, the more efficient are the algorithms since it reduces the search region and 
increases the quality of the constructed surrogate. The longer the objective functions take to evaluate, 
the more efficient are these algorithms. For this reason, this toolbox may not be very efficient for problems 
with computationally cheap function evaluations. Surrogate models are intended to be used when function
 evaluations take from several minutes to several hours or more. The toolbox is based on the following 
 published papers that should be cited when the toolbox is used for own research purposes:
\begin{enumerate}
\item Stochastic RBF: \cite{regis2007stochastic}
\item Parallel Stochastic RBF: \cite{regis2009parallel}
\item DYCORS: \cite{regis2013combining}
\item Surrogate optimization using mixture surrogates: \cite{muller2011mixture}
\item Surrogate optimization for Mixed-Integer problems: \cite{muller2013so}
\item Surrogate optimization for Integer problems: \cite{muller2014so}
\end{enumerate}
For easier understanding of the algorithms in this toolbox, it is recommended and helpful to read these 
papers. If you have any questions, or you encounter any bugs, please feel free to either submit a bug 
report on Github (recommended) or to contact me at the email address: dme65@cornell.edu. Keep 
an eye on the Github repository for updates and changes to both the toolbox and the documentation.

\section{Licensing} Please refer to LICENSE.txt

\section{Surrogate Model Algorithms}
Surrogates models (or response surfaces) are used to approximate an underlying function that has 
been evaluated for a set of points. During the optimization phase information from the surrogate model 
is used in order to guide the search for improved solutions, which has the advantage of not needing as
many function evaluations to find a good solution. Most surrogate model algorithms consist of the same 
steps as shown in the algorithm below. The general framework for a Surrogate Optimization algorithm 
can be seen in Algorithm \ref{alg:sync_algo}

\begin{algorithm}[!h]
%\SetAlgoNoLine
\LinesNumbered
\SetKw{KwGoTo}{go to}

\KwIn{Optimization problem, Experimental design, Adaptive sampling method, 
	Surrogate model, Stopping criterion, Restart criterion}
	\KwOut{Best solution and its corresponding function value}
	Generate an initial experimental design\;{\label{restart}}
	Evaluate the points in the experimental design\;
	Build a Surrogate model from the data\;
	\Repeat{Stopping criterion met}{
		\If{Restart criterion met}{
			Reset the Surrogate model and the Sample point strategy\;
			\KwGoTo \ref{restart}\;
		}
		Use the adaptive sampling method to generate new point(s) to evaluate\;
		Evaluate the point(s) generated using all computational resources\;
		Update the Surrogate model\;
	}
\caption{Synchronous Surrogate Optimization Algorithm}
\label{alg:sync_algo}
\end{algorithm} 

\noindent Typically used stopping criteria are a maximum number of allowed function evaluations 
(used in this toolbox), a maximum allowed CPU time, or a maximum number of failed iterative 
improvement trials.

\section{Installation}
Before starting you will need Python 2.7.x and pip. You must also have numpy and scipy 
installed and \textbf{we recommend installing Anaconda for Python 2.7} \newline 
(https://www.continuum.io/downloads). \newline \ \newline Next you need to decide what 
optional packages you want:

\begin{itemize}
\item If you want to use MARS you need to install the py-earth toolbox 
\newline (http://github.com/jcrudy/py-earth). It is possible to install py-earth with pip:
\begin{python}
pip install https://github.com/jcrudy/py-earth/archive/master.zip
\end{python}
\item If you want to use the GUI you need to install PySide. This can be done with pip:
\begin{python}
pip install PySide
\end{python}
\end{itemize}

\noindent There are currently two ways to install pySOT.
\begin{enumerate}
\item \textbf{(Recommended)} The easiest way to install pySOT is through pip in which case 
the following command should suffice:
\begin{python}
pip install pySOT
\end{python}
\item 
\begin{enumerate}
\item Clone the repository: 
\begin{python}
git clone https://github.com/dme65/pySOT
\end{python} 
or alternatively download the repository directly:
\begin{enumerate}
\item Go to https://github.com/dme65/pySOT
\item Download the repository, extract the zip folder and change the name to pySOT
\end{enumerate}
\item Navigate to the repository using:
\begin{python}
cd pySOT
\end{python} 
\item Install pySOT (you may need to use sudo for UNIX):
\begin{python}
pip install inspyred
python setup.py install
\end{python} 
\item Several test problems are available at ./pySOT/test
\end{enumerate}
\end{enumerate}

\section{Sphinx documentation}
The necessary files to build the Sphinx documentation are provided in the docs subdirectory. 
We use the napoleon extension so you need to make sure you have this package. This can 
be done through pip
\begin{python}
pip install sphinxcontrib-napoleon
\end{python}
To build the documentation run the command:
\begin{python}
make html
\end{python}

\section{Options}
These are the the components and the supported options:
\subsection{Experimental design} 
\label{expdes}
The experimental design generates the initial points to be evaluated. A well-chosen 
experimental design is critical in order to fit a Surrogate model that captures the behavior 
of the underlying objective function. Any implementation must have the following method
\begin{itemize}
\item generate\_\,points(): Returns an experimental design of size $npts \times d$ where
 npts is the number of points in the initial design, which was specified when the object 
 was created 
\end{itemize}
The following experimental designs are supported:
\begin{itemize}

\item \textbf{LatinHypercube}. Arguments:
\begin{itemize}
\item \textbf{dim:} Number of dimensions
\item \textbf{npts:} Number of points to generate ($2( \text{dim}+1)$ is recommended)
\end{itemize} 
\ \newline Example: 
\begin{python}
from pySOT import LatinHypercube
exp_des = LatinHypercube(dim=3, npts=10)
\end{python}
creates a Latin hypercube design with 10 points in 3 dimensions

\item \textbf{SymmetricLatinHypercube} Arguments:
\begin{itemize}
\item \textbf{dim:} Number of dimensions
\item \textbf{npts:} Number of points to generate ($2 \text{dim}+1$ is recommended)
\end{itemize}
\ \newline Example: 
\begin{python}
from pySOT import SymmetricLatinHypercube
exp_des = SymmetricLatinHypercube(dim=3, npts=10)
\end{python}
creates a symmetric Latin hypercube design with 10 points in 3 dimensions

\item \textbf{TwoFactorial}. The corners of the unit hypercube. Arguments:
\begin{itemize}
\item \textbf{dim:} Number of dimensions
\end{itemize} 
\ \newline Example: 
\begin{python}
from pySOT import TwoFactorial
exp_des = TwoFactorial(dim=3)
\end{python}
creates a symmetric Latin hypercube design with $2^3$ points in 3 dimensions

\item \textbf{BoxBehnken}. Box-Behnken design with one center point. Arguments:
\begin{itemize}
\item \textbf{dim:} Number of dimensions
\end{itemize} 
\end{itemize}
\ \newline Example: 
\begin{python}
from pySOT import BoxBehnken
exp_des = BoxBehnken(dim=3)
\end{python}
creates a symmetric BoxBehnken design with $13$ points in 3 dimensions.

\subsection{Surrogate model} 
\label{surrogate}
The surrogate model approximates the underlying objective function given all of the 
points that have been evaluated. Any implementation of a surrogate model must 
have the following attributes and methods
\begin{itemize}
\item[] {Attributes}: 
\begin{itemize}
\item nump: Number of data points (integer)
\item maxp: Maximum number of data points (integer)
\end{itemize}
\item[] {Required methods}:
\begin{itemize}
\item reset(): Resets the surrogate model
\item get\_\,x(): Returns a numpy array of size $nump \times d$ of the data points
\item get\_\,fx(): Returns a numpy array of length $nump$ with the function values
\item add\_\,point(x,\,f): Adds a point $x$ with value $f$ to the surrogate model
\item eval(x): Evaluates the surrogate model at one point $x$
\item evals(x): Evaluates the surrogate model at multiple points
\end{itemize}
\item[] {Optional methods}:
\begin{itemize}
\item deriv(x): Returns a numpy array with the gradient at one point $x$
\end{itemize}
\end{itemize}
The following surrogate models are supported:
\begin{itemize}
\item \textbf{RBFInterpolant}. A radial basis function interpolant. Arguments:
\begin{itemize}
\item \textbf{surftype}: Kernel function. The options are 
\begin{itemize}
\item \textbf{LinearRBFSurface:} Linear RBF (comes with a constant tail)
\item \textbf{CubicRBFSurface:} Cubic RBF (comes with a linear tail)
\item \textbf{TPSSurface:} Thin-Plate RBF (comes with a linear tail)
\end{itemize}
\item \textbf{maxp:} Initial maximum number of points (can grow). Default is 100.
\end{itemize}
\ \newline Example:
\begin{python}
from pySOT import RBFInterpolant, CubicRBFSurface
fhat = RBFInterpolant(surftype=CubicRBFSurface, maxp=500)
\end{python}
creates a cubic RBF with a linear tail with a capacity for 500 points.  \newline \ \newline
\textbf{Note:} The RBF surfaces automatically applies damping to the RBF system 
in order to keep the system well-conditioned. 
\item \textbf{KrigingInterpolant:} A Kriging interpolant. Arguments:
\begin{itemize}
\item \textbf{maxp:} Maximum number of points (can grow). Default is 100
\end{itemize}
\ \newline Example:
\begin{python}
from pySOT import KrigingInterpolant
fhat = KrigingInterpolant(maxp=500)
\end{python}
creates a Kriging interpolant with a capacity of 500 points.
\item \textbf{MARSInterpolant:} Generate a Multivariate Adaptive 
Regression Splines (MARS) model. Arguments:
\begin{itemize}
\item \textbf{maxp:} Maximum number of points (can grow). Default is 100
\end{itemize}
\ \newline Example: 
\begin{python}
from pySOT import MARSInterpolant
fhat = MARSInterpolant(maxp=500)
\end{python}
creates a MARS interpolant with a capacity of 500 points.
\item \textbf{EnsembleSurrogate:} We also provide the option of using multiple surrogates 
for the same problem. Suppose we have $M$ surrogate models, then the ensemble 
surrogate takes the form
\begin{equation*}
s(x) = \sum_{j=1}^M w_j s_j(x)
\end{equation*}
where $w_j$ are non-negative weights that sum to 1. Hence the value of the ensemble 
surrogate is the weighted prediction of the $M$ surrogate models. We use leave-one-out 
for each surrogate model to predict the function value at the removed point and then 
compute several statistics such as correlation with the true function values, RMSE, etc. 
Based on these statistics we use Dempster-Shafer Theory to compute the pignistic 
probability for each model, and take this probability as the weight. Surrogate models 
that does a good job predicting the removed points will generally be given a large 
weight. The arguments are:

\begin{itemize}
	\item \textbf{model\_list:} A list of surrogate model objects to be used.
	\item \textbf{maxp:} Maximum number of points (can grow). Default is 100
\end{itemize}
\ \newline Example: 

\begin{python}
from pySOT import RBFInterpolant, CubicRBFSurface, LinearRBFSurface, \ 
		  TPSSurface, EnsembleSurrogate

models = [
	RBFInterpolant(surftype=CubicRBFSurface, maxp=500),
	RBFInterpolant(surftype=LinearRBFSurface, maxp=500),
	RBFInterpolant(surftype=TPSSurface, maxp=500)
]

response_surface = EnsembleSurrogate(model_list=models, maxp=500)
\end{python}
creates an ensemble surrogate with three surrogate models, namely a 
Cubic RBF Interpolant, a Linear RBF Interpolant, and a TPS RBF Interpolant.
\end{itemize}
\textbf{Note:} The user is responsible for resetting the response surface after 
each experiment and this is done by calling the reset() method.

\subsection{Capped RBF model} Functions with very large function values can 
cause the fitted surface to oscillate wildly. In the case of the RBFInterpolant we 
therefore provide a capped version that transforms the function values. The 
default is to replace all function values larger than the median of the function 
values by the median, but it is possible to provide an other transformation. Arguments: \\

\begin{itemize}
	\item \textbf{model}: Surrogate model.
	\item \textbf{transformation:} Function value transformation
\end{itemize}

\ \newline Example: 

\begin{python}
from pySOT import RSCapped, RBFInterpolant, CubicRBFSurface

# Set inf and nan to the largest value observed so far	          
def transform(fvalues):
	ind = np.isfinite(fvalues)
	fvalues[np.logical_not(ind)] = np.max(fvalues[ind])
	return fvalues
	
fhat = RSCapped(RBFInterpolant(model=CubicRBFSurface, maxp=500), \ 
		transformation=transform)
\end{python}

creates a cubic RBF with a linear tail with a capacity for 500 points with capping that 
transforms inf and nan to the largest finite function value found so far.

\subsection{Scaled RBF model} If the domain is large the linear system of equations 
for the RBF coefficients gets very ill-conditioned. One way of dealing with this problem 
is to rescale to domain to the unit hypercube, but the sampling methods may want to 
know exactly what the domain looks like. For this purpose we provide a wrapper for 
the surrogate model that internally rescales the domain to the unit hypercube, which 
keeps the interpolant stable while not affecting the other objects.  Arguments: \\

\begin{itemize}
	\item \textbf{model}: Surrogate model.
	\item \textbf{data:} Optimization problem object
\end{itemize}

\ \newline Example: 

\begin{python}
from pySOT import RSUnitBox, RBFInterpolant, CubicRBFSurface

# data is the optimization problem
fhat = RSUnitBox(RBFInterpolant(model=CubicRBFSurface, maxp=500), \ 
		data=data)
\end{python}
creates a cubic RBF with a linear tail with a capacity for 500 points with capping that 
transforms inf and nan to the largest finite function value found so far.


\subsection{Objective function} 
\label{objfun}
The objective function is its own object and must have certain attributes and methods 
in order to work with the framework. We start by giving an example of a mixed-integer 
optimization problem with constraints. The following attributes must always be 
specified in the objective function class:
\begin{itemize}
\item \textbf{xlow:} Lower bounds for the variables.
\item \textbf{xup:} Upper bounds for the variables.
\item \textbf{dim:} Number of dimensions
\item \textbf{integer:} Specifies the integer variables. If no variables have 
integer constraints, set to [\,]
\item \textbf{continuous:} Specifies the continuous variables. If no variables 
are continuous, set to [\,]
\end{itemize}
\ \newline The following methods must also exist.
\begin{itemize}
\item \textbf{objfunction:} Takes one input in the form of an numpy.ndarray with 
shape (1, dim), which corresponds to one point in dim dimensions. Returns the 
value (a scalar) of the objective function at this point.
\item \textbf{eval\_ineq\_constraints:}  Only necessary if there are non-constraints. 
All constraints must be inequality constraints and the must be written in the form 
$g_i(x) \leq 0$. The function takes one input in the form of an numpy.ndarray of 
shape (n, dim), which corresponds to $n$ points in dim dimensions. Returns an 
numpy.ndarray of size $n \times M$ where $M$ is the number of inequality constraints.
\end{itemize}
\ \newline \noindent What follows is an example of an objective function in 5 dimensions 
with 3 integer and 2 continuous variables. There are also 3 inequality constraints that are 
not bound constraints which means that we need to implement the 
eval\_ineq\_constraints method.
\begin{python}
import numpy as np

class LinearMI:
    def __init__(self):
        self.xlow = np.zeros(5)
        self.xup = np.array([10, 10, 10, 1, 1])
        self.dim = 5
        self.min = -1
        self.integer = np.arange(0, 3)
        self.continuous = np.arange(3, 5)

    def eval_ineq_constraints(self, x):
        vec = np.zeros((x.shape[0], 3))
        vec[:, 0] = x[:, 0] + x[:, 2] - 1.6
        vec[:, 1] = 1.333 * x[:, 1] + x[:, 3] - 3
        vec[:, 2] = - x[:, 2] - x[:, 3] + x[:, 4]
        return vec

    def objfunction(self, x):
        if len(x) != self.dim:
            raise ValueError('Dimension mismatch')
        return - x[0] + 3 * x[1] + 1.5 * x[2] + 2 * x[3] - 0.5 * x[4]
\end{python}
\textbf{Note:} The method \textit{validate} which is available in pySOT is helpful 
in order to test that the objective function is compatible with the framework. \newline

\subsection{Generation of next point to evaluate} 
\label{search}
We provide several different methods for selecting the next point to evaluate. All 
methods in this version are based in generating candidate points by perturbing the 
best solution found so far or in some cases just choose a random point. We also 
provide the option of using many different strategies in the same experiment and 
how to cycle between the different strategies. Each implementation of this object 
is required to have the following attributes and methods
\begin{itemize}
\item[] {Required attributes}:
\begin{itemize}
\item \textit{proposed\_points}: Numpy array of the points that have been proposed 
for evaluation
\end{itemize}
\item[] {Required methods}:
\begin{itemize}
\item \textit{init(start\_\,sample,\,fhat,\,budget)}: This initializes the sampling strategy 
by providing the points that were evaluated in the experimental design phase, the 
response surface, and also provides  the evaluation budget.
\item \textit{remove\_\,point(x)}: Removes point $x$ from list of proposed\_points if 
the evaluation crashed or was never carried out by the strategy. Returns True if the 
point was removed and False if the removal failed.
\item \textit{make\_\,points(npts, xbest, sigma, subset=None, proj\_\,fun=None)}: 
This is the method that proposes \textit{npts} new evaluations to the strategy. It 
needs to know the number of points to propose, the best data point evaluated so 
far, the preferred sample radius of the strategy (w.r.t the unit box), the coordinates 
that the strategy wants to perturb, and a way to project points onto the feasible region. 
\end{itemize}
\end{itemize}
We now list the different options and describe shortly how they work.
\begin{itemize}
\item \textbf{CandidateSRBF:} Generate perturbations around the best solution 
found so far
\item \textbf{CandidateSRBF\_INT:} Uses CandidateSRBF but only perturbs the 
integer variables
\item \textbf{CandidateSRBF\_CONT:} Uses CandidateSRBF but only perturbs 
the continuous variables
\item \textbf{CandidateDYCORS} Uses a DYCORS strategy which perturbs 
each coordinate with some iteration dependent probability. This probability is 
a monotonically decreasing function with the number of iteration.
\item \textbf{CandidateDYCORS\_CONT:} Uses CandidateDYCORS but only 
perturbs the continuous variables
\item \textbf{CandidateDYCORS\_INT:} Uses CandidateDYCORS but only 
perturbs the integer variables
\item \textbf{CandidateDDS}: Uses the DDS strategy where only a few candidate 
points are generated and the one with the best surrogate prediction is picked 
for evaluation
\item \textbf{CandidateDDS\_CONT:} Uses CandidateDDS but only perturbs 
the continuous variables
\item \textbf{CandidateDDS\_INT:} Uses CandidateDDS but only perturbs the 
integer variables
\item \textbf{CandidateUniform:} Chooses a new point uniformly from the 
box-constrained domain
\item \textbf{CandidateUniform\_CONT:} Given the best solution found so far 
the continuous variables are chosen uniformly from the box-constrained domain
\item \textbf{CandidateUniform\_INT:} Given the best solution found so far the 
integer variables are chosen uniformly from the box-constrained domain
\end{itemize}
The CandidateDYCORS algorithm is the bread-and-butter algorithm for any 
problems with more than 5 dimensions whilst CandidateSRBF is recommended 
for problems with only a few dimensions. It is sometimes efficient in mixed-integer 
problems to perturb the integer and continuous variables separately and we 
therefore provide such method for each of these algorithms. Finally, uniformly 
choosing a new point has the advantage of creating diversity to avoid getting 
stuck in a local minima. Each method needs an objective function object as 
described in the previous section (the input name is data) and how many 
perturbations should be generated around the best solution found so far 
(the input name is numcand). Around 100 points per dimension, but no more 
than 5000, is recommended. Next is an example on how to generate a multi-start 
strategy that uses CandidateDYCORS, CandidateDYCORS\_CONT, 
CandidateDYCORS\_INT, and CandidateUniform and that cycles evenly between 
the methods i.e., the first point is generated using CandidateDYCORS, the 
second using CandidateDYCORS\_CONT and so on.
\begin{python}
from pySOT import LinearMI, MultiSampling, CandidateDYCORS, \
			  CandidateDYCORS_CONT, CandidateDYCORS_INT, \
			  CandidateUniform

data = LinearMI()  # Optimization problem
sampling_methods = [CandidateDYCORS(data=data, numcand=100*data.dim),
                     CandidateDYCORS_CONT(data=data, numcand=100*data.dim),
                     CandidateDYCORS_INT(data=data, numcand=100*data.dim),
                     CandidateUniform(data=data, numcand=100*data.dim)]
cycle = [0, 1, 2, 3]
sampling_methods = MultiSampling(sampling_methods, cycle)
\end{python}

\section{POAP}
pySOT uses POAP, which an event-driven framework for building and combining a
synchronous optimization strategies. There are two main components in POAP, 
namely controllers and strategies. The controller is  capable of asking workers to 
run function evaluations and the strategy decides where to evaluate next. POAP 
works with external black-box objective functions and handles potential crashes 
in the objective function evaluation. There is also a logfile from which all function 
evaluations can be accessed after the run finished. In its simplest form, an 
optimization code with POAP that evaluates a function predetermined set of 
points using NUM\_WORKERS threads may look the following way:

\begin{python}
from poap.strategy import FixedSampleStrategy
from poap.strategy import CheckWorkStrategy
from poap.controller import ThreadController
from poap.controller import BasicWorkerThread

# samples = list of sample points ...

controller = ThreadController()
sampler = FixedSampleStrategy(samples)
controller.strategy = CheckWorkerStrategy(controller, sampler)

for i in range(NUM_WORKERS):
    t = BasicWorkerThread(controller, objective)
    controller.launch_worker(t)

result = controller.run()
print 'Best result: {0} at {1}'.format(result.value, result.params)
\end{python}

\subsection{Controller} pySOT needs only the ThreadController, where we create 
a team of workers (threads) that carry our objective function evaluations. If the 
objective function is an external program we use workers of the class 
ProcessWorkerThread, whilst if the objective function isn't external we can just 
use the BasicWorkerThread class.

\subsection{Strategies} pySOT provides two strategies:
\begin{itemize}
\item \textbf{SyncStrategyNoConstraints:} This strategy is to be used in case 
there are only bound constraints and no additional constraints. The arguments 
to this strategy are:
\begin{itemize}
\item \textbf{worker\_id:} An idea that the controller can use to distinguish between 
multiple simultaneously running optimization problems.
\item \textbf{data:} Objective function object, as described in Section \ref{objfun}
\item \textbf{response\_surface:} Response surface object, as described in 
Section \ref{surrogate}
\item \textbf{maxeval:} Maximum number of function evaluations
\item \textbf{nsamples:} Maximum number of simultaneous function evaluations 
(can be set to the number of workers/threads)
\item \textbf{exp\_design:} Experimental design to do the initial evaluations, as 
described in Section \ref{expdes}. Default is a Latin Hypercube with 2dim+1 points
\item \textbf{search\_procedure} Method to propose new evaluations, as described in 
Section \ref{search}. Default is Candidate DyCORS with 100dim candidate points.
\item \textbf{extra:} Additional point to be added to the experimental design. If a good 
solution is known, you can use this argument to make sure this point is evaluated early. 
\end{itemize}
\item \textbf{SyncStrategyPenalty:} If there are additional non-bound constraints we 
provide a penalty based strategy. This strategy assumes that it makes sense to evaluate 
the objective function outside the feasible region. The strategy also assumes that there 
is a method eval\_ineq\_constraints that works exactly as described in Section \ref{objfun}. 
The strategy takes the same argument as SyncStrategyNoConstraints plus one addition 
argument which is the penalty to be used in the penalty method. Given a penalty $\mu$ 
set by the user we try to solve the box-constrained optimization problem 
\begin{align*}
\underset{x}{\operatorname{minimize}} 
\qquad &\widetilde{f}(x)=f(x)+\mu \sum_{i=1}^M \max(0,g_i(x))^2 \\
\operatorname{subject\;to:} \qquad &-\infty<\ell_i \leq x_i \leq u_i<\infty, \quad i = 1,\ldots,n \\
\end{align*}
where $x \in \Rb^n$ and there are $M$ inequality constraints of the form $g_i(x) \leq 0,$ for 
$i=1,\ldots,M$. If you want the resulting solution to be feasible, just set $\mu$ to a very large 
value. This will force the algorithms to work there way towards a feasible solution. Candidate 
points are generated based on the solution with the smallest value of $\widetilde{f}$. In order 
to rank function value prediction by the response surface we set all infeasible solutions to have 
the same prediction as the worst feasible candidate point. The reason for this is that large 
penalties make it impossible for the weighted distance criteria to distinguish between feasible 
points. This modified approach will make the algorithm prefer feasible candidate points over 
infeasible candidate points as long as the function value is weighted higher than the minimum 
distance.
\item \textbf{SyncStrategyProjection:} If there is an easy way to project infeasible points onto 
the feasible region we also provide a strategy that uses this projection operator in order to 
only evaluate feasible points. The user is responsible for making sure that the projection 
operator returns a feasible point. Candidate points are all projected onto the feasible region 
before the merit function is evaluated.
\end{itemize}

\section{Guidelines for selecting parameters and components}
\begin{center}
\begin{tabular}{ c c | c }
  	\hline			
	Dimensions & Problem type & Search Strategies \\
	\hline  
   	$\leq 10$ & Continuous & CandidateSRBF \\
   	$> 10$ & Continuous & CandidateDYCORS\\
   	\hline
   	$\leq 10$ & Integer & CandidateSRBF\_INT \\
   	$> 10$ & Integer & CandidateDYCORS\_INT \\
   	\hline
   	$\leq 10$ & Mixed &  \multirow{1}{*}{[CandidateSRBF,} \\ & & 
   	CandidateSRBF\_INT,  \\ & & CandidateSRBF\_CONT] \\
   	$> 10$ & Mixed &  \multirow{1}{*}{[CandidateDYCORS,} \\ & & 
   	CandidateDYCORS\_INT, \\ & & CandidateDYCORS\_CONT] \\
   	\hline
\end{tabular}
\end{center}

\ \newline

\begin{center}
\begin{tabular}{ c | c }
  \hline			
Non-bound constraints & Optimization Strategy \\
  \hline  
  No & SyncStrategyNoConstraints \\
  Yes & SyncStrategyPenalty or SyncStrategyProjection \\
  \hline
\end{tabular}
\end{center}

\ \newline

\begin{center}
\begin{tabular}{ c | c }
  	\hline			
	Evaluation budget & Experimental design \\
  	\hline  
  	$< 10 \text{dim}$ & Latin Hypercube with $\text{dim}+1$ points \\
  	$\geq 10 \text{dim}$ & Symmetric Latin Hypercube with $2\text{dim}+1$ points \\
  	\hline
\end{tabular}
\end{center}

\ \newline

\begin{itemize}
\item \textbf{Response surface:} By default we recommend the CubicRBFSurface 
(with capping if necessary). We recommend not using Kriging since it is very slow.
\item \textbf{Number of threads:}  Setting both the number of simultaneous evaluations 
and the number of threads to the number of available cores.
\end{itemize}

\section{Graphical user interface}
pySOT comes with a graphical user interface (GUI) built in PySide. In order to use the 
GUI you need to have PySide installed together with all other dependencies of pySOT. 
Initializing the GUI is as easy as typing from the terminal:
\begin{python}
python
from pySOT import GUI
GUI()
\end{python} 
or more compactly:
\begin{python}
python -c 'from pySOT import GUI; GUI()'
\end{python}
The objective function has to be implemented in a separate file and this file must satisfy 
the requirements mentioned above for an objective function. In addition, the separate 
python implementation is only allowed to contain one class and this class has to have 
the same name as the file name (excluding .py). As an example, this is an implementation 
of the Ackley function in a separate file with file name Ackley.py:

\begin{python}
import numpy as np

class Ackley:
    def __init__(self, dim=10):
        self.xlow = -15 * np.ones(dim)
        self.xup = 20 * np.ones(dim)
        self.dim = dim
        self.info = str(dim)+"-dimensional Ackley function \n" +\
                             "Global optimum: f(0,0,...,0) = 0"
        self.integer = []
        self.continuous = np.arange(0, dim)

    def objfunction(self, x):
        if len(x) != self.dim:
            raise ValueError('Dimension mismatch')
        n = float(len(x))
        return -20.0 * np.exp(-0.2*np.sqrt(sum(x**2)/n)) - \
        		      np.exp(sum(np.cos(2.0*np.pi*x))/n)
\end{python}
Note that both the file name and the class names are the same.
\FloatBarrier
The four figures in Figure \ref{fig:gui} show what the GUI looks like and how the optimization 
results are reported to the user. If the user want a search strategy that uses DYCORS, 
DYCORS, SRBF, DYCORS, DYCORS, SRBF, ... this can be achieved in the GUI by 
adding CandidateDYCORS twice and CandidateSRBF once.
\begin{figure}
        \centering
        \begin{subfigure}{0.49\textwidth}\centering%no!\hfill
                    \includegraphics[width=\linewidth]{./Pics/GUI1}
                \caption{After initializing the GUI}
  \label{fig:First_figure}
       \end{subfigure}
    \hfill
        \begin{subfigure}{0.49\textwidth}\centering
                    \includegraphics[width=\linewidth]{./Pics/GUI2}
                \caption{After importing an Optimization problem}
  \label{fig:Second_figure}
       \end{subfigure}%
    \hfill
        \begin{subfigure}{0.49\textwidth}\centering
                    \includegraphics[width=\linewidth]{./Pics/GUI3}
                \caption{After choosing optimization components}
  \label{fig:Third_figure}
       \end{subfigure}%
    \hfill
        \begin{subfigure}{0.49\textwidth}\centering
                    \includegraphics[width=\linewidth]{./Pics/GUI4}
                \caption{After running the optimizer}
  \label{fig:First_figure}
       \end{subfigure}%
       \caption{Illustration of the GUI}
 \label{fig:gui}
 \end{figure}
 
\section{Examples}
The GitHub repository contains several examples for how to use pySOT.

\section{Hierarchy of POAP + pySOT}
\begin{figure}[!ht] 
	\centering
	\resizebox{0.9\textwidth}{!}{ 
		\includegraphics{RBFabstraction}} 
	\caption{Overview of pySOT and POAP} 
\end{figure}
\FloatBarrier

\section{Future changes}
\begin{itemize}
\item Add support for continuing from a logfile
\item Add an asynchronous strategy
\item Add more methods for handling constraints, especially a barrier method
\item Support for Python 3.x
\end{itemize}

\bibliographystyle{unsrtnat}
\bibliography{references}


\end{document}
