\documentclass[]{article}

% definitions
\def\D{\partial}

\def\solidline{{$\textcolor{blue}{\overline{\hskip 0.5cm}}\ $}}
\def\dashedline{{$\textcolor{red}{\overline{\hskip 0.1cm}\ \overline{\hskip 0.1cm}\ \overline{\hskip 0.1cm}\ }$}}
\def\dashdottedline{$\overline{\hskip 0.1cm}\ \overline{\hskip 0.03cm}\ \overline{\hskip 0.1cm}\ $}
\def\dottedline{$\overline{\hskip 0.04cm}\ \overline{\hskip 0.04cm}\ \overline{\hskip 0.04cm}\ $}

\usepackage{amsmath,amssymb,amsthm}
\newtheorem{thm}{Theorem}
\usepackage{lettrine}
\usepackage{amsfonts}
\usepackage[pdftex]{graphicx}
\usepackage[latin1]{inputenc}
\usepackage{epstopdf}
\usepackage{hyperref}
\usepackage{placeins}
\usepackage{bigints}
\usepackage{gensymb}
\usepackage{float}
\usepackage{color}
\usepackage{graphicx}
\usepackage{graphics}
\usepackage[hmarginratio=1:1,top=25mm,columnsep=20pt]{geometry}
\usepackage[font=it]{caption}
\usepackage{paralist}
\usepackage{multicol}
\usepackage{lettrine}
\usepackage[nodayofweek]{datetime}
\usepackage{centernot}
\usepackage{ mathrsfs }
\usepackage{textcomp}
\usepackage{color}
%\usepackage{listings}
\usepackage{setspace}
\usepackage{xcolor}
\usepackage{subcaption}
\usepackage{color}
\usepackage[procnames]{listings}
\usepackage{setspace}
\usepackage{palatino}
\usepackage{multirow}
\usepackage{algorithm2e}
\usepackage[authoryear]{natbib}

\renewcommand{\lstlistlistingname}{Code Listings}
\renewcommand{\lstlistingname}{Code Listing}
\definecolor{gray}{gray}{0.5}
\definecolor{green}{rgb}{0,0.5,0}
\definecolor{lightgreen}{rgb}{0,0.7,0}
\definecolor{purple}{rgb}{0.5,0,0.5}
\definecolor{darkred}{rgb}{0.5,0,0}
\lstnewenvironment{python}[1][]{
\lstset{
language=python,
basicstyle=\ttfamily\small\setstretch{1},
stringstyle=\color{green},
showstringspaces=false,
alsoletter={1234567890},
otherkeywords={\ , \}, \{},
keywordstyle=\color{blue},
emph={access,and,as,break,class,continue,def,del,elif,else,%
except,exec,finally,for,from,global,if,import,in,is,%
lambda,not,or,pass,print,raise,return,try,while,assert},
emphstyle=\color{orange}\bfseries,
emph={[2]self},
emphstyle=[2]\color{gray},
emph={[4]ArithmeticError,AssertionError,AttributeError,BaseException,%
DeprecationWarning,EOFError,Ellipsis,EnvironmentError,Exception,%
False,FloatingPointError,FutureWarning,GeneratorExit,IOError,%
ImportError,ImportWarning,IndentationError,IndexError,KeyError,%
KeyboardInterrupt,LookupError,MemoryError,NameError,None,%
NotImplemented,NotImplementedError,OSError,OverflowError,%
PendingDeprecationWarning,ReferenceError,RuntimeError,RuntimeWarning,%
StandardError,StopIteration,SyntaxError,SyntaxWarning,SystemError,%
SystemExit,TabError,True,TypeError,UnboundLocalError,UnicodeDecodeError,%
UnicodeEncodeError,UnicodeError,UnicodeTranslateError,UnicodeWarning,%
UserWarning,ValueError,Warning,ZeroDivisionError,abs,all,any,apply,%
basestring,bool,buffer,callable,chr,classmethod,cmp,coerce,compile,%
complex,copyright,credits,delattr,dict,dir,divmod,enumerate,eval,%
execfile,exit,file,filter,float,frozenset,getattr,globals,hasattr,%
hash,help,hex,id,input,int,intern,isinstance,issubclass,iter,len,%
license,list,locals,long,map,max,min,object,oct,open,ord,pow,property,%
quit,range,raw_input,reduce,reload,repr,reversed,round,set,setattr,%
slice,sorted,staticmethod,str,sum,super,tuple,type,unichr,unicode,%
vars,xrange,zip},
emphstyle=[4]\color{purple}\bfseries,
upquote=true,
morecomment=[s][\color{lightgreen}]{"""}{"""},
commentstyle=\color{red}\slshape,
literate={>>>}{\textbf{\textcolor{darkred}{>{>}>}}}3%
         {...}{{\textcolor{gray}{...}}}3,
procnamekeys={def,class},
procnamestyle=\color{blue}\textbf,
framexleftmargin=1mm, framextopmargin=1mm, frame=shadowbox,
rulesepcolor=\color{blue},#1
}}{}


%%%%% Titling
\usepackage[compact]{titlesec}
\newcommand{\overbar}[1]{\mkern 1.5mu\overline{\mkern-1.5mu#1\mkern-1.5mu}\mkern 1.5mu}

\newenvironment{corollary}[1]{\par\noindent\underline{Corollary:}\space#1}{}
\newenvironment{lemma}[1]{\par\noindent\underline{Lemma:}\space#1}{}
\newenvironment{theorem}[1]{\par\noindent\underline{Theorem:}\space#1}{}
\newenvironment{claim}[1]{\par\noindent\underline{Claim:}\space#1}{}
\newenvironment{proposition}[1]{\par\noindent\underline{Proposition:}\space#1}{}
\newenvironment{claimproof}[1]{\par\noindent\underline{Proof:}\space#1}{\hfill\ensuremath{\square}}

\include{pythonlisting}

% ------
% Header/footer
\usepackage{fancyhdr}
	\pagestyle{fancy}
	\fancyhead{}
	\fancyfoot{}
	\fancyhead[C]{Surrogate Optimization Toolbox in Python (pySOT) - 0.1.18 $\bullet$ Tutorial $\bullet$ David Eriksson, David Bindel, Christine Shoemaker $\bullet$ \today}
	\fancyfoot[RO,LE]{\thepage}

%%% ---- MATLAB CODE ---- %%%

\definecolor{Blue}{rgb}{0.1,0.1,0.3}
\hypersetup{colorlinks=true,linkcolor=Blue,citecolor=Blue,urlcolor=Blue}

\definecolor{listinggray}{gray}{0.9} 
\definecolor{lbcolor}{rgb}{0.9,0.9,0.9}
\definecolor{dorange}{rgb}{1.000000,0.549020,0.000000}
\definecolor{lblue}{rgb}{0.529412,0.807843,0.980392}
\lstset{escapeinside={<@}{@>}}


\definecolor{orange}{rgb}{1,0.5,0}
\DeclareMathOperator{\sgn}{sgn}
\DeclareMathOperator{\tr}{tr}
\DeclareMathOperator{\argmax}{argmax}
\DeclareMathOperator{\X}{\mathcal{X}}
\DeclareMathOperator{\Y}{\mathcal{Y}}
\DeclareMathOperator{\Z}{\mathbb{Z}}
\DeclareMathOperator{\Fx}{\mathcal{F}_{\X}}
\DeclareMathOperator{\Fy}{\mathcal{F}_{\Y}}
\DeclareMathOperator{\Fz}{\mathcal{F}_{\Z}}
\DeclareMathOperator{\Nc}{\mathcal{N}}
\DeclareMathOperator{\Rc}{\mathcal{R}}
\DeclareMathOperator{\B}{\mathcal{B}}
\DeclareMathOperator{\C}{\mathbb{C}}
\DeclareMathOperator{\Rb}{\mathbb{R}}
\DeclareMathOperator{\Sb}{\mathcal{S}}
\DeclareMathOperator{\N}{\mathbb{N}}
\DeclareMathOperator{\Q}{\mathbb{Q}}
\DeclareMathOperator{\limn}{\lim\limits_{n \to \infty}}
\DeclareMathOperator{\limm}{\lim\limits_{m \to \infty}}
\DeclareMathOperator{\R}{Re}
\DeclareMathOperator{\I}{Im}
\DeclareMathOperator{\spann}{span}
\newcommand*{\QED}{\hfill\ensuremath{\square}}%

%%%% Title
\title{\vspace{-15mm}%
	\fontsize{18pt}{10pt}\selectfont
	\textbf{Surrogate Optimization Toolbox (pySOT) - 0.1.18 \\ Tutorial}
	}	
\author{%
	\Large\textsc{David Eriksson} \\[2mm]
	\Large\textsc{David Bindel} \\[2mm]
	\Large\textsc{Christine Shoemaker} \\[2mm]
		\normalsize	Cornell University \\
	\normalsize Center for Applied Mathematics \\
	\normalsize	dme65@cornell.edu \\ 
	}
\date{\today}

\begin{document}
\fontsize{12}{14}\rm

\maketitle
\thispagestyle{fancy}
\tableofcontents
\newpage

\section{Change history:}
\begin{itemize}

\item (\textbf{0.1.19})
\begin{itemize}
\item 	Too much regularization was added to the RBF surface when the volume of the domain was large. This has been fixed.
\end{itemize}

\item (\textbf{0.1.18})
\begin{itemize}
\item Significant restructuring of the code base
\item make\_points now takes an argument that specifies the number of new points to be generated
\item Added Box-Behnken and 2-factorial to the experimental designs
\item Simplified the penalty method strategy by moving evals and derivs into a surrogate wrapper
\end{itemize}

\item (\textbf{0.1.17})
\begin{itemize}
\item Added the possibility to input the penalty for the penalty method in the GUI
\item Added the possibility of making a performance plot using matplotlib that adds new points dynamically as evaluations are finished
\item Switched from subprocess to subprocess32
\end{itemize}

\item (\textbf{0.1.16})
\begin{itemize}
\item Added a projection strategy
\end{itemize}

\item (\textbf{0.1.15})
\begin{itemize}
\item Added an example test\_subprocess\_files that shows how to use pySOT in case the objective function needs to read the input from a textfile
\end{itemize}

\item (\textbf{0.1.14}) 
\begin{itemize}
\item Updated the Tutorial to reflect the changes for the last few months
\item Simplified the object creation from strings in the GUI by importing directly from the namespace.
\end{itemize}

\item (\textbf{0.1.13}) 
\begin{itemize}
\item Allowed to still import the rest of pySOT when PySide is not found. In this case, the GUI will be unavailable.
\end{itemize}

\item (\textbf{0.1.12}) 
\begin{itemize}
\item The capping can now take in a general transformation that is used to transform the function values. Default is median capping.
\item The Genetic Algorithm now defaults to initialize the population using a symmetric latin hypercube design
\item DYCORS uses the remaining evaluation budget to change the probabilities after a restart instead of using the total budget 
\end{itemize}

\item (\textbf{0.1.11}) 
\begin{itemize}
\item Fixed a bug in the capped response surface
\item pySOT now internally works on the unit hypercube
\item The distance can be passed to the RBF after being computed when generating candidate points so it is not computed twice anymore
\item Fixed some bugs in the candidate functions
\item GA and Multi-Search gradient perturb the best solution in the case when the best solution is a previously evaluated point
\item Added an additional test for the multi-search strategy
\end{itemize}

\item (\textbf{0.1.10}) 
\begin{itemize}
\item README.md not uploaded to pypi which caused the pip install to fail
\end{itemize}

\item (\textbf{0.1.9}) 
\begin{itemize}
\item Fixed a bug in the merit function and several bugs in the DYCORS strategy
\item Added a DDS candidate based strategy for searching on the surrogate
\end{itemize}

\item (\textbf{0.1.8}) 
\begin{itemize}
\item Multi Start Gradient method that uses the L-BFGS-B algorithm to search on the surroagate
\end{itemize}

\item (\textbf{0.1.7}) 
\begin{itemize}
\item Fixed some parameters (and bugs) to improve the DYCORS results. Using DYCORS together with the genetic algorithm is recommended.
\item Added polynomial regression (not yet in the GUI)
\item Changed so that candidate points are generated using truncated normal distribution to avoid projections onto the boundary
\item Removed some accidental scikit dependencies in the ensemble surrogate
\end{itemize}

\item (\textbf{0.1.6}) 
\begin{itemize}
\item GUI inactivates all buttons but the stop button while running
\item Bug fixes
\end{itemize}

\item (\textbf{0.1.5}) 
\begin{itemize}
\item GUI now has support for multiple search strategies and ensemble surrogates
\item Reallocation bug in the ensemble surrogates fixed
\item Genetic algorithm added to search on the surrogate
\end{itemize}

\item (\textbf{0.1.4}) 
\begin{itemize}
\item GUI now has improved error handling 
\item Strategies informs the user if they get constraints when not expecting constraints (and the other way) before the run starts
\end{itemize}

\item (\textbf{0.1.3}) 
\begin{itemize}
\item Experimental (but not documented) GUI added. You need PySide to use it.
\item Changes in testproblems.py to allow external objective functions that implement ProcessWorkerThread
\item Added GUI test examples in documentation (Ackley.py, Keane.py, SphereExt.py)
\end{itemize}

\item (\textbf{0.1.2})
\begin{itemize}
\item 	Changed to using the logging module for all the logging in order to conform to the changes in POAP 0.1.9
\item The quiet and stream arguments in the strategies were removed and the tests updated accordingly
\item Turned sleeping of in the sub process test, to avoid platform dependency issues
\end{itemize}

\item (\textbf{0.1.1})
\begin{itemize}
\item surrogate\_optimizer.py was removed, so the user now has to create his own controller
\item constraint\_method.py is gone, and the constraint handling is handled in specific strategies instead
\item 	There are now two strategies, SyncStrategyNoConstraints and SyncStrategyPenalty
\item The search strategies now take a method for providing surrogate predictions rather than keeping a copy of the response surface
\item It is now possible for the user to provide additional points to be added to the initial design, in case a 'good starting point' is known.
\item Ensemble surrogates have been added to the toolbox
\item 	The strategies takes an additional option 'quiet' so that all of the printing can be avoided if the user wants
\item There is also an option 'stream' in case the printing should be redirected somewhere else, for example to a text file. Default is printing to stdout.
\item 	Several examples added to pySOT.test
\end{itemize}
\item (\textbf{0.1.0})
\begin{itemize}
\item 	Initial release
\end{itemize}
\end{itemize}

\section{Introduction}
This is a tutorial (user guide) for the Surrogate Optimization Toolbox (pySOT) for global deterministic optimization problems. The main purpose of the toolbox is for optimization of computationally expensive black-box objective functions with continuous and/or integer variables. We support inequality constraints of any form through a penalty method approach, but cannot yet efficiently handle equality constraints. All variables are assumed to have bound constraints in some form where none of the bounds are infinity. The tighter the bounds, the more efficient are the algorithms since it reduces the search region and increases the quality of the constructed surrogate. The longer the objective functions take to evaluate, the more efficient are these algorithms. For this reason, this toolbox may not be very efficient for problems with computationally cheap function evaluations. Surrogate models are intended to be used when function evaluations take from several minutes to several hours or more. The toolbox is based on the following published papers that should be cited when the toolbox is used for own research purposes:
\begin{enumerate}
\item Stochastic RBF: \cite{regis2007stochastic}
\item Parallel Stochastic RBF: \cite{regis2009parallel}
\item DYCORS: \cite{regis2013combining}
\item Surrogate optimization using mixture surrogates: \cite{muller2011mixture}
\item Surrogate optimization for Mixed-Integer problems: \cite{muller2013so}
\item Surrogate optimization for Integer problems: \cite{muller2014so}
\end{enumerate}
For easier understanding of the algorithms in this toolbox, it is recommended and helpful to read these papers. If you have any questions, or you encounter any bugs, please feel free to either submit a bug report on Github (recommended) or to contact me at the email address: dme65@cornell.edu. Keep an eye on the Github repository for updates and changes to both the toolbox and the documentation.

\section{Licensing} Please refer to LICENSE.txt

\section{Surrogate Model Algorithms}
Surrogates models (or response surfaces) are used to approximate an underlying function that has been evaluated for a set of points. During the optimization phase information from the surrogate model is used in order to guide the search for improved solutions, which has the advantage of not needing as many function evaluations to find a good solution. Most surrogate model algorithms consist of the same steps as shown in the algorithm below.
The general framework for a Surrogate Optimization algorithm can be seen in Algorithm \ref{alg:sync_algo}

\begin{algorithm}[!h]
%\SetAlgoNoLine
\LinesNumbered
\SetKw{KwGoTo}{go to}

\KwIn{Initial experimental design, Sample point strategy, Surrogate model, Stopping criterion, Restart criterion}
\KwOut{Best solution and its corresponding function value}

Generate an initial experimental design\;{\label{restart}}
Evaluate the points in the experimental design\;
Build a Surrogate model from the data\;
\Repeat{Stopping criterion met}{
	\If{Restart criterion met}{
		Reset the Surrogate model and the Sample point strategy\;
		\KwGoTo \ref{restart}\;
	}
	Use the Sample point strategy to generate new point(s) to evaluate\;
	Evaluate the point(s) generated using all computational resources\;
	Update the Surrogate model\;
}
\caption{Synchronous Surrogate Optimization Algorithm}
\label{alg:sync_algo}
\end{algorithm} 

\noindent Typically used stopping criteria are a maximum number of allowed function evaluations (used in this toolbox), a maximum allowed CPU time, or a maximum number of failed iterative improvement trials.

\section{Installation}
Before starting you will need Python 2.7.x and pip. You must also have numpy and scipy installed and we recommend installing the Anaconda package \newline (https://www.continuum.io/downloads). \newline \ \newline Next you need to decide what optional packages you want:
\begin{itemize}
\item If you want to use MARS you need to install the py-earth toolbox \newline (http://github.com/jcrudy/py-earth)  
\item If you want to use the GUI you need to install PySide (pip install PySide)
\end{itemize}

\noindent There are currently two ways to install pySOT.
\begin{enumerate}
\item \textbf{(Recommended)} The easiest way to install pySOT is through pip in which case the following command should suffice (you may need sudo for UNIX):
\begin{python}
pip install pySOT
\end{python}
\item 
\begin{enumerate}
\item Clone the repository: 
\begin{python}
git clone https://github.com/dme65/pySOT
\end{python} 
or alternatively download the repository directly:
\begin{enumerate}
\item Go to https://github.com/dme65/pySOT
\item Download the repository, extract the zip folder and change the name to pySOT
\end{enumerate}
\item Navigate to the repository using:
\begin{python}
cd pySOT
\end{python} 
\item Install pySOT (you may need to use sudo for UNIX):
\begin{python}
pip install inspyred
python setup.py install
\end{python} 
\item Several test problems are available at ./pySOT/test
\end{enumerate}
\end{enumerate}

\section{Sphinx documentation}
The necessary files to build the Sphinx documentation are provided in the docs subdirectory. We use the napoleon extension so you need to make sure you have this package. This can be done through pip
\begin{python}
pip install sphinxcontrib-napoleon
\end{python}
To build the documentation run the command:
\begin{python}
make html
\end{python}

\section{Options}
These are the the components and the supported options:
\subsection{Experimental design} 
\label{expdes}
The experimental design generates the initial points to be evaluated. A well-chosen experimental design is critical in order to fit a Surrogate model that captures the behavior of the underlying objective function. Any implementation must have the following method
\begin{itemize}
\item generate\_\,points(): Returns an experimental design of size $npts \times d$ where npts is the number of points in the initial design, which was specified when the object was created 
\end{itemize}
The following experimental designs are supported:
\begin{itemize}

\item \textbf{LatinHypercube}. Arguments:
\begin{itemize}
\item \textbf{dim:} Number of dimensions
\item \textbf{npts:} Number of points to generate ($2( \text{dim}+1)$ is recommended)
\end{itemize} 
\ \newline Example: 
\begin{python}
from pySOT import LatinHypercube
exp_des = LatinHypercube(dim=3, npts=10)
\end{python}
creates a Latin hypercube design with 10 points in 3 dimensions

\item \textbf{SymmetricLatinHypercube} Arguments:
\begin{itemize}
\item \textbf{dim:} Number of dimensions
\item \textbf{npts:} Number of points to generate ($2 \text{dim}+1$ is recommended)
\end{itemize}
\ \newline Example: 
\begin{python}
from pySOT import SymmetricLatinHypercube
exp_des = SymmetricLatinHypercube(dim=3, npts=10)
\end{python}
creates a symmetric Latin hypercube design with 10 points in 3 dimensions

\item \textbf{TwoFactorial}. The corners of the unit hypercube. Arguments:
\begin{itemize}
\item \textbf{dim:} Number of dimensions
\end{itemize} 
\ \newline Example: 
\begin{python}
from pySOT import TwoFactorial
exp_des = TwoFactorial(dim=3)
\end{python}
creates a symmetric Latin hypercube design with $2^3$ points in 3 dimensions

\item \textbf{BoxBehnken}. Box-Behnken design with one center point. Arguments:
\begin{itemize}
\item \textbf{dim:} Number of dimensions
\end{itemize} 
\end{itemize}
\ \newline Example: 
\begin{python}
from pySOT import BoxBehnken
exp_des = BoxBehnken(dim=3)
\end{python}
creates a symmetric BoxBehnken design with $13$ points in 3 dimensions.

\subsection{Surrogate model} 
\label{surrogate}
The surrogate model approximates the underlying objective function given all of the points that have been evaluated. Any implementation of
a surrogate model must have the following attributes and methods
\begin{itemize}
\item[] {Attributes}: 
\begin{itemize}
\item nump: Number of data points (integer)
\item maxp: Maximum number of data points (integer)
\end{itemize}
\item[] {Required methods}:
\begin{itemize}
\item reset(): Resets the surrogate model
\item get\_\,x(): Returns a numpy array of size $nump \times d$ of the data points
\item get\_\,fx(): Returns a numpy array of length $nump$ with the function values
\item add\_\,point(x,\,f): Adds a point $x$ with value $f$ to the surrogate model
\item eval(x): Evaluates the surrogate model at one point $x$
\item evals(x): Evaluates the surrogate model at multiple points
\end{itemize}
\item[] {Optional methods}:
\begin{itemize}
\item deriv(x): Returns a numpy array with the gradient at one point $x$
\end{itemize}
\end{itemize}
The following surrogate models are supported:
\begin{itemize}
\item \textbf{RBFInterpolant}. A radial basis function interpolant. Arguments:
\begin{itemize}
\item \textbf{surftype}: Kernel function. The options are 
\begin{itemize}
\item \textbf{LinearRBFSurface:} Linear RBF (comes with a constant tail)
\item \textbf{CubicRBFSurface:} Cubic RBF (comes with a linear tail)
\item \textbf{TPSSurface:} Thin-Plate RBF (comes with a linear tail)
\end{itemize}
\item \textbf{maxp:} Initial maximum number of points (can grow). Default is 100.
\end{itemize}
\ \newline Example:
\begin{python}
from pySOT import RBFInterpolant, CubicRBFSurface
fhat = RBFInterpolant(surftype=CubicRBFSurface, maxp=500)
\end{python}
creates a cubic RBF with a linear tail with a capacity for 500 points.  \newline \ \newline
\textbf{Note:} The RBF surfaces automatically applies damping to the RBF system in order to keep the system well-conditioned. 
\item \textbf{KrigingInterpolant:} A Kriging interpolant. Arguments:
\begin{itemize}
\item \textbf{maxp:} Maximum number of points (can grow). Default is 100
\end{itemize}
\ \newline Example:
\begin{python}
from pySOT import KrigingInterpolant
fhat = KrigingInterpolant(maxp=500)
\end{python}
creates a Kriging interpolant with a capacity of 500 points.
\item \textbf{MARSInterpolant:} Generate a Multivariate Adaptive Regression Splines (MARS) model. Arguments:
\begin{itemize}
\item \textbf{maxp:} Maximum number of points (can grow). Default is 100
\end{itemize}
\ \newline Example: 
\begin{python}
from pySOT import MARSInterpolant
fhat = MARSInterpolant(maxp=500)
\end{python}
creates a MARS interpolant with a capacity of 500 points.
\item \textbf{EnsembleSurrogate:} We also provide the option of using multiple surrogates for the same problem. Suppose we have $M$ surrogate models, then the ensemble surrogate takes the form
\begin{equation*}
s(x) = \sum_{j=1}^M w_j s_j(x)
\end{equation*}
where $w_j$ are non-negative weights that sum to 1. Hence the value of the ensemble surrogate is the weighted prediction of the $M$ surrogate models. We use leave-one-out for each surrogate model to predict the function value at the removed point and then compute several statistics such as correlation with the true function values, RMSE, etc.  Based on these statistics we use Dempster-Shafer Theory to compute the pignistic probability for each model, and take this probability as the weight. Surrogate models that does a good job predicting the removed points will generally be given a large weight. The arguments are:
\begin{itemize}
\item \textbf{model\_list:} A list of surrogate model objects to be used.
\item \textbf{maxp:} Maximum number of points (can grow). Default is 100
\end{itemize}
\ \newline Example: 
\begin{python}
from pySOT import RBFInterpolant, CubicRBFSurface, LinearRBFSurface, \ 
		  TPSSurface, EnsembleSurrogate

models = [
	RBFInterpolant(surftype=CubicRBFSurface, maxp=500),
	RBFInterpolant(surftype=LinearRBFSurface, maxp=500),
	RBFInterpolant(surftype=TPSSurface, maxp=500)
]

response_surface = EnsembleSurrogate(model_list=models, maxp=500)
\end{python}
creates an ensemble surrogate with three surrogate models, namely a Cubic RBF Interpolant, a Linear RBF Interpolant, and a TPS RBF Interpolant.
\end{itemize}
\textbf{Note:} The user is responsible for resetting the response surface after each experiment and this is done by calling the reset() method.

\subsection{Capped RBF model} Functions with very large function values can cause the fitted surface to oscillate wildly. In the case of the RBFInterpolant we therefore provide a capped version that transforms the function values. The default is to replace all function values larger than the median of the function values by the median, but it is possible to provide an other transformation. Arguments: \\
\begin{itemize}
\item \textbf{model}: Surrogate model.
\item \textbf{transformation:} Function value transformation
\end{itemize}
\ \newline Example: 
\begin{python}
from pySOT import RSCapped, RBFInterpolant, CubicRBFSurface

# Set inf and nan to the largest value observed so far	          
def transform(fvalues):
	ind = np.isfinite(fvalues)
	fvalues[np.logical_not(ind)] = np.max(fvalues[ind])
	return fvalues
	
fhat = RSCapped(RBFInterpolant(model=CubicRBFSurface, maxp=500), \ 
		transformation=transform)
\end{python}
creates a cubic RBF with a linear tail with a capacity for 500 points with capping that transforms inf and nan to the largest finite function value found so far.

\subsection{Scaled RBF model} If the domain is large the linear system of equations for the RBF coefficients gets very ill-conditioned. One way of dealing with this problem is to rescale to domain to the unit hypercube, but the sampling methods may want to know exactly what the domain looks like. For this purpose we provide a wrapper for the surrogate model that internally rescales the domain to the unit hypercube, which keeps the interpolant stable while not affecting the other objects.  Arguments: \\
\begin{itemize}
\item \textbf{model}: Surrogate model.
\item \textbf{data:} Optimization problem object
\end{itemize}
\ \newline Example: 
\begin{python}
from pySOT import RSUnitBox, RBFInterpolant, CubicRBFSurface

# data is the optimization problem
fhat = RSUnitBox(RBFInterpolant(model=CubicRBFSurface, maxp=500), \ 
		data=data)
\end{python}
creates a cubic RBF with a linear tail with a capacity for 500 points with capping that transforms inf and nan to the largest finite function value found so far.


\subsection{Objective function} 
\label{objfun}
The objective function is its own object and must have certain attributes and methods in order to work with the framework. We start by giving an example of a mixed-integer optimization problem with constraints. The following attributes must always be specified in the objective function class:
\begin{itemize}
\item \textbf{xlow:} Lower bounds for the variables.
\item \textbf{xup:} Upper bounds for the variables.
\item \textbf{dim:} Number of dimensions
\item \textbf{integer:} Specifies the integer variables. If no variables have integer constraints, set to [\,]
\item \textbf{continuous:} Specifies the continuous variables. If no variables are continuous, set to [\,]
\end{itemize}
\ \newline The following methods must also exist.
\begin{itemize}
\item \textbf{objfunction:} Takes one input in the form of an numpy.ndarray with shape (1, dim), which corresponds to one point in dim dimensions. Returns the value (a scalar) of the objective function at this point.
\item \textbf{eval\_ineq\_constraints:}  Only necessary if there are non-constraints. All constraints must be inequality constraints and the must be written in the form $g_i(x) \leq 0$. The function takes one input in the form of an numpy.ndarray of shape (n, dim), which corresponds to $n$ points in dim dimensions. Returns an numpy.ndarray of size $n \times M$ where $M$ is the number of inequality constraints.
\end{itemize}
\ \newline \noindent What follows is an example of an objective function in 5 dimensions with 3 integer and 2 continuous variables. There are also 3 inequality constraints that are not bound constraints which means that we need to implement the eval\_ineq\_constraints method.
\begin{python}
import numpy as np

class LinearMI:
    def __init__(self):
        self.xlow = np.zeros(5)
        self.xup = np.array([10, 10, 10, 1, 1])
        self.dim = 5
        self.min = -1
        self.integer = np.arange(0, 3)
        self.continuous = np.arange(3, 5)

    def eval_ineq_constraints(self, x):
        vec = np.zeros((x.shape[0], 3))
        vec[:, 0] = x[:, 0] + x[:, 2] - 1.6
        vec[:, 1] = 1.333 * x[:, 1] + x[:, 3] - 3
        vec[:, 2] = - x[:, 2] - x[:, 3] + x[:, 4]
        return vec

    def objfunction(self, x):
        if len(x) != self.dim:
            raise ValueError('Dimension mismatch')
        return - x[0] + 3 * x[1] + 1.5 * x[2] + 2 * x[3] - 0.5 * x[4]
\end{python}
\textbf{Note:} The method \textit{validate} which is available in pySOT is helpful in order to test that the objective function is compatible with the framework. \newline

\subsection{Generation of next point to evaluate} 
\label{search}
We provide several different methods for selecting the next point to evaluate. All methods in this version are based in generating candidate points by perturbing the best solution found so far or in some cases just choose a random point. We also provide the option of using many different strategies in the same experiment and how to cycle between the different strategies. Each implementation of this object is required to have the following attributes and methods
\begin{itemize}
\item[] {Required attributes}:
\begin{itemize}
\item \textit{proposed\_points}: Numpy array of the points that have been proposed for evaluation
\end{itemize}
\item[] {Required methods}:
\begin{itemize}
\item \textit{init(start\_\,sample,\,fhat,\,budget)}: This initializes the sampling strategy by providing the points that were evaluated in the experimental design phase, the response surface, and also provides  the evaluation budget.
\item \textit{remove\_\,point(x)}: Removes point $x$ from list of proposed\_points if the evaluation crashed or was never carried out by the strategy. Returns True if the point was removed and False if the removal failed.
\item \textit{make\_\,points(npts, xbest, sigma, subset=None, proj\_\,fun=None)}: This is the method that proposes \textit{npts} new evaluations to the strategy. It needs to know the number of points to propose, the best data point evaluated so far, the preferred sample radius of the strategy (w.r.t the unit box), the coordinates that the strategy wants to perturb, and a way to project points onto the feasible region. 
\end{itemize}
\end{itemize}
We now list the different options and describe shortly how they work.
\begin{itemize}
\item \textbf{CandidateSRBF:} Generate perturbations around the best solution found so far
\item \textbf{CandidateSRBF\_INT:} Uses CandidateSRBF but only perturbs the integer variables
\item \textbf{CandidateSRBF\_CONT:} Uses CandidateSRBF but only perturbs the continuous variables
\item \textbf{CandidateDYCORS} Uses a DYCORS strategy which perturbs each coordinate with some iteration dependent probability. This probability is a monotonically decreasing function with the number of iteration.
\item \textbf{CandidateDYCORS\_CONT:} Uses CandidateDYCORS but only perturbs the continuous variables
\item \textbf{CandidateDYCORS\_INT:} Uses CandidateDYCORS but only perturbs the integer variables
\item \textbf{CandidateDDS}: Uses the DDS strategy where only a few candidate points are generated and the one with the best surrogate prediction is picked for evaluation
\item \textbf{CandidateDDS\_CONT:} Uses CandidateDDS but only perturbs the continuous variables
\item \textbf{CandidateDDS\_INT:} Uses CandidateDDS but only perturbs the integer variables
\item \textbf{CandidateUniform:} Chooses a new point uniformly from the box-constrained domain
\item \textbf{CandidateUniform\_CONT:} Given the best solution found so far the continuous variables are chosen uniformly from the box-constrained domain
\item \textbf{CandidateUniform\_INT:} Given the best solution found so far the integer variables are chosen uniformly from the box-constrained domain
\end{itemize}
The CandidateDYCORS algorithm is the bread-and-butter algorithm for any problems with more than 5 dimensions whilst CandidateSRBF is recommended for problems with only a few dimensions. It is sometimes efficient in mixed-integer problems to perturb the integer and continuous variables separately and we therefore provide such method for each of these algorithms. Finally, uniformly choosing a new point has the advantage of creating diversity to avoid getting stuck in a local minima. Each method needs an objective function object as described in the previous section (the input name is data) and how many perturbations should be generated around the best solution found so far (the input name is numcand). Around 100 points per dimension, but no more than 5000, is recommended. Next is an example on how to generate a multi-start strategy that uses CandidateDYCORS, CandidateDYCORS\_CONT, CandidateDYCORS\_INT, and CandidateUniform and that cycles evenly between the methods i.e., the first point is generated using CandidateDYCORS, the second using CandidateDYCORS\_CONT and so on.
\begin{python}
from pySOT import LinearMI, MultiSearchStrategy, CandidateDYCORS, \
			  CandidateDYCORS_CONT, CandidateDYCORS_INT, \
			  CandidateUniform

data = LinearMI()  # Optimization problem
sampling_methods = [CandidateDYCORS(data=data, numcand=100*data.dim),
                     CandidateDYCORS_CONT(data=data, numcand=100*data.dim),
                     CandidateDYCORS_INT(data=data, numcand=100*data.dim),
                     CandidateUniform(data=data, numcand=100*data.dim)]
cycle = [0, 1, 2, 3]
sampling_methods = MultiSampling(sampling_methods, cycle)
\end{python}

\section{POAP}
pySOT uses POAP, which an event-driven framework for building and combining asynchronous optimization strategies. There are two main components in POAP, namely controllers and strategies. The controller is  capable of asking workers to run function evaluations and the strategy decides where to evaluate next. POAP works with external black-box objective functions and handles potential crashes in the objective function evaluation. There is also a logfile from which all function evaluations can be accessed after the run finished. In its simplest form, an optimization code with POAP that evaluates a function predetermined set of points using NUM\_WORKERS threads may look the following way:

\begin{python}
from poap.strategy import FixedSampleStrategy
from poap.strategy import CheckWorkStrategy
from poap.controller import ThreadController
from poap.controller import BasicWorkerThread

# samples = list of sample points ...

controller = ThreadController()
sampler = FixedSampleStrategy(samples)
controller.strategy = CheckWorkerStrategy(controller, sampler)

for i in range(NUM_WORKERS):
    t = BasicWorkerThread(controller, objective)
    controller.launch_worker(t)

result = controller.run()
print 'Best result: {0} at {1}'.format(result.value, result.params)
\end{python}

\subsection{Controller} pySOT needs only the ThreadController, where we create a team of workers (threads) that carry our objective function evaluations. If the objective function is an external program we use workers of the class ProcessWorkerThread, whilst if the objective function isn't external we can just use the BasicWorkerThread class.

\subsection{Strategies} pySOT provides two strategies:
\begin{itemize}
\item \textbf{SyncStrategyNoConstraints:} This strategy is to be used in case there are only bound constraints and no additional constraints. The arguments to this strategy are:
\begin{itemize}
\item \textbf{worker\_id:} An idea that the controller can use to distinguish between multiple simultaneously running optimization problems.
\item \textbf{data:} Objective function object, as described in Section \ref{objfun}
\item \textbf{response\_surface:} Response surface object, as described in Section \ref{surrogate}
\item \textbf{maxeval:} Maximum number of function evaluations
\item \textbf{nsamples:} Maximum number of simultaneous function evaluations (can be set to the number of workers/threads)
\item \textbf{exp\_design:} Experimental design to do the initial evaluations, as described in Section \ref{expdes}. Default is a Latin Hypercube with 2dim+1 points
\item \textbf{search\_procedure} Method to propose new evaluations, as described in Section \ref{search}. Default is Candidate DyCORS with 100dim candidate points.
\item \textbf{extra:} Additional point to be added to the experimental design. If a good solution is known, you can use this argument to make sure this point is evaluated early. 
\end{itemize}
\item \textbf{SyncStrategyPenalty:} If there are additional non-bound constraints we provide a penalty based strategy. This strategy assumes that it makes sense to evaluate the objective function outside the feasible region. The strategy also assumes that there is a method eval\_ineq\_constraints that works exactly as described in Section \ref{objfun}. The strategy takes the same argument as SyncStrategyNoConstraints plus one addition argument which is the penalty to be used in the penalty method. Given a penalty $\mu$ set by the user we try to solve the box-constrained optimization problem 
\begin{align*}
\underset{x}{\operatorname{minimize}} \qquad &\widetilde{f}(x)=f(x)+\mu \sum_{i=1}^M \max(0,g_i(x))^2 \\
\operatorname{subject\;to:} \qquad &-\infty<\ell_i \leq x_i \leq u_i<\infty, \quad i = 1,\ldots,n \\
\end{align*}
where $x \in \Rb^n$ and there are $M$ inequality constraints of the form $g_i(x) \leq 0,$ for $i=1,\ldots,M$. If you want the resulting solution to be feasible, just set $\mu$ to a very large value. This will force the algorithms to work there way towards a feasible solution. Candidate points are generated based on the solution with the smallest value of $\widetilde{f}$. In order to rank function value prediction by the response surface we set all infeasible solutions to have the same prediction as the worst feasible candidate point. The reason for this is that large penalties make it impossible for the weighted distance criteria to distinguish between feasible points. This modified approach will make the algorithm prefer feasible candidate points over infeasible candidate points as long as the function value is weighted higher than the minimum distance.
\item \textbf{SyncStrategyProjection:} If there is an easy way to project infeasible points onto the feasible region we also provide a strategy that uses this projection operator in order to only evaluate feasible points. The user is responsible for making sure that the projection operator returns a feasible point. Candidate points are all projected onto the feasible region before the merit function is evaluated.
\end{itemize}

\section{Guidelines for selecting parameters and components}
\begin{center}
\begin{tabular}{ c c | c }
  \hline			
Dimensions & Problem type & Search Strategies \\
  \hline  
  $\leq 10$ & Continuous & CandidateSRBF \\
   $> 10$ & Continuous & CandidateDYCORS\\
   \hline
     $\leq 10$ & Integer & CandidateSRBF\_INT \\
   $> 10$ & Integer & CandidateDYCORS\_INT \\
   \hline
   $\leq 10$ & Mixed &  \multirow{1}{*}{[CandidateSRBF,} \\ & & CandidateSRBF\_INT,  \\ & & CandidateSRBF\_CONT] \\
      $> 10$ & Mixed &  \multirow{1}{*}{[CandidateDYCORS,} \\ & & CandidateDYCORS\_INT, \\ & & CandidateDYCORS\_CONT] \\
      \hline
\end{tabular}
\end{center}

\ \newline

\begin{center}
\begin{tabular}{ c | c }
  \hline			
Non-bound constraints & Optimization Strategy \\
  \hline  
  No & SyncStrategyNoConstraints \\
  Yes & SyncStrategyPenalty or SyncStrategyProjection \\
  \hline
\end{tabular}
\end{center}

\ \newline

\begin{center}
\begin{tabular}{ c | c }
  \hline			
Evaluation budget & Experimental design \\
  \hline  
  $< 10 \text{dim}$ & Latin Hypercube with $\text{dim}+1$ points \\
  $\geq 10 \text{dim}$ & Symmetric Latin Hypercube with $2\text{dim}+1$ points \\
  \hline
\end{tabular}
\end{center}

\ \newline

\begin{itemize}
\item \textbf{Response surface:} By default we recommend the CubicRBFSurface (with capping if necessary). We recommend not using Kriging since it is very slow.
\item \textbf{Number of threads:}  Setting both the number of simultaneous evaluations and the number of threads to the number of available cores.
\end{itemize}

\section{Graphical user interface}
pySOT comes with a graphical user interface (GUI) built in PySide. In order to use the GUI you need to have PySide installed together with all other dependencies of pySOT. Initializing the GUI is as easy as typing from the terminal:
\begin{python}
python
from pySOT import GUI
GUI()
\end{python} 
or more compactly:
\begin{python}
python -c 'from pySOT import GUI; GUI()'
\end{python}
The objective function has to be implemented in a separate file and this file must satisfy the requirements mentioned above for an objective function. In addition, the separate python implementation is only allowed to contain one class and this class has to have the same name as the file name (excluding .py). As an example, this is an implementation of the Ackley function in a separate file with file name Ackley.py:

\begin{python}
import numpy as np

class Ackley:
    def __init__(self, dim=10):
        self.xlow = -15 * np.ones(dim)
        self.xup = 20 * np.ones(dim)
        self.dim = dim
        self.info = str(dim)+"-dimensional Ackley function \n" +\
                             "Global optimum: f(0,0,...,0) = 0"
        self.integer = []
        self.continuous = np.arange(0, dim)

    def objfunction(self, x):
        if len(x) != self.dim:
            raise ValueError('Dimension mismatch')
        n = float(len(x))
        return -20.0 * np.exp(-0.2*np.sqrt(sum(x**2)/n)) - \
        		      np.exp(sum(np.cos(2.0*np.pi*x))/n)
\end{python}
Note that both the file name and the class names are the same.
\FloatBarrier
The four figures in Figure \ref{fig:gui} show what the GUI looks like and how the optimization results are reported to the user. If the user want a search strategy that uses DYCORS, DYCORS, SRBF, DYCORS, DYCORS, SRBF, ... this can be achieved in the GUI by adding CandidateDYCORS twice and CandidateSRBF once.
\begin{figure}
        \centering
        \begin{subfigure}{0.49\textwidth}\centering%no!\hfill
                    \includegraphics[width=\linewidth]{./Pics/GUI1}
                \caption{After initializing the GUI}
  \label{fig:First_figure}
       \end{subfigure}
    \hfill
        \begin{subfigure}{0.49\textwidth}\centering
                    \includegraphics[width=\linewidth]{./Pics/GUI2}
                \caption{After importing an Optimization problem}
  \label{fig:Second_figure}
       \end{subfigure}%
    \hfill
        \begin{subfigure}{0.49\textwidth}\centering
                    \includegraphics[width=\linewidth]{./Pics/GUI3}
                \caption{After choosing optimization components}
  \label{fig:Third_figure}
       \end{subfigure}%
    \hfill
        \begin{subfigure}{0.49\textwidth}\centering
                    \includegraphics[width=\linewidth]{./Pics/GUI4}
                \caption{After running the optimizer}
  \label{fig:First_figure}
       \end{subfigure}%
       \caption{Illustration of the GUI}
 \label{fig:gui}
 \end{figure}
 
\section{Examples}
The GitHub repository contains several examples for how to use pySOT.

\section{Hierarchy of POAP + pySOT}
\begin{figure}[!ht] 
	\centering
	\resizebox{0.9\textwidth}{!}{ 
		\includegraphics{RBFabstraction}} 
	\caption{Overview of the pySOT hierarchy} 
\end{figure}
\FloatBarrier

\section{Future changes}
\begin{itemize}
\item Add support for continuing from a logfile
\item Add an asynchronous strategy
\item Add more methods for handling constraints, especially a barrier method
\item Support for Python 3.x
\end{itemize}

\bibliographystyle{unsrtnat}
\bibliography{references}


\end{document}
