\documentclass[10pt]{report}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{helvet}

\setlength{\paperheight}{3in}
\setlength{\paperwidth}{4in}
\pdfpagewidth=\paperwidth
\pdfpageheight=\paperheight

\setlength{\textwidth}{3.75in}
\setlength{\textheight}{2.5in}

\setlength{\oddsidemargin}{-1.0in}
\setlength{\evensidemargin}{-1.0in}
\setlength{\topmargin}{-1.25in}

\newcommand{\sld}[1]{\newpage{\noindent\Large \ \ \ {\bfseries\slshape #1}}\vspace*{4pt}}
\newcommand{\spc}{\hspace*{0.25in}}

\newcounter{gmlrx}
\newcounter{gmlry}
\newcommand{\gmnode}[3]{\put(#1,#2){\circle{20}}\put(#1,#2){\makebox(0,0){$#3$}}}
\newcommand{\gmplate}[5]{
\setcounter{gmlrx}{#1}\addtocounter{gmlrx}{#3}
\setcounter{gmlry}{#2}\addtocounter{gmlry}{-#4}
\put(#1,#2){\line(1,0){#3}}
\put(#1,#2){\line(0,-1){#4}}
\put(\value{gmlrx},\value{gmlry}){\line(-1,0){#3}}
\put(\value{gmlrx},\value{gmlry}){\line(0,1){#4}}
\setcounter{gmlrx}{#1}\addtocounter{gmlrx}{5}
\setcounter{gmlry}{#2}\addtocounter{gmlry}{-6}
\put(\value{gmlrx},\value{gmlry}){\makebox(0,0){$#5$}}
}

\newcommand{\mypart}[1]{\newpage\vspace*{36pt}\noindent\ \ \ \ {\huge #1}}


\begin{document}
\sf%
\vspace*{12pt}
%
\noindent
\spc{\Large\bfseries\slshape Stan:  A (Bayesian)}
\\[8pt]
\spc{\Large\bfseries\slshape Directed Graphical Model Compiler}
\\[18pt]
\spc{\large Bob Carpenter}
\\[4pt]
\spc\spc{\footnotesize with Matt Hoffman, Ben Goodrich, Daniel Lee}
\\
\spc\spc{\footnotesize  Jiqiang Guo, Michael Malecki, and Andrew Gelman}
\\[4pt]
\spc{\small\slshape Columbia University, Department of Statistics}

\vfill
\noindent
\spc{\footnotesize NYC Machine Learning Meetup: January 2012}
\hfill
\includegraphics[width=27.5pt]{img/logo.png}

\sld{Three Kinds of Scaling}
%
\begin{enumerate}
\item Data
\item Parameters
\item Model Structure
\end{enumerate}
\vspace*{12pt}
\begin{itemize}
\item Talk focuses mostly on 3
\end{itemize}

\sld{The Big Picture}
%
\begin{itemize}
\item {\slshape\bfseries Application}: Fit rich Bayesian statistical models
\end{itemize}
%
\vspace*{8pt}
\begin{itemize}
\item {\slshape Problem}: Gibbs too slow, Metropolis too problem-specific
\item {\slshape Solution}: Hamiltonian Monte Carlo
%
\vspace*{8pt}
\item {\slshape Problem}:  Interpreters too slow, won't scale
\item {\slshape Solution}: Compilation
%
\vspace*{8pt}
\item {\slshape Problem}:  Need gradients of log posterior for HMC
\item {\slshape Solution}: Reverse-mode algorithmic differentation
\end{itemize}

\sld{The Big Picture (cont.)}
%
\begin{itemize}
\item {\slshape Problem}:  Existing algo-diff slow, limited, unextensible
\item {\slshape Solution}: Our own algo-diff
%
\vspace*{8pt}
\item {\slshape Problem}:  Algo-diff requires fully templated functions
\item {\slshape Solution}: Our own density library, Eigen linear
 algebra
%
\vspace*{8pt}
\item {\slshape Problem}:  Need unconstrained parameters for HMC
\item {\slshape Solution}: Variable transforms w. Jacobian determinants
%
\end{itemize}

\sld{The Big Picture (cont.)}
%
\begin{itemize}
\item {\slshape Problem}:  Need ease of use of BUGS
\item {\slshape Solution}: Compile directed graphical model language
%
\vspace*{8pt}
\item {\slshape Problem}:  Need to tune parameters for HMC
\item {\slshape Solution}: Auto tuning, adaptation
%
\vspace*{8pt}
\item {\slshape Problem}:  Efficient up-to-proportion density calcs
\item {\slshape Solution}: Density template metaprogramming 
%
\end{itemize}

\sld{The Big Picture (conclusion)}
\begin{itemize}
\item {\slshape Problem}:  Limited error checking, recovery
\item {\slshape Solution}: Static model typing, informative exceptions
%
\vspace*{8pt}
\item {\slshape Problem}:  Poor boundary behavior
\item {\slshape Solution}: Calculate limits (e.g. $\lim_{x \rightarrow
    0} x \log x$)
%
\vspace*{8pt}
\item {\slshape Problem}:  Restrictive licensing (e.g., closed, GPL, etc.)
\item {\slshape Solution}: Open-source, BSD license
\end{itemize}

\sld{Bayesian Data Analysis}
\begin{itemize}
\item ``By {Bayesian data analysis}, we mean {practical methods}
for making {inferences} from {data} using {probability models}
for quantities we {observe} and about which we {wish to learn}.''
%
\item ``The essential characteristic of Bayesian methods is
their {\bfseries explict use of probability for quantifying uncertainty}
in inferences based on statistical analysis.''
\end{itemize}
%
\vfill\hfill{\small [Gelman et al., {\slshape Bayesian Data Analysis},
  2003]}



\sld{The Process}
%
\begin{enumerate}
\item Set up full probability model 
\vspace*{-4pt}
\begin{itemize}
\item for all observable \& unobservable quantities
\item consistent w. problem knowledge \& data collection
\end{itemize}
%
\item Condition on observed data
\vspace*{-4pt}
\begin{itemize}
\item caclulate posterior probability of unobserved quantities
  conditional on observed quantities
\end{itemize}
%
\item Evaluate 
\vspace*{-4pt}
\begin{itemize}
\item model fit 
\item implications of posterior
\end{itemize}
\end{enumerate}

\vfill\hfill {\small [{\slshape Ibid.}]}

\sld{Basic Quantities}
%
\begin{itemize}
\item Basic Quantities
\vspace*{-4pt}
\begin{itemize}
\item $y$: \ observed data
\item $\tilde{y}$: \ unknown, potentially observable quantities
\item $\theta$: \ parameters (and other unobserved quantities)
\item $x$: \ constants, predictors for conditional models
\end{itemize}
\item Random models for things that could've been otherwise
\begin{itemize}
\item Everyone: Model data $y$ as random
\item Bayesians:  Model parameters $\theta$ as random
\end{itemize}
\end{itemize}

\sld{Basic Distributions}

\begin{itemize}
\item Joint: \ $p(y,\theta)$
\item Sampling / Likelihood: \ $p(y|\theta)$
\item Prior: \ $p(\theta)$
\item Posterior: \ $p(\theta|y)$
\item Data Marginal: \ $p(y)$
\item Posterior Predictive: \ $p(\tilde{y}|y)$
\end{itemize}

\vfill
$y$ observed data, \ $\theta$ parameters, \ $\tilde{y}$ predictions

\sld{Bayes's Rule: The Big Inversion}
%
\begin{itemize}
\item Suppose the data $y$ is fixed (i.e., observed).  Then
%
\vspace*{2pt}
\begin{eqnarray*}
p(\theta|y) 
 \hspace*{8pt} = \hspace*{8pt} \frac{p(y,\theta)}{p(y)}
& = & \frac{p(y|\theta) \, p(\theta)}{p(y)}
\\[6pt]
& = & \frac{p(y|\theta) \, p(\theta)}{\int p(y,\theta) \ d\theta}
\\[6pt]
& = & \frac{p(y|\theta) \, p(\theta)}{\int p(y|\theta) \, p(\theta) \ d\theta}
\\[6pt]
& \propto & p(y|\theta) \, p(\theta) \ \ = \ \ p(y,\theta)
\end{eqnarray*}
\item Posterior proportional to likelihood times prior (i.e., joint)
\end{itemize}

\sld{Directed Graphical Models}
%
\begin{itemize}
\item Directed acyclic graph
\item Nodes are data or parameters
\item Edges represent dependencies
\item Generative model
\begin{itemize}
\item Start at top
\item Sample each node conditioned on parents
\end{itemize}
\item Determines joint probability
\end{itemize}

\sld{BUGS Declarative Model Language}
%
\begin{itemize}
\item Declarative specification of directed graphical models
\item Variables are (potentially) random quantities
\item Full set of arithmetic, functional, and matrix expressions
\item Sampling: {\tt y $\sim$ Foo(theta); }
\item Assignment: {\tt y <- bar(x); }
\item For Loops: {\tt for (n in 1:N) \{ ... \} }
\item Constants modeled if on left of sampling {\tt ~}
\begin{itemize}
\item usually modeled: outcomes
\item not usually modeled: predictors, data sizes
\end{itemize}
\end{itemize}


\sld{Normal (Sampling)}

\begin{itemize}
\item[]
\begin{verbatim}
for (n in 1:N) 
   y[n] ~ normal(0,1);

\end{verbatim}
%
\item Sampling: data $(N)$, params $(y)$
\end{itemize}

\sld{Normal (Full)}
\begin{itemize}
\item[]
\begin{verbatim}
mu ~ normal(0,10);
sigma_sq ~ inv_gamma(1,1);
for (n in 1:N)
   y[n] ~ normal(mu,sigma_sq);

\end{verbatim}
\item Estimation: data $(y,N)$, params $(\mu,\sigma)$
\item Sampling:  data $(\mu,\sigma^2,N)$, params $(y)$
\end{itemize}

\sld{Naive Bayes}
\begin{itemize}
\item
\begin{verbatim}
pi ~ Dirichlet(alpha);
for (d in 1:D) {
   z[d] ~ Discrete(pi);
   for (n in 1:N[d])
       w[d,n] ~ Discrete(phi[z[d]]);
}
for (k i 1:K)
  phi[k] ~ Dirichlet(beta);
\end{verbatim}
\vfill
\item Estimation: data $(w,z,D,N,\alpha,\beta)$, params $(\pi,\phi)$
\item Prediction: data $(w,D,N,\pi,\phi,\alpha,\beta)$, params $(z)$
\item Clustering: data $(w,D,N,\alpha,\beta)$, params $(z,\phi,\pi)$
\end{itemize}

\sld{Supervision: Full, Semi-, and Un-}
%
\begin{itemize}
\item How variable is used
\begin{itemize}
\item Supervised: declared as data
\item Unsupervised: declared as parameter
\item Semi-supervised: partly data, partly parameter
\end{itemize}
\item Full probability model does not change
\item E.g., Semi-supervised naive Bayes
\begin{itemize}
\item partly estimation, known categories $z[n]$ supervised
\item partly clustering, unknown $z[n]$ unsupervised
\end{itemize}
\end{itemize}




\sld{Latent Dirichlet Allocation}
\begin{itemize}
\item[]
\begin{verbatim}
for (d in 1:D) {
    theta[d] ~ Dirichlet(alpha);
    for (n in 1:N[d]) {
        z[d,n] ~ Discrete(theta[d]);
        w[d,n] ~ Discrete(phi[z[d,n]]);
    }
}
for (k i 1:K)
   phi[k] ~ Dirichlet(beta);
\end{verbatim}
\item Clustering: data $(w,\alpha,\beta,D,K,N)$, params
  $(\theta,\phi,z)$
\end{itemize}
\vfill\hfill {\small (Blei et al. 2003)}

\sld{Logistic Regression}
\begin{itemize}
\item
\begin{verbatim}
for (k in 1:K)
  beta[k] ~ cauchy(0,2.5); 
for (n in 1:N)
  y[n] ~ bern(inv_logit(x[n] * beta));
\end{verbatim}
\item Estimate: data $(y,x,K,N)$, params $(\beta)$
\item Predict: data $(\beta,x,K,N)$, params $(y)$
\item Pluggable prior
\begin{itemize}
\item Cauchy, fat tails (allows concentration around mean)
\item Normal (L2), strong due to relatively thin tails
\item Laplace (L1), sparse only with point estimates
\end{itemize}
\end{itemize}

\sld{BUGS to Joint Probability}

\begin{itemize}
\item BUGS Model
\begin{verbatim}
mu ~ normal(0,10);
for (n in 1:N) 
    y[n] ~ normal(mu,1);
\end{verbatim}
%
\item Joint Probability
\begin{eqnarray*}
p(\mu,y) 
& = & \mbox{\sf Normal}(\mu|0,10)
\\[4pt]
& & {} \ \ \times \ \prod_{n = 1}^N \mbox{\sf Normal}(y_n|0,1)
\end{eqnarray*}
\end{itemize}

\sld{Monte Carlo Methods}
\begin{itemize}
\item For integrals that are impossible to solve analytically
\item But for which sampling and evaluation is tractable
\item Compute plug-in estimates of statistics based on
randomly generated variates (e.g., means, variances,
quantiles/intervals, comparisons)
\item Accuracy with $M$ (independent) samples proportional to
\[
\frac{1}{\sqrt{M}}
\]
e.g., 100 times more samples per decimal place!
\end{itemize}
\vfill\hfill
{\small (Metropolis and Ulam 1949)}


\sld{Monte Carlo Example}
\begin{itemize}
\item Posterior expectation of $\theta$:
\[
\mathbb{E}[\theta|y] = \int \theta \ p(\theta|y) \ d\theta.
\]
\item Bayesian estimate minimizing expected square error: 
\[
\hat{\theta} 
= \arg\min_{\theta'}
\mathbb{E}[(\theta - \theta')^2|y]
= \mathbb{E}[\theta|y] 
\]
\item Generate samples $\theta^{(1)}, \theta^{(2)}, \ldots,
 \theta^{(M)}$ drawn from $p(\theta|y)$
\item Monte Carlo Estimator plugs in average for expectation:
\[
\mathbb{E}[\theta|y] \approx \frac{1}{M} \sum_{m=1}^M \theta^{(m)}
\]
\end{itemize}

\sld{Monte Carlo Example II}
\begin{itemize}
\item Bayesian alternative to frequentist hypothesis testing
\item Use probability to summarize results
\item Bayesian comparison: probability $\theta_1 > \theta_2$ given
  data $y$?
\begin{eqnarray*}
\mbox{Pr}[\theta_1 > \theta_2|y] 
& = & 
\int \int 
\mathbb{I}(\theta_1 > \theta_2) \ p(\theta_1|y) \ p(\theta_2|y)  
\ d\theta_1 \ d\theta_2
\\
& \approx & 
\frac{1}{M} \sum_{m=1}^M \mathbb{I}(\theta_1^{(m)} > \theta_2^{(m)})
\end{eqnarray*}
%
\item (Bayesian hierarchical model ``adjusts'' for multiple comparisons)
\end{itemize}

\sld{Markov Chain Monte Carlo}
\begin{itemize}
\item When sampling independently from $p(\theta|y)$ impossible
\item $\theta^{(m)}$ drawn via a Markov chain $p(\theta^{(m)}|y,\theta^{(m-1)})$
\item Require MCMC marginal $p(\theta^{(m)}|y)$ equal to true
  posterior marginal
\item Leads to auto-correlation in samples
  $\theta^{(1)},\ldots, \theta^{(m)}$
\item Effective sample size $M_{\mbox{\tiny eff}}$ divides out
  auto-correlation (must be estimated)
\item Estimation accuracy proportional to $1 / \sqrt{M_{\mbox{\tiny eff}}}$
\end{itemize}

\sld{Gibbs Sampling}

\begin{itemize}
\item Samples a parameter given data and other parameters
\item Requires conditional posterior $p(\theta_n|y,\theta_{-n})$
\item Conditional posterior easy in directed graphical model
\item Requires general unidimensional sampler for non-conjugacy

\begin{itemize}
\item JAGS uses slice sampler
\item BUGS uses adaptive rejection sampler
\end{itemize}
\item Conditional sampling and general unidimensional sampler 
can both lead to slow convergence and mixing
\end{itemize}
\vfill\hfill
{\small (Geman and Geman 1984)}

\sld{Metropolis-Hastings Sampling}
\begin{itemize}
\item Proposes new point by changing all parameters randomly
\item Computes accept probability of new point based
on ratio of new to old log probability (and proposal density)
\item Only requires evaluation of $p(\theta|y)$
\item Requires good proposal mechanism to be effective
\item Acceptance requires small changes in log probability
\item But small step sizes lead to random walks and slow convergence
  and mixing
\end{itemize}
\vfill\hfill
{\small (Metropolis et al. 1953; Hastings 1970)}

\sld{Hamiltonian Monte Carlo}

\begin{itemize}
\item Converges faster and explores posterior faster
when posterior is complex
\item Function of interest is log posterior (up to proportion)
\[
\log p(\theta|y) 
\propto
\log p(y|\theta) + \log p(\theta)
\]

\item HMC exploits its gradient 
%
\begin{eqnarray*}
g & = & \nabla_{\theta} \log p(\theta|y) 
\\[8pt]
& = & \left( \frac{d}{d\theta_1} \log p(\theta|y), 
             \ldots
             \frac{d}{d\theta_K} \log p(\theta|y)
      \right)
\end{eqnarray*}
%
\end{itemize}
\vfill\hfill
{\small (Duane et al. 1987; Neal 1994)}

\sld{HMC's Physical Analogy}
%
\begin{enumerate}
\item Negative log posterior $-\log p(\theta|y)$ is potential energy
\item Start point mass at current parameter position $\theta$
\item Add random kinetic energy (momentum)
\item Simulate trajectory of the point mass over time $t$
\item Return new parameter position${}^{*}$
\end{enumerate}
\vfill
\begin{itemize}
\item[${}^{*}$] In practice, Metropolis adjust for imprecision
in trajectory simulation due to discretizing Hamiltonian dynamics
\end{itemize}

\sld{A (Simple) HMC Update}

\begin{enumerate}
\item $m \sim \mbox{\sf Norm}(0,\mbox{\rm I})$
\hspace*{24pt}
$H = \frac{m^{\top} m}{2} - \log p(\theta|y)$
\item $\theta^{\mbox{\tiny new}} = \theta$
\item repeat $L$ times:
\begin{enumerate}
\item $m = m - \frac{1}{2} \ \epsilon \ g(\theta^{\mbox{\tiny new}})$
\vspace*{2pt}
\item $\theta^{\mbox{\tiny new}} = \theta^{\mbox{\tiny new}} +
  \epsilon \ m$
\vspace*{2pt}
\item $m = m - \frac{1}{2} \ \epsilon \ g(\theta^{\mbox{\tiny new}})$
\end{enumerate}
\item  $H^{\mbox{\tiny new}} = \frac{m^{\top} m}{2} - \log
p(\theta^{\mbox{\tiny new}}|y)$
\item if \ $\mbox{Unif}(0,1) < \exp(H - H^{\mbox{\tiny new}})$, 
then $\theta^{\mbox{\tiny new}}$, else $\theta$
\end{enumerate}

\sld{HMC Example Trajectory}

\includegraphics[width=0.5\textwidth]{img/hamiltonian-path.pdf}

{\small
\begin{itemize}
\item Blue ellipse is contour of target distribution
\item Initial position at black solid circle
\item Arrows indicate a U-turn in momentum
\end{itemize}
}

\sld{No-U-Turn Sampler (NUTS)}

\begin{itemize}
\item HMC highly sensitive to tuning parameters
\begin{itemize}
\item discretization step size $\epsilon$
\item discretization number of steps $L$
\end{itemize}
\item NUTS sets $\epsilon$ during burn-in by stochastic optimization (Nesterov-style
  dual averaging)
\item NUTS chooses $L$ online per-sample using no-U-turn idea:
\begin{quote}
keep simulating as
  long as position gets further away from initial position
\end{quote}
\item Number of steps just a bit of bookkeeping on top of HMC
\end{itemize}
\vfill\hfill
{\small (Hoffman and Gelman, 2011)}

\sld{NUTS vs.\ Gibbs and Metropolis}

\includegraphics[width=0.9\textwidth]{img/nuts-vs.pdf}
\begin{itemize}
\item Two dimensions of highly correlated 250-dim distribution
\item 1M samples from Metropolis, 1M from Gibbs (thin to 1K)
\item 1K samples from NUTS, 1K independent draws
\end{itemize}

\sld{NUTS vs.\ Basic HMC}

\includegraphics[width=0.9\textwidth]{img/nuts-ess-1.pdf}
{\small
\begin{itemize}
\item 250-D normal and logistic regression models
\item Vertical axis is effective sample size per sample (bigger better)
\item Left) NUTS; \ \ Right) HMC with increasing $t = \epsilon L$
\end{itemize}
}

\sld{NUTS vs.\ Basic HMC II}

\includegraphics[width=0.9\textwidth]{img/nuts-ess-2.pdf}

{\small
\begin{itemize}
\item Hierarchical logistic regression and stochastic volatility
\item Simulation time $t$ is $\epsilon \ L$, step size ($\epsilon$)
times number of steps ($L$)
\item NUTS can beat optimally tuned HMC (latter very expensive)
\end{itemize}
}

\sld{Stan C++ Library}

\begin{itemize}
\item Beta available from Google code; 1.0 release soon
\item C++, with heavy use of templates
\item HMC and NUTS continuous samplers (Metropolis in v2)
\item Gibbs (bounded) and slice (unbounded) for discrete
\item Model (probability, gradient) extends abstract base class
\item Automatic gradient w.\ algorithmic differentiation
\item Fully templated densities, cumulative densities, transforms
\item (New) BSD licensed
\end{itemize}

\sld{Stan --- Graphical Model Compiler}

\begin{itemize}
\item Compiler for directed graphical model language ($\sim$ BUGS)
\item Generates C++ model class
\item Compile model from command line
\item Run model from command line
\begin{itemize}
\item random seeds
\item multiple chains (useful for convergence monitoring)
\item parameter initialization
\item HMC parameters and NUTS hyperparameters
\item CSV sample output
\end{itemize}
\end{itemize}

\sld{Stan Integration with R}
%
\begin{itemize}
\item Effective sample size calcs (variogram-based)
\item Convergence monitoring (split $\hat{R}$)
\item Plots of posteriors
\item Statistical summaries and comparisons
\vfill
\item Python, MATLAB to come
\end{itemize}

\sld{Extensions to BUGS Language}
\begin{itemize}
\item User-defined functions (JAGS, Stan)
\item Data Transformations (JAGS, Stan)
\item General matrix solvers (Stan)
\item Local variables (Stan)
\end{itemize}

\sld{Variable Typing}

\begin{itemize}
\item Classes of variables (Stan): 
\\
{\small 
data, transformed data, 
parameters, transformed parameters,
derived quantities, local}
\item Static variable typing (Stan):
\\ Unconstrained: {\small int, double, vector, row vector, matrix, list}
\\ Constrained: {\small (half) bounded, simplex, ordered, correlation
  matrix, covariance matrix}
\end{itemize}

\sld{Algorithmic Differentiation}

\begin{itemize}
\item Forward-mode fast for single derivative
\item Reverse-mode uses dynamic programming to evaluate
gradient in time proportional to function eval (independently of
number of dimensions)
\item Functional Behavior
\begin{itemize}
\item Write function templating out scalar variables
\item Instantiate template with algo-dif variables
\item Call function
\item Fetch gradient
\end{itemize}
\end{itemize}

\sld{Algorithmic Differentiation (cont.)}
\begin{itemize}
\item Override all built-in scalar ops (operators, lib functions)
\begin{itemize}
{\small
\item Calculate values and partial derivates w.r.t.\ all arguments
\item Object-oriented design supports user extensions}
\end{itemize}
\item Algo-dif uses templated variables to build expression tree
\item Nodes of tree represent intermediate expressions
\item Nodes topologically sorted on a stack
\item Custom arena-based memory management (thread localizable at 20\%
performance hit)
\item Propagate partial derivatives down along edges
\end{itemize}

\sld{Algorithmic Differentiation (cont.)}
%
\begin{itemize}
\item Non-negligible cost compared to well-coded derivatives
\item Space per operation: 24 bytes + 8 bytes/argument
\begin{itemize}
\item especially problematic for iterative algorithms
\end{itemize}
\item Time per operation: about 4 times slower than basic function evaluation
\begin{itemize}
\item Mostly due to partial derivative virtual function
\end{itemize}
\item Can partially evaluate some expressions and vectorize repeated
operations with shared suboperations
\end{itemize}

\sld{Variable Transforms}

\begin{itemize}
\item HMC works best with unconstrained variables
\item (Technically possible to bounce off boundaries)
\item Automatically transform variables from unconstrained to
  constrained
\item Add log of the absolute determinant of the Jacobian of the
  transform
\item Jacobian is the matrix of output variable gradients with respect
to each input variable
\end{itemize}

\sld{Example Transforms}
\begin{itemize}
\item Lower bound 0: $x \mapsto \exp(x)$
\item Constrained $(0,1)$: $x \mapsto \mbox{logit}^{-1}(x)$
\item Simplex: $x \mapsto \mbox{softmax}(x)$ (or hyperspherical +
  Weierstrss); $K-1$ degrees of freedom
\item Ordered: $(x_1,x_2) \mapsto (x_1,x_1 + \exp(x_2))$
\item Correlation Matrix: Lewandowski et al.\ C-vines transform; 
${K \choose 2}$ degrees of freedom
\item Covariance Matrix: Scale correlation matrix; $K + {K \choose 2}$
degrees of freedom
\end{itemize}

\sld{Calculating Prop-to Log Densities}

\begin{itemize}
\item Only need calculations to proportion
\item Drop additive terms that only have constants
\item Consider log of normal distribution:
\[
\log \mbox{\sf Normal}(y|\mu,\sigma)
= - \log \sqrt{2 \, \pi}
- 0.5 \log \sigma
+ \frac{(y - \mu)^2}{2 \sigma^2}
\]
\begin{itemize}{\small
\item Drop first term always if only need proportion
\item Drop second term if $\sigma$ is constant
\item Drop third term if all arguments constant}
\end{itemize}
\end{itemize}

\sld{Templates for Proportionality}

\begin{itemize}
\item Type traits to statically test fixed values
\item
\begin{verbatim}
template <typename T_out, 
          typename T_loc, 
          typename T_scale>
typename promote_args<T_out,T_loc,T_scale>::type
normal_log(T_out y, T_loc mu, T_scale sigma) { 
    ...
    if (is_variable<T_scale>::value)
        result += 0.5 * log(sigma);
    ...
}
\end{verbatim}
\end{itemize}

\sld{Stan's Namesake}
\begin{itemize}
\item Stanislaw Ulam (1909--1984)
\item Co-inventor of Monte Carlo method (and hydrogen bomb)
\item[]
\begin{center}
\includegraphics[width=0.25\textwidth]{img/ulam-fermiac.jpg}
\end{center}
{\small
\item Ulam holding the Fermiac, Enrico Fermi's physical Monte Carlo simulator
for random neutron diffusion}
\end{itemize}

\clearpage
\mbox{ }
\vfill
\begin{center}
{\huge\bfseries The End}
\end{center}
\vfill
\mbox{ }


\end{document}


