\documentclass[a4paper,11pt]{article}
\usepackage[latin1]{inputenc}
\usepackage[english]{babel}
\usepackage{epsfig}
\usepackage{amsmath}
\usepackage{amsfonts}

\newcommand\R{{\mathbb R}}
\newcommand\x{{\mathbf x}}
\newcommand\X{{\mathbf X}}
\newcommand\K{{\mathbf K}}
\newcommand\J{{\mathbf J}}
\newcommand\LL{{\mathbf L}}
\newcommand\ELL{{\Ivec \ell}}
%%\newcommand\L{{\mathbf L}}
%%\DeclareMathOperator*{\argmax}{arg\,max}
%% \DeclareMathOperator*{\argmax}{argmax} %% (argmax wihtouth mid space)
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\var}{var}
\DeclareMathOperator*{\dm}{dm}
\newcommand{\Rvec}[1]{{\bf #1}}
\newcommand{\Ivec}[1]{\mbox{\boldmath $#1$}}

\title{Kernels}
\author{Emanuele Olivetti}

\begin{document}

\maketitle

\section{Introduction}
This document gives a detailed description of kernels implemented in
PyMVPA together with derivation of their gradients. These kernels are
used mainly by Gaussian Process Regression but can be of use for other
kernel methods. Gradients are useful when trying to maximize the
marginal likelihood of a Gaussian Process, i.e., during model
selection. In some cases a further reparametrization in log scale is
provided in order to reduce numerical stability issues when computing
likelihoods.

The following notation and definitions are used:
\begin{itemize}
\item $\x \in \R^D$ : a $D$-dimensional column vector, $\x =
  (x_1,\ldots,x_D)$.
\item $\X = (\x_1^{\top},\ldots,\x_N^{\top})$ : a $N \times D$ matrix
  where each row is a $D$-dimensional vector. $\X$ is also called set
  of \emph{samples}. $\X_{* i}$ indicates the $i$-th column of $\X$
  and is a column vector. $\X_{j *}$ indicates the $j$-th row of $\X$
  and is a row vector.
\item $k: \R^D \times \R^D \rightarrow \R$ : a covariance (or kernel)
  function.
\item $\K(\X,\X')$ : the matrix extension of $k$, i.e., $\K_{pq} =
  k(\X_{*p},\X'_{*q})$. If $\X$ is a $N \times D$ matrix and $\X'$ is
  $N' \times D$ then $\K(\X,\X')$ is a $N \times N'$ matrix.
\item $\J_{n,m}$ : the $n \times m$ matrix of ones, i.e., a matrix
  where each element is 1.
\item $\|\mathbf{z}\|_p$ : the $p$-norm of vector $\mathbf{z}$ defined
  as $\|\mathbf{z}\|_p = (\sum_{i=1}^D \|z_i\|^p)^{\frac{1}{p}}$.
  Euclidean norm is $p=2$, then $\|\mathbf{z}\|_2 =
  \sqrt{\mathbf{z}^{\top}\mathbf{z}} = \sqrt{\sum_{i=1}^D z_i^2}$.
\item $\|\mathbf{z},\mathbf{w}\|_p$ : the \emph{weighted} $p$-norm of
  vector $\mathbf{z}$ defined as $\|\mathbf{z},\mathbf{w}\|_p =
  (\sum_{i=1}^D w_i|z_i|^p)^{\frac{1}{p}}$.  Euclidean norm is
  $p=2$, then $\|\mathbf{z},\mathbf{w}\|_2 = \sqrt{\mathbf{z}^{\top}
    \mathbf{W}^{-1} \mathbf{z}} = \sqrt{\sum_{i=1}^D w_i z_i^2}$,
  where $\mathbf{W} = diag(\mathbf{w})$.
\item $\dm(\X,\X')$ : the \emph{Euclidean distance matrix} between
  $\X$ and $\X'$ defined element by element as $\dm(\X,\X')_{pq} =
  \|\X_{p *} - \X'_{q *}\|_2 = \sqrt{\sum_{i=1}^D (\X_{p i} - \X'_{q
      i})^2}$. If $\X$ is a $N \times D$ matrix and $\X'$ is $N'
  \times D$ then $\dm(\X,\X1)$ is a $N \times N'$ matrix. Note that
  $\dm(\X,X')$ is the square root of what it is usually called
  ``distance matrix''.
\item $\dm(\X,\X',\mathbf{w})$ : the \emph{weighted} Euclidean
  distance matrix between $\X$ and $\X'$ defined element by element as
  $\dm(\X,\X',\mathbf{w})_{pq} = \|(\X_{p *} - \X'_{q *})^{\top},
  \mathbf{w}\|_2 = \sqrt{\sum_{i=1}^D w_i(\X_{p i} - \X'_{q i})^2}$
  through the weight vector $\mathbf{w} \in \R^D$. It is
  straightforward that $\dm(\X,\X') = \dm(\X,\X',\mathbf{J}_{D,1})$.
\item $\X \bullet \mathbf{Y}$ : the Hadamard (or Schur) matrix
  product, i.e. the entry-wise product between matrices of the same
  size. Let $\mathbf{Z} = \X \bullet \mathbf{Y}$, then $z_{ij} =
  x_{ij} y_{ij}$.
\item $\X^{\alpha}$ : $(\X^{\alpha})_{ij} = (X_{ij})^{\alpha}$.
\end{itemize}

\section{Constant kernel}
$$k(\x,\x') = \sigma_0^2$$
where $\sigma_0 \ge 0$ is the standard deviation of the Gaussian prior
probability $\mathcal{N}(0,\sigma_0^2)$ of the value of the constant.
$$\K(\X,\X') = \sigma_0^2 \J_{N,N'}$$
$$\mathbf{\Theta} = \{\sigma_0\}$$
$$\frac{\partial k}{\partial \sigma_0}(\x,\x') = 2\sigma_0$$
$$\frac{\partial \K}{\partial \sigma_0} = 2\sigma_0 \J_{N,N'}$$
$$A = \sigma_0^2$$
$$A \ge 0$$
$$\mathbf{\Theta}^* = \{A\}$$
$$k(\x,\x') = A$$
$$\K(\X,\X') = A \J_{N,N'}$$
$$\frac{\partial k}{\partial A} = 1$$
$$\nabla_A \K = \frac{\partial \K}{\partial A} = \J_{N,N'}$$
Note that using $A$ as hyperparameter the gradient becomes constant.

\section{Linear kernel}
Let $\Ivec{\Sigma}_p$ be the $D \times D$ covariance matrix of the Gaussian
prior probability $\mathcal{N}(\Ivec{0},\Ivec{\Sigma}_p)$ of the weights of
the Bayesian linear regression.
$$k(\x,\x') = \x^{\top} \Ivec{\Sigma}_p \x' + \sigma_2^2$$
$$\K(\X,\X') = \X \Ivec{\Sigma}_p \X'^{\top} + \Ivec{I}\sigma_0^2$$
In order to simplify formulas we assume $\Ivec{\Sigma}_p$ is diagonal, i.e.,
$\Ivec{\Sigma}_p = diag(\Ivec{\sigma}^2_p)$ where $\Ivec{\sigma}^2_p =
({\sigma^2_p}_1,\ldots,{\sigma^2_p}_D)$:
$$k(\x,\x') = \sum_{i=1}^D {\sigma^2_p}_i x_i x'_i + \sigma_0^2$$
$$\mathbf{\Theta} = \{{\sigma_p}_1,\ldots,{\sigma_p}_D,\sigma_0\}$$
$$\frac{\partial k}{\partial {\sigma_p}_i} = 2 {\sigma_p}_i x_i x'_i$$
$$A_i = {\sigma_p^2}_i$$
$$A_i \ge 0$$
$$\mathbf{A} = (A_1,\ldots,A_D)^{\top}$$
$$\frac{\partial k}{\partial \sigma_0} = 2 \sigma_0$$
$$B = \sigma_0^2$$
$$\mathbf{\Theta}^* = \{ \mathbf{A} , B\}$$
$$k(\x,\x') = \x^{\top} diag(\mathbf{A}) \x' + B$$
$$\K(\X,\X') = \X diag(\mathbf{A}) \X'^{\top} + \Ivec{I}B$$
$$\frac{\partial k}{\partial A_i} = x_i x'_i$$
$$\frac{\partial \K}{\partial A_i} = \X_{* i} {\X'_{* i}}^{\top}$$
$$\nabla_{\mathbf{A}} \K = ( \X_{* 1} {\X'_{* 1}}^{\top}, \ldots,
\X_{* D} {\X'_{* D}}^{\top})$$
$$\frac{\partial k}{\partial B} = 1$$
$$\frac{\partial \K}{\partial B} = \Ivec{I}$$
As expected the gradient is independent of the hyperparameters values
and can be computed once for all at the beginning.

Logscale:
$$A_i = \ln{\sigma_{p_i}^2}$$
$$\sigma_{p_i}^2 = e^{A_i}$$
$$\frac{\partial k}{\partial A_i} = \sigma_{p_i}^2 x_i x_j$$
$$B = \ln{\sigma_0}$$
$$\sigma_0^2 = e^{2B}$$
$$\frac{\partial k}{\partial B} = 2 e^{2B} = 2 \sigma_0^2$$
Another reparametrization:
$$C = \ln{\sigma_0^2}$$
$$\sigma_0^2 = e^C$$
$$\frac{\partial k}{\partial C} = \sigma_0^2$$

\section{Polynomial kernel}
$$k(\x,\x') = (\sigma_0^2 + \x^{\top} \Ivec{\Sigma}_p \x')^p =
(\sigma_0^2 + \sum_{i=1}^D {\sigma^2_p}_i x_i x'_i)^p$$
$$\K(\X,\X') = (\sigma_0^2 \mathbf{J}_{N,N'} + \X \Ivec{\Sigma}_p
\X'^{\top})^p$$
In order to simplify formulas we assume $\Ivec{\Sigma}_p$ is diagonal, i.e.,
$\Ivec{\Sigma}_p = diag(\Ivec{\sigma}^2_p)$ where $\Ivec{\sigma}^2_p =
({\sigma^2_p}_1,\ldots,{\sigma^2_p}_D)$.
$$\sigma_0 \ge 0$$
$$\Ivec{\sigma}_p = ({\sigma_p}_1,\ldots,{\sigma_p}_D)$$
$${\sigma_p}_i \ge 0$$
$$\mathbf{\Theta} = \{\sigma_0,\Ivec{\sigma}_p, p\}$$
$$A = \sigma_0^2$$
$$B_i = {\sigma^2_p}_i$$
$$\mathbf{B} = (B_1,\ldots,B_D)$$
$$\mathbf{\Theta}^* = \{A,\mathbf{B}, p\}$$
$$k(\x,\x') = (A + \x^{\top} diag(\mathbf{B}) \x')^p = (A +
\sum_{i=1}^D B_i x_i x'_i)^p$$
$$\frac{\partial k}{\partial A} = p(A + \sum_{i=1}^D B_i x_i
x'_i)^{p-1}$$
$$\frac{\partial \K}{\partial A} = p(A\mathbf{J}_{N,N'} + \sum_{i=1}^D
B_i \X_{*i} {\X'_{*i}}^{\top})^{p-1}$$
$$\frac{\partial k}{\partial B_i} = p(A + \sum_{i=1}^D B_i x_i
x'_i)^{p-1} x_i x'_i = \frac{\partial k}{\partial A} x_i x'_i$$
$$\frac{\partial \K}{\partial B_i} = \frac{\partial \K}{\partial A} \X_{*i} {\X'_{*i}}^{\top}$$
$$\frac{\partial k}{\partial p} = k(\x,\x') \ln(A + \sum_{i=1}^D B_i x_i
x'_i)$$
$$\frac{\partial \K}{\partial p} = \K(\X,\X') \bullet \ln(A \mathbf{J}_{N,N'}+ \sum_{i=1}^D B_i \X_{*i} \X'_{*i})$$
$$\mathbf{M} = A \mathbf{J}_{N,N'}+ \sum_{i=1}^D B_i \X_{*i} \X'_{*i}$$
$$\nabla_{A,\mathbf{B},p} \K = \left(p\mathbf{M}^{p-1},\left\{p\mathbf{M}^{p-1} \X_{*i} {\X'_{*i}}^{\top} \right\}_{i=1,\ldots,D}, \K(\X,\X') \bullet \ln(\mathbf{M}) \right)$$

\section{Exponential kernel}
\subsection{Scalar Length-scale $\ell$}
$$k(\x,\x') = \sigma_f^2 e^{-\frac{\|\x-\x'\|_2}{\ell}}$$
$$\ell > 0$$
$$\sigma_f \ge 0$$
$$\mathbf{\Theta} = \{ \sigma_f, \ell \}$$
$$\K(\X,\X') = \sigma_f^2 e^{-\frac{1}{\ell}\dm(\X,\X')}$$
$$A = \sigma_f^2$$
$$A \ge 0$$
$$B = -\frac{1}{\ell}$$
$$B < 0$$
$$\mathbf{\Theta}^* = \{ A, B \}$$
$$k(\x,\x') = A e^{B\|\x-\x'\|_2}$$
$$\K(\X,\X') = A e^{B \dm(\X-\X')}$$
$$\frac{\partial k}{\partial A} = e^{B\|\x-\x'\|_2} = \frac{1}{A}k(\x,\x')$$
$$\frac{\partial \K}{\partial A} = e^{B \dm(\X,\X')} = \frac{1}{A} \K(\X,\X')$$
$$\frac{\partial k}{\partial B} = A e^{B\|\x-\x'\|_2} \|\x-\x'\|_2 =
k(\x,\x') \|\x-\x'\|_2$$
$$\frac{\partial \K}{\partial B} = \K(\X,\X') \bullet \dm(\X-\X')$$
$$\nabla_{A,B} \K = (\frac{1}{A} \K(\X,\X'), \K(\X,\X') \bullet \dm(\X-\X'))$$

Note that if $\K(\X,\X')$ is precomputed, then the gradient consists
in just two element-by-element products, the second being against a
constant matrix independent of the hyperparameters.

\subsection{Vector of Length-Scales $\Ivec{\ell}$}
Given $\Ivec{\ell} = (\ell_1,\ldots,\ell_D)$, $\ell_i \ge 0$ and
$\ELL^{-1} = (1/\ell_1,\ldots,1/\ell_D)$
$$k(\x,\x') = \sigma_f^2 e^{-\|\x-\x',\ELL^{-2}\|_2}= \sigma_f^2
  e^{-\sqrt{\sum_{i=1}^D \left(\frac{x_i - x'_i}{\ell_i}\right)^2}}$$
$$K(\X,\X') = \sigma_f^2 e^{-\dm(\X,\X',\ELL^{-2})}$$
$$\mathbf{\Theta} = \{ \sigma_f, \ELL\}$$
$$\sigma_f \ge 0$$
$$\ell_i > 0$$
$$A = \sigma_f^2$$
$$\mathbf{B} = \ELL^{-2}$$
$$A \ge 0$$
$$B_i > 0$$
$$\mathbf{\Theta}^* = \{ A, \mathbf{B}\}$$
$$k(\x,\x') = A e^{-\|\x-\x',\mathbf{B}\|_2} = A e^{-\sqrt{\sum_{i=1}^D
    B_i(x_i - x'_i)^2}}$$  
$$K(\X,\X') = A e^{-\dm(\X,\X',\mathbf{B})}$$
$$\frac{\partial k}{\partial A} = e^{-\sqrt{\sum_{i=1}^D B_i(x_i -
    x'_i)^2}} = \frac{k(\x,\x')}{A}$$
$$\frac{\partial \K}{\partial A} = e^{-\dm(\X,\X',\mathbf{B})} =
\frac{1}{A}\K(\X,\X')$$ 
$$\frac{\partial k}{\partial B_i} = A e^{-\|\x-\x',\mathbf{B}\|_2}
\left( -\frac{1}{2} \|\x-\x',\mathbf{B}\|_2^3 \right) (x_i - x'_i)^2 =
-\frac{1}{2} k(\x,\x') \|\x-\x',\mathbf{B}\|_2^3 (x_i - x'_i)^2 $$
$$\frac{\partial \K}{\partial B_i} = -\frac{1}{2} K(\X,\X')
\bullet \dm(\X-\X',\mathbf{B})^3 \bullet (\X_{*i}J_{1,N'}-
(\X'_{*i}J_{1,N})^{\top})^2$$
$$\nabla_{A,\mathbf{B}} \K = \left(\frac{1}{A}\K(\X,\X'), \left\{-\frac{1}{2}
    K(\X,\X') \bullet \dm(\X-\X',\mathbf{B})^3 \bullet
    (\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2 \right\}_{i=1 \ldots
    D} \right)$$

Note that $\frac{\partial \K}{\partial A}$ requires just the
multiplication of a constant by the kernel matrix $\K(\X,\X')$ whose
values usually already available. Instead $\frac{\partial \K}{\partial
  B_i}$ is a entry-wise product of 3 matrices: $\K(\X,\X')$ (usually
already available), $\dm(\X-\X',\mathbf{B})^3$ (which is, apart the
cube, part of the computation of $\K(\X,\X')$ so it can be stored in
advance), and $(\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2$ which
does not depend upon $A$ and $\mathbf{B}$ so it can be computed once
for all. Note that in NumPy $(\X_{*i}J_{1,N'}-
(\X'_{*i}J_{1,N})^{\top})^2$ can be computed as {\ttfamily
  numpy.subtract.outer($X_{*i},X'_{*i}$)**2}.

\section{Squared Exponential kernel}

\subsection{Scalar Length-Scale $\ell$}
$$k(\x,\x') = \sigma_f^2 e^{-\frac{1}{2\ell^2} (\x-\x')^{\top}(\x-\x')}
  = \sigma_f^2 e^{-\frac{1}{2\ell^2} \sum_{i=1}^D (x_i - x'_i)^2} =
  \sigma_f^2 e^{-\frac{1}{2\ell^2} \|\x-\x'\|_2^2}$$ 
$$\K(\X,\X') = \sigma_f^2 e^{-\frac{1}{2\ell^2} \dm(\X,\X')^2}$$
$$\sigma_f \ge 0$$
$$\ell > 0$$
$$\mathbf{\Theta} = \{ \sigma_f, \ell \}$$
$$\frac{\partial k}{\partial \sigma_f} = \frac{2}{\sigma_f}k$$
$$\frac{\partial \K}{\partial \sigma_f} = \frac{2}{\sigma_f}\K$$
$$\frac{\partial k}{\partial \ell} = \ell^{-3} k \|\x-\x'\|_2^2 $$
$$\frac{\partial \K}{\partial \ell} = \ell^{-3} \K \bullet \dm(\X,\X')^2$$
$$\nabla_{\sigma_f,\ell} \K = \left(\frac{2}{\sigma_f}K(\X,\X'), \ell^{-3} \K \bullet \dm(\X-\X')^2 \right)$$
Logscale:
$$A = \ln{\sigma_f}$$
$$\sigma_f = e^A$$
$$\frac{\partial k}{\partial A} = 2k$$
$$\frac{\partial \K}{\partial \sigma_f} = 2\K$$
$$B = \ln{\ell}$$
$$\ell = e^B$$
$$\frac{\partial k}{\partial B} = \ell^{-2} k \|\x-\x'\|_2^2$$
$$\frac{\partial \K}{\partial B} = \ell^{-2} \K \bullet \dm(\X,\X')^2 = \K \bullet \dm(\X,\X',\ell^{-2}\J_N)^2$$
$$\nabla_{A,B} \K = \left( 2\K(\X,\X'), \ell^{-2} \K \bullet \dm(\X-\X')^2 \right)$$
Another mapping:
$$A = \sigma_f^2$$
$$B = -\frac{1}{\ell^2}$$
$$A \ge 0$$
$$B < 0$$
$$\mathbf{\Theta}^* = \{ A, B\}$$
$$k(\x,\x') = A e^{B (\x-\x')^{\top}(\x-\x')}$$
$$\K(\X,\X') = A e^{B \dm(\X,\X')^2}$$
$$\frac{\partial k}{\partial A} = \frac{k(\x,\x')}{A}$$
$$\frac{\partial \K}{\partial A} = \frac{1}{A}\K(\X,\X')$$
$$\frac{\partial k}{\partial B} = k(\x,\x') \|\x-\x'\|_2^2$$
$$\frac{\partial \K}{\partial B} = K(\X,\X') \bullet \dm(\X,\X')^2$$
$$\nabla_{A,B} \K = (\frac{1}{A} \K(\X,\X'), \dm(\X-\X')^2 \bullet
\K(\X,\X'))$$

Note that $\nabla_{A,B} \K$ is similar to that of the exponential
kernel so almost all comments made before applies here as well.


\subsection{Vector of Length-Scales $\ELL$}
Let $\mathbf{L} = diag(\ELL)$:
$$k(\x,\x') = \sigma_f^2 e^{-\frac{1}{2}(\x-\x')^{\top} \LL^{-2}
  (\x-\x')} = \sigma_f e^{-\frac{1}{2}\sum_{i=1}^D \frac{(x_i -
    x'_i)^2}{\ell_i^2}} = \sigma_f^2
e^{-\frac{1}{2}\|\x-\x',\ELL^{-2}\|_2^2}$$
$$\K(\X,\X') = \sigma_f^2 e^{-\frac{1}{2} \dm(\X,\X',\ELL^{-2})^2}$$
$$\sigma_f \ge 0$$
$$\ell_i > 0$$
$$\mathbf{\Theta} = \{ \sigma_f, \ELL\}$$
$$\frac{\partial k}{\partial \sigma_f} = \frac{2}{\sigma_f}k$$
$$\frac{\partial \K}{\partial \sigma_f} = \frac{2}{\sigma_f}\K$$
$$\frac{\partial k}{\partial \ell_i} = \ell_i^{-3} k \|\x_i-\x_i'\|_2^2 $$
$$\frac{\partial \K}{\partial \ell_i} = \ell_i^{-3} \K \bullet (\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2$$
$$\nabla_{\sigma_f,\ell} \K = \left(\frac{2}{\sigma_f}K(\X,\X'), \ell_i^{-3} \K \bullet (\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2 \right)$$
Logscale:
$$A = \ln{\sigma_f}$$
$$\sigma_f = e^A$$
$$\frac{\partial k}{\partial A} = 2k$$
$$\frac{\partial \K}{\partial \sigma_f} = 2\K$$
$$B_i = \ln{\ell_i}$$
$$\ell_i = e^B_i$$
$$\frac{\partial k}{\partial B_i} = \ell_i^{-2} k \|\x_i-\x'_i\|_2^2$$
$$\frac{\partial \K}{\partial B_i} = \ell^{-2} \K \bullet \dm(\X,\X')^2 = \ell_i^{-2} \K \bullet (\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2$$
$$\nabla_{A,\mathbf{B}} \K = \left( 2\K(\X,\X'), \left\{ \ell_i^{-2} \K \bullet (\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2 \right\}_{i=1 \ldots D} \right)$$
Another mapping:
$$A = \sigma_f$$
$$\mathbf{B} = -\frac{1}{2}\ELL^{-2} =
\left(-\frac{1}{2\ell_1^2},\ldots,-\frac{1}{2\ell_D^2} \right)$$
$$A \ge 0$$
$$B_i < 0$$
$$\mathbf{\Theta}^* = \{ A, \mathbf{B}\}$$
$$k(\x,\x') = A e^{(\x-\x')^{\top} diag(\mathbf{B}) (\x-\x')} = A
e^{\sum_{i=1}^D B_i (x_i - x'_i)^2}$$ 
$$\K(\X,\X') = A e^{\dm(\X,\X',\mathbf{B})}$$
$$\frac{\partial k}{\partial A} = \frac{k(\x,x')}{A}$$
$$\frac{\partial \K}{\partial A} = \frac{1}{A}\K(\X,X')$$
$$\frac{\partial k}{\partial B_i} = k(\x,\x') (x_i -x'_i)^2$$
$$\frac{\partial \K}{\partial B_i} = \K(\X,\X') \bullet
(\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2$$
$$\nabla_{A,\mathbf{B}} \K = \left(\frac{\K(\X,\X')}{A}, \left\{ K(\X,\X')
  \bullet (\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2 \right\}_{i=1
  \ldots D} \right)$$

Note that $\nabla_{A,\mathbf{B}} \K$ requires to compute $K(\X,\X')$
(which is usually already available), and its entry-wise product with
$(\X_{*i}J_{1,N'}- (\X'_{*i}J_{1,N})^{\top})^2$ which is independent
of the value of the hyperparameters and can be precomputed once for
all.

\section{$\gamma$-Exponential kernels}
$$k(\x,\x') = \sigma_f^2 e^{-(\frac{\x-\x'}{\ell})^\gamma}$$
$$k(\x,\x') = \sigma_f^2 e^{-(\frac{\x-\x'}{\ELL})^\gamma}$$

\section{Mat\'ern kernels}
%% $$k_{\mbox{Mat\'ern}}(\x,\x') = \frac{2^{1-\nu}}{\Gamma(\nu)} \left(\sqrt{2\nu} \frac{\|\x-\x'\|_2}{\ell}\right)^{\nu} \K_{\nu}\left(\sqrt{2}\nu \frac{\|\x-\x'\|_2}{\ell} \right)$$
%% $$\ell > 0$$
%% $$\nu > 0$$
%% where $K_{\nu}$ is a modified Bessel function (REMOVE?).
Let $\nu$ be half integer, i.e., $\nu = p + 1/2$ ($p \in
\mathbb{Z}^+$), then
$$k_{\nu=p+1/2}(\x,\x') = e^{-\sqrt{2\nu}\frac{\|\x-\x'\|}{\ell}}
\frac{\Gamma(p+1)}{\Gamma(2p+1)} \sum_{i=0}^p
\frac{(p+i)!}{i!(p-i)!}\left(\sqrt{8\nu}\frac{\|\x-\x'\|}{\ell}\right)^{p-i}$$
is the class of Mat\'ern covariance functions with half-integer $\nu$.
$$\nu > 0$$
$$\ell > 0$$
When $\nu \rightarrow \infty$ we obtain the squared exponential
covariance function. Most popular cases of the Mat\'ern functions are
$p=0$ (exponential kernel), $p=1$ and $p=2$:
$$k_{\nu=1/2}(\x,\x') = e^{-\frac{\|\x-\x'\|_2}{\ell}}$$
$$k_{\nu=3/2}(\x,\x') = \left(1+\sqrt{3}\frac{\|\x-\x'\|_2}{\ell}
\right) e^{-\sqrt{3}\frac{\|\x-\x'\|_2}{\ell}}$$ 
$$k_{\nu=5/2}(\x,\x') = \left(1+\sqrt{5}\frac{\|\x-\x'\|_2}{\ell}
 + \frac{5\|\x-\x'\|_2^2}{3\ell^2}\right)
e^{-\sqrt{5}\frac{\|\x-\x'\|_2}{\ell}}$$


\section{Rational Quadratic kernels}
$$k_{RQ} = \left(1+\frac{\|\x-\x'\|_2^2}{2\alpha\ell^2}
\right)^{-\alpha}$$
$$\alpha > 0$$
$$\ell > 0$$



\end{document}

% LocalWords:  Logscale
