#LyX 2.3 created this file. For more info see http://www.lyx.org/
\lyxformat 544
\begin_document
\begin_header
\save_transient_properties true
\origin unavailable
\textclass article
\use_default_options true
\maintain_unincluded_children false
\language english
\language_package default
\inputencoding auto
\fontencoding global
\font_roman "default" "default"
\font_sans "default" "default"
\font_typewriter "default" "default"
\font_math "auto" "auto"
\font_default_family default
\use_non_tex_fonts false
\font_sc false
\font_osf false
\font_sf_scale 100 100
\font_tt_scale 100 100
\use_microtype false
\use_dash_ligatures true
\graphics default
\default_output_format default
\output_sync 0
\bibtex_command default
\index_command default
\paperfontsize default
\use_hyperref false
\papersize default
\use_geometry false
\use_package amsmath 1
\use_package amssymb 1
\use_package cancel 1
\use_package esint 1
\use_package mathdots 1
\use_package mathtools 1
\use_package mhchem 1
\use_package stackrel 1
\use_package stmaryrd 1
\use_package undertilde 1
\cite_engine basic
\cite_engine_type default
\use_bibtopic false
\use_indices false
\paperorientation portrait
\suppress_date false
\justification true
\use_refstyle 1
\use_minted 0
\index Index
\shortcut idx
\color #008000
\end_index
\secnumdepth 3
\tocdepth 3
\paragraph_separation indent
\paragraph_indentation default
\is_math_indent 0
\math_numbering_side default
\quotes_style english
\dynamic_quotes 0
\papercolumns 1
\papersides 1
\paperpagestyle default
\tracking_changes false
\output_changes false
\html_math_output 0
\html_css_as_file 0
\html_be_strict false
\end_header

\begin_body

\begin_layout Section
Kernels
\end_layout

\begin_layout Standard
\begin_inset FormulaMacro
\newcommand{\d}{\mathrm{d}}
\end_inset


\end_layout

\begin_layout Standard
The original idea was to support arbitrary combinations of kernels in a
 product kernel.
 The problem is that this is either slow in a dynamic language or hard to
 realize and still not extremely fast in a compiled language.
 The easiest solution would be an array of function pointers in C.
 Even there, the function call overhead would be significant without inlining.
 So to capture 90% of use cases, we support only
\end_layout

\begin_layout Itemize
RBF kernels
\end_layout

\begin_layout Itemize
Product kernels with the same kernel in all dimensions
\end_layout

\begin_layout Standard
Say we have a product kernel
\begin_inset Formula 
\begin{equation}
k=k_{1}(x_{1}\lambda_{1})k(x_{2}\lambda_{2})\dots
\end{equation}

\end_inset

Then
\begin_inset Formula 
\begin{equation}
\frac{\partial k}{\partial x_{i}}=\lambda_{i}k_{1}k_{2}\dots k_{i}^{\prime}\dots=\lambda_{i}k\frac{k_{i}^{\prime}}{k_{i}}
\end{equation}

\end_inset

and
\begin_inset Formula 
\begin{equation}
\frac{\partial k}{\partial\lambda_{i}}=x_{i}k_{1}k_{2}\dots k_{i}^{\prime}\dots=x_{i}k\frac{k_{i}^{\prime}}{k_{i}}
\end{equation}

\end_inset

If there's only a single length scale, we have
\begin_inset Formula 
\begin{equation}
\frac{\partial k}{\partial l}=\left(x_{1}k_{1}^{\,\prime}k_{2}\dots+x_{2}k_{1}k_{2}^{\,\prime}\dots+\dots\right)=k\sum\frac{x_{i}}{k_{i}}k_{i}^{\prime}
\end{equation}

\end_inset

For NLL:
\begin_inset Formula 
\[
-\log p=\frac{1}{2}y^{T}K_{y}^{-1}y+\frac{1}{2}\log|K_{y}^{-1}|+\frac{n}{2}\log2\pi
\]

\end_inset

Derivative
\begin_inset Formula 
\begin{align}
-\d\log p & =\frac{1}{2}y^{T}\d K_{y}^{-1}y+\frac{1}{2}\d\log|K_{y}|\nonumber \\
 & =-\frac{1}{2}y^{T}K_{y}^{-1}\d K_{y}K_{y}^{-1}y+\frac{1}{2}\frac{1}{|K_{y}|}\mathrm{tr}\left(K_{y}\d K_{y}\right)
\end{align}

\end_inset


\begin_inset Formula 
\begin{equation}
\d K=\frac{\partial K}{\partial l_{i}}\d l_{i}
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
Details are now found in draft/generate_kern_vec/test_gen_kern.ipynb
\end_layout

\begin_layout Section
Likelihood
\end_layout

\begin_layout Standard
Value and derivative of the negative log-likelihood are
\begin_inset Formula 
\begin{align}
-2\log p & =\boldsymbol{y}^{T}K_{y}^{-1}\boldsymbol{y}+\log|K_{y}|+n\log2\pi.\\
-2\frac{\partial}{\partial\theta_{i}}\log p & =-\boldsymbol{y}^{T}K_{y}^{-1}\frac{\partial K_{y}}{\partial\theta_{i}}K_{y}^{-1}\boldsymbol{y}+\mathrm{tr}K_{y}^{-1}\frac{\partial K_{y}}{\partial\theta_{i}}\\
 & =\mathrm{tr}\left((K_{y}^{-1}-\boldsymbol{\alpha}\boldsymbol{\alpha}^{T})\frac{\partial K_{y}}{\partial\theta_{i}}\right)
\end{align}

\end_inset

with 
\begin_inset Formula $\boldsymbol{\alpha}=K_{y}^{-1}\boldsymbol{y}$
\end_inset

.
 In indexed notation this means
\begin_inset Formula 
\begin{align}
-2\frac{\partial}{\partial\theta_{i}}\log p & =\sum_{j,k,l}\delta_{jl}\left((K_{y\,jk}^{-1}-a_{j}a_{k})\left(\frac{\partial K_{y}}{\partial\theta_{i}}\right)_{kl}\right)\nonumber \\
 & =\sum_{j,k}(K_{y\,jk}^{-1}-a_{j}a_{k})\left(\frac{\partial K_{y}}{\partial\theta_{i}}\right)_{kj}.
\end{align}

\end_inset


\end_layout

\end_body
\end_document
