%Copyright (C) 2007-2015  Brian Langenberger
%This work is licensed under the
%Creative Commons Attribution-Share Alike 3.0 United States License.
%To view a copy of this license, visit
%http://creativecommons.org/licenses/by-sa/3.0/us/ or send a letter to
%Creative Commons,
%171 Second Street, Suite 300,
%San Francisco, California, 94105, USA.

\chapter{the Basics}

In order to best illustrate how different codecs operate,
this document consists of format diagrams, pseudocode,
and examples.

\section{Format Diagrams}

Format diagrams provide an overview of the format, such as:
\begin{figure}[h]
  \includegraphics{figures/diagram.pdf}
\end{figure}
\par
\noindent
Each box represents a field, and each field has a name.
In this case, our names are a, b, c, d and e.
Where possible, each field will have a starting bit
in its lower-left corner and an ending bit in its lower-right.
For example, a is comprised of bits 0 and 1, making it a 2-bit value.
The b field uses bits 2 through 4, making it a 3-bit value, and so on.
Variable-length fields such as C-strings may not have starting or ending
bit values.
Fields may also be nested or looped, such as:
\begin{figure}[h]
  \includegraphics{figures/diagram2.pdf}
\end{figure}
\par
\noindent
As a convention, compound fields are named with capital letters
while individual fields use lowercase.
\par
For formats with little or no logic, such as ID3 metadata,
a diagram may be all that's needed for illustration.
However, for more complex codecs, the format diagram serves as a map
which accompanies its pseudocode.
For example, a file format may be composed of many frames, each frame
consists of many subframes, and each subframe consists of many fields.
By placing a diagram alongside the subframe decoding pseudocode,
one can see where you are (a subframe),
where you've been (the enclosing frame)
and where you're going next (the subframe's fields).

\clearpage

\section{Pseudocode}

Pseudocode provides a language-agnostic way of illustrating the flow of logic
required to decode or encode a file format.
These blocks are prefixed by the input expected and the output produced.
One or more operations are performed and something is returned.
For example:
\ALGORITHM{x and y values}{a multiplied value}
\Return $x \times y$\;
\EALGORITHM
\par
\noindent
Any variables not passed into a pseudocode block are created
using the assignment operator, such as:
\par
\noindent
\begin{algorithm}[H]
  \DontPrintSemicolon
  $x \leftarrow 1$\;
  $y \leftarrow 2$\;
\end{algorithm}
\par
\noindent
which creates the variables $x$ and $y$ and assigns them the values of 1 and 2.
In C, this would be implemented as:
\begin{Verbatim}[xleftmargin=.25in]
int x = 1;
int y = 2;
\end{Verbatim}
Comparisons are performed using the usual math operators
$<$, $\leq$, $=$, $\neq$, $\geq$, $>$.  So:
\par
\noindent
\begin{algorithm}[H]
  \DontPrintSemicolon
  $x = y$\;
\end{algorithm}
\par
\noindent
is implemented in C as:
\begin{Verbatim}[xleftmargin=.25in]
x == y
\end{Verbatim}
Variables with subscripts are arrays,
which are also created through assignment:
\par
\noindent
\begin{algorithm}[H]
  \DontPrintSemicolon
  $a_0 \leftarrow x \times y$\;
\end{algorithm}
\par
\noindent
corresponds to:
\begin{Verbatim}[xleftmargin=.25in]
a[0] = x * y;
\end{Verbatim}
Multiple subscripts indicate multi-dimensional arrays:
\par
\noindent
\begin{algorithm}[H]
  \DontPrintSemicolon
  $b_{0~1~2} \leftarrow a_0 \times z$\;
\end{algorithm}
\par
\noindent
corresponds to:
\begin{Verbatim}[xleftmargin=.25in]
b[0][1][2] = a[0] * z;
\end{Verbatim}

\clearpage

\noindent
Variables are processed using familiar statements such as if/then/else:
\par
\noindent
\begin{algorithm}[H]
  \DontPrintSemicolon
  \eIf{$x > y$}{
    $z \leftarrow 1$\;
  }{
    $z \leftarrow 2$\;
  }
\end{algorithm}
\par
\noindent
which corresponds to:
\begin{Verbatim}[xleftmargin=.25in]
if (x > y) {
  z = 1;
} else {
  z = 2;
}
\end{Verbatim}
while loops:
\par
\noindent
\begin{algorithm}[H]
  \DontPrintSemicolon
  \While{$x < y$}{
    $x \leftarrow x \times 2$\;
  }
\end{algorithm}
\par
\noindent
implemented as:
\begin{Verbatim}[xleftmargin=.25in]
  while (x < y) {
    x = x * 2;
  }
\end{Verbatim}
and for loops which are inclusive of the first element
but \textit{not} inclusive of the last element:
\par
\noindent
\begin{algorithm}[H]
  \DontPrintSemicolon
  \For{$i \leftarrow 0$ \emph{\KwTo}5}{
    $a_i \leftarrow i \times 2$\;
  }
\end{algorithm}
\par
\noindent
which corresponds to:
\begin{Verbatim}[xleftmargin=.25in]
  for (i = 0; i < 5; i++) {
    a[i] = i * 2;
  }
\end{Verbatim}
whose final result is the same as $a \leftarrow \texttt{[0, 2, 4, 8]}$
\par
\noindent
Downto loops are \textit{not} inclusive of the first element,
but are inclusive of the last:
\par
\noindent
\begin{algorithm}[H]
  \SetKw{KwDownTo}{downto}
  \DontPrintSemicolon
  \For{$i \leftarrow 5$ \emph{\KwDownTo}0}{
    $b_i \leftarrow i \times 3$\;
  }
\end{algorithm}
\par
\noindent
corresponds to:
\begin{Verbatim}[xleftmargin=.25in]
  for (i = 5 - 1; i >= 0; i--) {
    b[i] = i * 3;
  }
\end{Verbatim}

\clearpage
\noindent
Switch blocks do not imply automatic fall-through:
\par
\noindent
\begin{algorithm}[H]
  \DontPrintSemicolon
  \Switch{$a$}{
    \uCase{0}{
      $b \leftarrow 1$\;
    }
    \uCase{1}{
      $b \leftarrow 2$\;
    }
    \Other{
      $b \leftarrow 100$\;
    }
  }
\end{algorithm}
\par
\noindent
corresponds to:
\begin{Verbatim}[xleftmargin=.25in]
switch (a) {
case 0:
  b = 1;
  break;
case 1:
  b = 2;
  break;
default:
  b = 100;
  break;
}
\end{Verbatim}

\subsection{Structures}

In rare instances, multiple values are packed into a single field
for easier representation, such as:
\par
\noindent
\begin{algorithm}[H]
\SetKwData{FIELD}{field}
\DontPrintSemicolon
  $\text{\FIELD} \leftarrow \left\lbrace\begin{tabular}{l}
  $a$ \\
  $b$ \\
  $c$ \\
  $d$ \\
  $e$ \\
  \end{tabular}\right.$\;
\end{algorithm}
\par
\noindent
which corresponds to:
\begin{Verbatim}[xleftmargin=.25in]
struct field {
  int a;
  int b;
  int c;
  int d;
  int e;
} field = {a, b, c, d, e};
\end{Verbatim}

\clearpage

\subsection{Bit Shifts}

Pseudocode is optimized to use regular mathematical operations
with as few variables and steps of operation as possible
while remaining understandable to the reader.
However, this may not be the most efficient way to implement
an algorithm in code.
One common example is the use of powers of two for multiplication
and division.
Where one sees code such as:
\begin{algorithm}[H]
  \DontPrintSemicolon
  $a \leftarrow b \times 2 ^ c$\;
\end{algorithm}
\par
\noindent
although one could implement it na\"ively as:
\begin{Verbatim}[xleftmargin=.25in]
a = (int)((double)b * pow(2.0, (double)c));
\end{Verbatim}
\par
\noindent
it's best to optimize that operation with a left shift:
\begin{Verbatim}[xleftmargin=.25in]
a = b << c;
\end{Verbatim}
\par
\noindent
Division is handled similarly where:
\par
\noindent
\begin{algorithm}[H]
  \DontPrintSemicolon
  $a \leftarrow \lfloor b \div 2 ^ c \rfloor$\;
\end{algorithm}
\par
\noindent
is best implemented with a right shift:
\begin{Verbatim}[xleftmargin=.25in]
a = b >> c;
\end{Verbatim}
\par
\noindent
Many codecs make heavy use of these powers of two operations
precisely because they can be implemented with
efficient bit shift operations.

\clearpage

\section{Input}

Input from a given file typically operates on some
combination of signed or unsigned bits, such as:
\par
\noindent
\begin{algorithm}[H]
  \DontPrintSemicolon
  \SetKw{READ}{read}
  $a \leftarrow$ \READ 2 unsigned bits\;
  $b \leftarrow$ \READ 3 unsigned bits\;
  $c \leftarrow$ \READ 5 unsigned bits\;
  $d \leftarrow$ \READ 3 signed bits\;
  $e \leftarrow$ \READ 19 signed bits\;
\end{algorithm}
\par
\noindent
How to read these bits depends on whether the stream is
big-endian or little-endian.

\subsection{Big Endian Input}

In a big-endian stream, the bits are consumed from
most-significant to least-significant:
\begin{figure}[h]
  \includegraphics{figures/big_endian.pdf}
\end{figure}

\begin{table}[h]
\begin{tabular}{r>{$}r<{$}r}
  $a \leftarrow$ & \texttt{\color{blue}1} \times 2 ^ 1 + \texttt{\color{blue}0} \times 2 ^ 0 & = 2 \\
  $b \leftarrow$ & \texttt{\color{darkgreen}1} \times 2 ^ 2 + \texttt{\color{darkgreen}1} \times 2 ^ 1 + \texttt{\color{darkgreen}0} \times 2 ^ 0 & = 6 \\
  $c \leftarrow$ & \texttt{\color{fuchsia}0} \times 2 ^ 4 + \texttt{\color{fuchsia}0} \times 2 ^ 3 + \texttt{\color{fuchsia}1} \times 2 ^ 2 + \texttt{\color{fuchsia}1} \times 2 ^ 1 + \texttt{\color{fuchsia}1} \times 2 ^ 0 & = 7 \\
\end{tabular}
\end{table}

\subsubsection{Signed Values}

Signed values are typically stored as 2s-complement
which is parsed as follows:
\par
\noindent
\ALGORITHM{$n$ number of bits to read in big-endian}{a signed value}
\SetKwData{SIGN}{sign}
\SetKwData{UNSIGNED}{unsigned}
  $\SIGN \leftarrow $ \READ 1 unsigned bit\;
  $\UNSIGNED \leftarrow $ \READ $(n - 1)$ unsigned bits\;
  \eIf{$\SIGN = 0$}{
    \Return \UNSIGNED\;
  }{
    \Return $\UNSIGNED - 2 ^ {n - 1}$\;
  }
\EALGORITHM

\begin{table}[h]
  \begin{tabular}{r|r||r|l|>{$}r<{$}l}
    field & $n$ & \textsf{sign} & \textsf{unsigned} & \multicolumn{2}{r}{\text{signed value}} \\
    \hline
    $d$ & 3 & \texttt{\color{blue}1} & \texttt{\color{blue}0 1} & 1 - 2 ^ {3 - 1} &= -3 \\
    $e$ & 19 & \texttt{\color{orange}1} & \texttt{\color{orange}0 1 0 0 1 1 1 0 1 1 1 1 0 0 0 0 0 1} & 80833 - 2 ^ {19 - 1} &= -181311 \\
  \end{tabular}
\end{table}

\clearpage

\subsection{Little Endian Input}
In a little-endian stream, the bits are consumed from least-significant
to most-significant.
Using the same bit counts and signedness as before, the same four bytes are
read as follows:
\begin{figure}[h]
  \includegraphics{figures/little_endian.pdf}
\end{figure}
\begin{table}[h]
  \begin{tabular}{r>{$}r<{$}r}
    $a \leftarrow$ & \texttt{\color{blue}1} \times 2 ^ 0 + \texttt{\color{blue}0} \times 2 ^ 1 & = 1 \\
    $b \leftarrow$ & \texttt{\color{darkgreen}0} \times 2 ^ 0 + \texttt{\color{darkgreen}0} \times 2 ^ 1 + \texttt{\color{darkgreen}1} \times 2 ^ 2 & = 4 \\
    $c \leftarrow$ & \texttt{\color{fuchsia}1} \times 2 ^ 0 + \texttt{\color{fuchsia}0} \times 2 ^ 1 + \texttt{\color{fuchsia}1} \times 2 ^ 2 + \texttt{\color{fuchsia}1} \times 2 ^ 3 + \texttt{\color{fuchsia}0} \times 2 ^ 4 & = 13 \\
  \end{tabular}
\end{table}

\subsubsection{Signed Values}
Signed bits are consumed using a different 2s-complement method:
\par
\noindent
\ALGORITHM{$n$ number of bits to read in little-endian}{a signed value}
\SetKwData{SIGN}{sign}
\SetKwData{UNSIGNED}{unsigned}
$\UNSIGNED \leftarrow $ \READ $(n - 1)$ unsigned bits\;
$\SIGN \leftarrow $ \READ 1 unsigned bit\;
\eIf{$\SIGN = 0$}{
  \Return \UNSIGNED\;
}{
  \Return $\UNSIGNED - 2 ^ {n - 1}$\;
}
\EALGORITHM

\begin{table}[h]
  \begin{tabular}{r|r||r|l|>{$}r<{$}l}
    field & $n$ & \textsf{unsigned} & \textsf{sign} & \multicolumn{2}{r}{\text{signed value}} \\
    \hline
    $d$ & 3 & \texttt{\color{blue}1 1} & \texttt{\color{blue}0} & &= 3 \\
    $e$ & 19 & \texttt{\color{orange}1 1 1 1 1 0 1 1 1 0 0 1 0 0 0 0 0 1} & \texttt{\color{orange}1} & 133599 - 2 ^ {19 - 1} &= -128545 \\
  \end{tabular}
\end{table}

\clearpage

\section{Output}

As with input, output to a given file operates on some combination
of signed or unsigned bits, such as:
\par
\noindent
\begin{algorithm}[H]
  \DontPrintSemicolon
  \SetKw{WRITE}{write}
  $a \rightarrow$ \WRITE 2 unsigned bits\;
  $b \rightarrow$ \WRITE 3 unsigned bits\;
  $c \rightarrow$ \WRITE 5 unsigned bits\;
  $d \rightarrow$ \WRITE 3 signed bits\;
  $e \rightarrow$ \WRITE 19 signed bits\;
\end{algorithm}

\subsection{Big Endian Output}
As with input, these bits are inserted into each byte in the output file
from most-significant to least-significant.
\begin{table}[h]
\begin{tabular}{r|r|r||r}
field & value & $n$ & written as \\
\hline
$a$ & 2 & 2 & \texttt{\color{blue}1 0} \\
$b$ & 6 & 3 & \texttt{\color{darkgreen}1 1 0} \\
$c$ & 7 & 5 & \texttt{\color{fuchsia}0 0 1 1 1} \\
\end{tabular}
\end{table}

\subsubsection{Signed Values}
\par
\noindent
\ALGORITHM{a signed value, $n$ number of bits to write in big-endian}{1 or more bits}
\SetKwData{VALUE}{value}
\eIf(\tcc*[f]{positive value}){$\VALUE \geq 0$}{
  $0 \rightarrow$ \WRITE 1 unsigned bit\;
  $\VALUE \rightarrow$ \WRITE $(n - 1)$ unsigned bits\;
}(\tcc*[f]{negative value}){
  $1 \rightarrow$ \WRITE 1 unsigned bit\;
  $(2 ^ {n - 1} + \VALUE) \rightarrow$ \WRITE $(n - 1)$ unsigned bits\;
}
\EALGORITHM
\par
\noindent
\begin{table}[h]
  \begin{tabular}{r|r|r||r>{$}r<{$}l}
    field & value & $n$ & sign & \multicolumn{2}{r}{\text{unsigned}} \\
    \hline
    $d$ & -3 & 3 & \texttt{\color{blue}1} & 2 ^ {3 - 1} - 3 & = {\color{blue}1} \\
    $e$ & -181311 & 19 & \texttt{\color{orange}1} & 2 ^ {19 - 1} - 181311 & = {\color{orange}80833} \\
  \end{tabular}
\end{table}
\begin{figure}[h]
  \includegraphics{figures/big_endian.pdf}
\end{figure}

\subsection{Little Endian Output}
These bits are inserted into each byte in the output file
from least-significant to most-significant.
\begin{table}[h]
  \begin{tabular}{r|r|r||r}
    field & value & $n$ & written as \\
    \hline
    $a$ & 1 & 2 & \texttt{\color{blue}1 0} \\
    $b$ & 4 & 3 & \texttt{\color{darkgreen}0 0 1} \\
    $c$ & 13 & 5 & \texttt{\color{fuchsia}1 0 1 1 0} \\
  \end{tabular}
\end{table}

\subsubsection{Signed Values}
Signed values are written as follows:
\par
\noindent
\ALGORITHM{a signed value, $n$ number of bits to write in little-endian}{1 or more bits}
\SetKwData{VALUE}{value}
\eIf(\tcc*[f]{positive value}){$\VALUE \geq 0$}{
  $\VALUE \rightarrow$ \WRITE $(n - 1)$ unsigned bits\;
  $0 \rightarrow$ \WRITE 1 unsigned bit\;
}(\tcc*[f]{negative value}){
  $(2 ^ {n - 1} + \VALUE) \rightarrow$ \WRITE $(n - 1)$ unsigned bits\;
  $1 \rightarrow$ \WRITE 1 unsigned bit\;
}
\EALGORITHM
\par
\noindent
\begin{table}[h]
  \begin{tabular}{r|r|r||>{$}r<{$}lr}
    field & value & $n$ & \multicolumn{2}{r}{\text{unsigned}} & sign \\
    \hline
    $d$ & 3 & 3 & & = {\color{blue}3} & \texttt{\color{blue}0} \\
    $e$ & -128545 & 19 & 2 ^ {19 - 1} - 128545 & = {\color{orange}133599} & \texttt{\color{orange}1} \\
  \end{tabular}
\end{table}

\begin{figure}[h]
  \includegraphics{figures/little_endian.pdf}
\end{figure}

\clearpage

\section{Unary}

Lossless codecs often make use of variable length unary coding
to store a non-negative value.

\subsection{Reading Unary}
\ALGORITHM{a stop bit of 0 or 1}{a non-negative value}
\SetKwData{UNARY}{unary}
\SetKwData{STOPBIT}{stop bit}
$\UNARY \leftarrow 0$\;
$b \leftarrow$ \READ 1 unsigned bit\;
\While{$b \neq \STOPBIT$}{
  $\UNARY \leftarrow \UNARY + 1$\;
  $b \leftarrow$ \READ 1 unsigned bit\;
}
\Return \UNARY\;
\EALGORITHM
\par
\noindent
For example, processing:
\par
\noindent
\begin{algorithm}[H]
  \DontPrintSemicolon
  \SetKw{UNARY}{read unary}
  \For{$i \leftarrow 0$ \emph{\KwTo}5}{
    $u_i \leftarrow$ \UNARY with stop bit 1\;
  }
\end{algorithm}
\par
\noindent
over the big-endian stream:
\begin{figure}[h]
  \includegraphics{figures/unary1.pdf}
\end{figure}
\par
\noindent
results in $u = \texttt{[0, 1, 2, 3, 4]}$

\subsection{Writing Unary}
\ALGORITHM{a stop bit of 0 or 1, a non-negative value}{1 or more bits}
\SetKwData{VALUE}{value}
\SetKwData{STOPBIT}{stop bit}
\eIf{$\STOPBIT = 0$}{
  $(2 ^ \text{\VALUE} - 1) \rightarrow$ \WRITE \VALUE unsigned bits\;
}{
  $0 \rightarrow$ \WRITE \VALUE unsigned bits\;
}
$\STOPBIT \rightarrow$ \WRITE 1 unsigned bit\;
\EALGORITHM
