\documentclass[12pt,letterpaper]{article}

\usepackage{amsmath,amsfonts,amssymb,bbm}
\usepackage{palatino}
\usepackage[linkcolor=blue]{hyperref}
\usepackage{fullpage}
\usepackage{color}
\usepackage{enumitem}
\usepackage{titlesec}
\usepackage[textsize=tiny]{todonotes}
\usepackage{multirow}

\newcommand{\TODO}[1]{\todo[inline]{#1}}
\newcommand{\R}{\mathbbm{R}}
\newcommand{\mba}{\mathbf{a}}
\newcommand{\mbb}{\mathbf{b}}
\newcommand{\mbx}{\mathbf{x}}
\newcommand{\mbxt}{\tilde{\mathbf{x}}}
\newcommand{\Sigmat}{\tilde{\Sigma}}
\newcommand{\mbz}{\mathbf{z}}
\newcommand{\mbw}{\mathbf{w}}
\newcommand{\mcN}{\mathcal{N}}
\newcommand{\mcP}{\mathcal{P}}
\newcommand{\eps}{\epsilon}
\newcommand{\trans}{\intercal}
\newcommand{\Ut}{\tilde{U}}
\DeclareMathOperator*{\argmax}{arg\,max}
\newcommand{\angstrom}{\textup{\AA}}
\newcommand{\E}{\mathbb{E}}
\newcommand{\var}{\mathrm{var}}
\newcommand{\cov}{\mathrm{cov}}
\newcommand{\p}{\partial}

\renewcommand{\v}[1]{\boldsymbol{\mathbf{#1}}}

\begin{document}

\begin{center}
\LARGE{CS 182 Fall 2018 Math Review Notes}
\end{center}

\section{Probability Review}

A {\bf{random variable}} $X$ is a variable which could take on various values with specified probabilities (its distribution). The possible outcomes of $X$ are described through {\bf{events}}. For instance, an event $A$ could be that $X$ takes on the value $2$, or that $X < 4$. \\

\noindent The probability that $A$ happens is denoted as $P(A)$. For continuous random variables $X$, the probability that $X$ attains some value $x$ is described by a function $p$, called the {\bf{probability density function (PDF)}}. Similarly, for discrete random variables $X$, it is called the {\bf{probability mass function (PMF)}}. We won't be too strict about only writing events as arguments to the probability function $p$; we will use $p(X)$ to mean the distribution of $X$, and $p(x)$ to mean $p(X = x)$. \\

\noindent Events can be constructed from other events. For instance, given two events $A$ and $B$, we may be interested in the event $C = (A \; and \; B)$ or $C = A \cap B$ (the event that both events happen) or the event $D = (A \; or \;  B)$ or $D = A \cup B$ (the event that either of the events happens). \\

\noindent Two events $A$ and $B$ are {\bf{independent}} if the occurence of one does not influence the probability of the other. This can be expressed as $P(A \cap B) = P(A) P(B)$. Similarly, two random variables $X$ and $Y$ are independent if $p(x, y) = p(x)p(y)$.\\

\noindent  The {\bf{expected value}} (or {\bf{expectation, mean}}) $\E[X]$  of a random variable $X$ can be thought of as the ``weighted average'' of the possible outcomes of the random variable. It is often represented by $\mu$. \\

\noindent For discrete random variables:
\begin{align*}
  \E[X] &= \sum_{x \in \mathcal{X}} x \cdot p(x) \\
  \E[f(X)] &= \sum_{x \in \mathcal{X}} f(x) p(x)
\end{align*}
For continuous random variables:
\begin{align*}
  \E[X] &= \int_{\mathcal{X}} x \cdot p(x) dx \\
  \E[f(X)] &= \int_{\mathcal{X}} f(x) p(x) dx
\end{align*}

\bigskip

\noindent One of the important properties of expected values is the \textbf{linearity of expectation}. For any two random variables $X$ and $Y$, scaling coefficients $a$ and $b$, and some constant $c$, the following property of holds:
$$\E[aX + bY + c] = a\E[X] + b\E[Y] + c$$
This is easy to show from the definition above, and is true regardless of whether $X$ and $Y$ are dependent or independent. \\

\noindent The {\bf{variance}} of a random variable is its expected squared deviation from its mean:
$$\var(X) = \E[ (X - \E[X])^2 ] = \E[X^2] - (\E[X])^2$$

\noindent The {\bf{standard deviation}} of a random variable is the square root of the variance:
$$\sigma(X) = \sqrt{\var(X)}$$

\noindent Now consider probability distributions over multiple variables. Let $X$ and $Y$ be two random variables (for instance, each can correspond to the rolls of two regular 6-sided dice). Then, the probability of an event that depends on both $X$ and $Y$ (for example, the event that $X == 1$ and $Y == 2$) will be given by a value in the {\bf{joint probability distribution}} $p(x, y)$. \\

\noindent The {\bf{marginal probability distribution}} is the probability distribution of a subset of variables from a {\bf{joint probability distribution}}. For example, the marginal probability distribution $p(y)$ can be found by summing across all values of $x$ in $p(x, y)$:
\begin{align*}
p(x) = \sum_{y \in \mathcal{Y}} p(x, y) \\
p(x) = \int_{\mathcal{Y}} p(x,y) dy
\end{align*}

\noindent When we know that an event $B$ has happened, that could influence the probability of another event $A$. The new probability of $A$ given $B$ is the {\bf{conditional probability}} $P(A|B)$. If $B$ changes the distribution of a random variable $X$, we write the new random variable as $X|B$, and the new distribution $P(X|B)$ is the conditional distribution. \\

\noindent The conditional probability can be defined as:
$$ P(A | B) = \frac{P(A \;\; \text{and} \;\; B)}{P(B)}$$

\noindent Note that this has some direct implications: for instance, if $A$ is the event $X == x$ and $B$ is the event $Y == y$, we have an expression relating distributions (sometimes called the {\bf{product rule}}):
$$ p(x | y) = \frac{p(x, y)}{p(y)}$$
which implies that
$$ p(x, y) = p(x | y) p(y) = p(y | x) p(x)$$

\noindent Remember recursion? If there are more variables in the distribution, the above statement can be recursively extended:
\begin{align*}
p(x_1,...,x_n)&= p(x_1|x_2,...,x_n)p(x_2,...,x_n)\\
&= ... = p(x_1|x_2,...,x_n)...p(x_{n-1}|x_n)p(x_n)
\end{align*}

\noindent Finally, you may have heard of {\bf{Bayes' Theorem}} (also known as {\bf{Bayes' Rule}} or {\bf{Bayes' Law}}), a very prominent statement that relates conditional probabilities between events. For any two events $A$ and $B$ such that $P(B) > 0$,
$$ P(A | B) = \frac{P(B | A) P(A)}{P(B)} $$
As before, if we set events $A$ to be $X == x$ and $B$ to be $Y == y$, we get another common form of Bayes' Law:
$$ p(x | y) = \frac{p(y | x) p(x)}{p(y)} $$ \\
In machine learning, we are often looking for parameters $\theta$ based on the distribution of data $y$. Then, we interpret $p(\theta)$ as the {\bf{prior}} (the distribution of $\theta$ without knowing about $y$), $p(y)$ as the {\bf{evidence}} (the overall probability of this data without considering our parameters), $p( y | \theta)$ as the {\bf{likelihood}} (how likely are we to collect this data given the parameters?) and $p( \theta | y)$ as the {\bf{posterior}} (the distribution of $\theta$ after knowing about $y$).
\begin{align*}
\underbrace{p(\theta | y)}_{\text{posterior}}
&= \frac{\overbrace{ p(y | \theta) }^{\text{likelihood}}
	\overbrace{ p(\theta) }^{\text{prior}}}
{\underbrace{p(y)}_{\text{evidence}}}
\end{align*}

%\begin{align*}
%p(x | y) &= \frac{p(y | x) p(x)}{p(y)} \\
%&= \frac{p(y | x) p(x)}{ \int_{\mathcal{X}} p(x,y) dx}
%\end{align*}

%\noindent Note that $\E[X|Y]$ is a random variable (this is one form of $f(Y)$). Adam's law (law of iterated expectations) gives
%\begin{align*}
%\E[X]=\E[\E[X|Y]]
%\end{align*}
%There is an analogous property for variances (Eve's Law, or law of total variance)
%\begin{align*}
%\var[X] = \E[ \var[X|Y] ] + \var[ \E[X | Y] ]
%\end{align*}

\section{Linear Algebra}

We write a {\bf{vector}} as
$$  \mathbf{x} = (x_1, \dots, x_D)^\top $$
where $D$ is the dimension of the vector, and $x_1, \ldots, x_D$ are elements of the vector. We use column vectors by default. \\

\noindent We may commonly write $\mathbf{x} \in \R^D$. The symbol "$\R$" refers to the space of real numbers, "$\R^D$" refers to the D-dimensional space of real numbers, and the symbol "$\in$" means "in", so this is a concise way to say that $x$ is in the D-dimensional space of real numbers. \\

\noindent The size of a vector is measured with the vector norm. Usually, we are referring to the function:
\[
||x||_2 = \sqrt{x_1^2 + x_2^2 + \dots + x_D^2}
\]
We call this the 2-norm or $\L^2$-norm, and it is actually just one of many $\L^p$ norms. You may also see the 1-norm or the infinity-norm:
\begin{align*}
&||x||_1 = |x_1| + |x_2| + \dots |x_D| \\
&||x||_{\infty} = \max (|x_1|, |x_2|, \dots, |x_D|)
\end{align*}

\noindent An $n \times m$ {\bf{matrix}} $\mathbf{A}$ has $n$ rows and $m$ columns:
\[
\left(\begin{array}{c c c c}
a_{1, 1} & a_{1, 2} & \dots & a_{1, m} \\
a_{2, 1} & a_{2, 2} & \dots & a_{2, m} \\
\vdots & \vdots & \ddots & \vdots \\
a_{n, 1} & a_{n, 2} & \dots & a_{n, m} \\
\end{array}\right)
\]

\noindent An $n \times m$ matrix can be thought of as a linear transformation from $\R^n$ to $\R^m$, and, similarly to the vectors, we write $A \in \R^{n \times m}$.

\section{Multivariate Calculus}
%Often we are interested in functions $f: \mathcal{X} \rightarrow \mathcal{Y}$ and their derivatives to fit a model from observations.  The most salient is the objective function
%\begin{align*}
%  E_D(\mathbf{w}) &= \sum_{n=1}^N (t_n - \mathbf{w}^\trans \mathbf{x}_n)^2
%\end{align*}
%This section will focus on functions from many inputs (e.g. $\mathcal{X} = \R^D$ for some large $D$). We will consider $\mathcal{Y} = \R^M$, but the case of $M=1$ will be the most common and useful.

To review mathematical notation, we may write a function $f$ as
$$f: \mathcal{X} \in R^n \to \mathcal{Y} \in R^m$$
This means that the function's {\bf{domain}}, or set of legals inputs, is a set $\mathcal{X}$ of real-valued $n$-dimensional vectors, and the function returns a real-valued $m$-dimensional vector in the output set (or {\bf{range}}) $\mathcal{Y}$. \\

\noindent Recall the rules for {\bf{differentiation}} of a function with respect to one variable:
\begin{align*}
\text{Chain rule: } & \frac{d}{d x} f(g(x))= f'(g(x))g'(x)\\
\text{Product rule: }& \frac{d}{d x} f(x)g(x) = f'(x)g(x) + f(x)g'(x)\\
\text{Quotient rule: }& \frac{d}{d x} \frac{f(x)}{g(x)} = \frac{f'(x)g(x) - f(x)g'(x)}{g(x)^2}
\end{align*}

\noindent When calculating partial derivatives of a function of multiple variables, we write the {\bf{gradient vector}} as
\begin{align*}
  \nabla f(\mathbf{x}) &= \left( \frac{\p f(\mathbf{x})}{\p x_1}, \dots, \frac{\p f(\mathbf{x})}{\p x_D} \right)^\top
\end{align*}
Sometimes we only wish to differentiate with respect to some variables in the input. For example, $f(\mathbf{x},\alpha)$, then the derivative with respect to $\mathbf{x}$ is denoted
\begin{align*}
  \nabla_{\mathbf{x}} f(\mathbf{x},\alpha) = \frac{d f(\mathbf{x},\alpha)}{d \mathbf{x}} = \left( \frac{\p f(\mathbf{x},\alpha)}{\p x_1}, \dots, \frac{\p f(\mathbf{x},\alpha)}{\p x_D} \right)^\top
\end{align*}
The gradient vector points towards the direction of greatest ascent in $f(\mathbf{x})$ with respect to the parameters being differentiated. \\

\noindent If the function we are interested in has multiple outputs (it maps a vector $[x_1, \dots, x_m]$ to a vector $[f_1, \dots, f_n]$), we can generalize the gradient vector to the {\bf{Jacobian matrix}}:
\begin{align*}
\frac{d \mathbf{f}(\mathbf{x})}{d \mathbf{x}} = \begin{bmatrix}
\frac{\p f_1(\mathbf{x})}{\p x_1} & \cdots & \frac{\p f_1(\mathbf{x})}{\p x_m} \\
\vdots & \ddots & \vdots \\
\frac{\p f_n(\mathbf{x})}{\p x_1} & \cdots & \frac{\p f_n(\mathbf{x})}{\p x_m}
\end{bmatrix}
\end{align*}

\noindent For a function that has a single output $f(\mathbf{x})$, the {\bf{Hessian matrix}} is the generalization of the second derivative:
\begin{align*}
H(f(\mathbf{x}))=\begin{bmatrix}
\frac{\p^2 f(\mathbf{x})}{\p x_1^2} & \frac{\p^2 f(\mathbf{x})}{\p x_1 \p x_2} & \cdots &  \frac{\p^2 f(\mathbf{x})}{\p x_1 \p x_n}\\
\frac{\p^2 f(\mathbf{x})}{\p x_2 \p x_1} & \frac{\p^2 f(\mathbf{x})}{\p x_2^2} & \cdots & \frac{\p^2 f(\mathbf{x})}{\p x_1 \p x_n}\\
\vdots & \vdots & \ddots & \vdots \\
\frac{\p^2 f(\mathbf{x})}{\p x_n \p x_1} & \frac{\p^2 f(\mathbf{x})}{\p x_n \p x_2} & \cdots & \frac{\p^2 f(\mathbf{x})}{\p x_n^2}\\
\end{bmatrix}
\end{align*}

\noindent To find the local minima (or maxima) of a function $f(\mathbf{x})$, we can set the gradient equal to zero: $\nabla f(\mathbf{x})=0$. However, we may not always be able to find a closed-form solution. {\bf{Gradient descent}} is a numerical way to solve this problem. We start with an initial guess $\mathbf{x}_0$, and iteratively take small steps in the direction of greatest descent until we reach a point where the gradient is close to zero. Since this direction is exactly opposite the direction of the gradient vector, the "update" of our guess at each iteration is
\begin{align*}
\mathbf{x}_{i+1} = \mathbf{x}_i - \gamma \nabla f(\mathbf{x}_i)
\end{align*}
where $\gamma$ determines our step size. This algorithm forms the basis for much of optimization -- many of the common algorithms used within machine learning and other fields are variants and extensions of gradient descent. \\

%This matrix is symmetric and we can diagonalize it to find its eigenvalues. If all eigenvalues are positive, then we are at a local minimum. If all eigenvalues are negative, then we are at a local maximum. If there is a mix of positive and negative eigenvalues, then we are at a saddle point. If there are zero-eigenvalues, then there are directions in which $f(\mathbf{w})$ has no second order change. Besides allowing us to deduce the nature of turning points, the inverse of the Hessian matrix can also be used to aid gradient descent by using the update rule
%\begin{align*}
%\mathbf{x}_{i+1} = \mathbf{x}_i - [H(f(\mathbf{x}))]^{-1} \nabla f(\mathbf{x}_i)
%\end{align*}

%While we will not dive into optimization in this class, some of these concepts may be mentioned.

%A single output function $f(\mathbf{x})$ is convex if it satisfies Jensen's inequality
%\begin{align*}
%f\left( \sum_{n=1}^N a_n \mathbf{x}_n\right) \leq  \sum_{n=1}^N a_n f\left( \mathbf{x}_n\right)
%\end{align*}
%for all $\mathbf{x}_n\in \mathcal{X}$ and $\sum_{n=1}^N a_n=1$. Similarly, $f(\mathbf{x})$ is concave if the sign of the inequality is flipped. $f(\mathbf{x})$ is called strongly convex/concave if the inequalities are converted to strict inequalities. \\
%Strongly convex and concave functions have the benefit of having global minimums and maximums respectively, and so are easy to do gradient descent on.

\end{document}


\newpage
	\section*{}
	\textbf{Task}\\
	Let $f_n$ be the Fibonacci sequence. Prove that
	\begin{gather*}
		f_{n-1} f_{n+1}-f_{n}^{2}=(-1)^{n}
	\end{gather*}
	\\
	\textbf{Solution}\\
	Binet's formula:
	\begin{gather*}
		f_{n} =
		\frac
		{\left(\frac{1+\sqrt{5}}{2}\right)^{n} - \left(\frac{1-\sqrt{5}}{2}\right)^{n}}
		{\sqrt{5}}
	\end{gather*}
	Then
	\begin{gather*}
		f_{n-1} f_{n+1}-f_{n}^{2} =
		\\
		\\
		\frac{\left(\frac{1+\sqrt{5}}{2}\right)^{n-1} - \left(\frac{1-\sqrt{5}}{2}\right)^{n-1}}{\sqrt{5}}
		\cdot
		\frac{\left(\frac{1+\sqrt{5}}{2}\right)^{n+1} - \left(\frac{1-\sqrt{5}}{2}\right)^{n+1}}{\sqrt{5}}
		-
		\left(\frac{\left(\frac{1+\sqrt{5}}{2}\right)^{n} - \left(\frac{1-\sqrt{5}}{2}\right)^{n}}{\sqrt{5}}\right)^2 =
		\\
		\\
		\frac{1}{5}
		\left(
			\left(
				\left(\frac{1+\sqrt{5}}{2}\right)^{n-1} - \left(\frac{1-\sqrt{5}}{2}\right)^{n-1}
			\right)
			\cdot
			\left(
				\left(\frac{1+\sqrt{5}}{2}\right)^{n+1} - \left(\frac{1-\sqrt{5}}{2}\right)^{n+1}
			\right)
			-
			\left(
				\left(\frac{1+\sqrt{5}}{2}\right)^{n} - \left(\frac{1-\sqrt{5}}{2}\right)^{n}
			\right)^2
		\right) =
		\\
		\\
		\frac{1}{2^{2n} \cdot 5}
		\left(
			\left(
				(1+\sqrt{5})^{n-1} - (1-\sqrt{5})^{n-1}
			\right)
			\cdot
			\left(
				(1+\sqrt{5})^{n+1} - (1-\sqrt{5})^{n+1}
			\right)
			-
			\left(
				(1+\sqrt{5})^{n} - (1-\sqrt{5})^{n}
			\right)^2
		\right) =
		\\
		\\
		\frac{1}{2^{2n} \cdot 5}
		\Big(
			\left(
				(1+\sqrt{5})^{2n} + (1-\sqrt{5})^{2n} - (1+\sqrt{5})^{n-1}(1-\sqrt{5})^{n+1} - (1+\sqrt{5})^{n+1}(1-\sqrt{5})^{n-1}
			\right)
			-\\
			\left(
				(1+\sqrt{5})^{2n} + (1-\sqrt{5})^{2n} - 2(1+\sqrt{5})^{n}(1-\sqrt{5})^{n}
			\right)
		\Big) =
		\\
		\\
		\frac{1}{2^{2n} \cdot 5}
		\Big(
			(1+\sqrt{5})^{2n}
			+ (1-\sqrt{5})^{2n}
			- (1+\sqrt{5})^{n-1}(1-\sqrt{5})^{n+1}\\
			- (1+\sqrt{5})^{n+1}(1-\sqrt{5})^{n-1}
			- (1+\sqrt{5})^{2n}
			- (1-\sqrt{5})^{2n}
			+ 2(1+\sqrt{5})^{n}(1-\sqrt{5})^{n}
		\Big) =
		\\
		\\
		\frac{1}{2^{2n} \cdot 5}
		\Big(
			2(1+\sqrt{5})^{n}(1-\sqrt{5})^{n}
			- (1+\sqrt{5})^{n-1}(1-\sqrt{5})^{n+1}
			- (1+\sqrt{5})^{n+1}(1-\sqrt{5})^{n-1}
		\Big) =
		\\
		\\
		\frac{(1 - 5)^{n-1}}{2^{2n} \cdot 5}
		\Big(
			2 (1-\sqrt{5})(1+\sqrt{5})
			- (1-\sqrt{5})^{2}
			- (1+\sqrt{5})^{2}
		\Big) =
		\\
		\\
		\frac{(-4)^{n-1}}{4^{n} \cdot 5}
		\Big(
			2 (1-\sqrt{5})(1+\sqrt{5})
			- (6 - 2\sqrt{5})
			- (6 + 2\sqrt{5})
		\Big) =
		\\
		\\
		\frac{(-4)^{n-1}}{4^{n} \cdot 5}
		\Big(
			-8
			- 6
			- 6
		\Big) =
		\\
		\\
		\frac{(-4)^{n-1}}{4^{n} \cdot 5} (-20) =
		(-1)^{n}
	\end{gather*}


\subsection{Line integral of vector fields}

We may wish to integrate along a curve in a vector field.

We previously showed that we can write a curve as a function on the real line:

\(r:[a,b]\rightarrow C\)

The integral is therefore the sum of the function at all points, with some weighting. We write this:

\(\int_C f(r) ds=\lim_{\Delta s rightarrow 0 }\sum_{i=0}^n f(r(t_i))\Delta s_i\)

In a vector field we use

\(\int_C f(r) ds =\int_a^b f(r(t)).r'(t) dt\)


% based on example 8 in pythontex_gallery
% https://github.com/gpoore/pythontex/

\documentclass[12pt]{pylatex}
\usepackage{examples}

\begin{document}

\section*{Step-by-step integration}

This is another nice example drawn from the Pythontex gallery, see
\ \url{https://github.com/gpoore/pythontex}.

It shows the step-by-step computations of a simple triple integral.

\begin{python}
   from sympy import *

   x, y, z = symbols('x,y,z')
   f = Symbol('f(x,y,z)')

   # Define limits of integration
   x_max = 2;   y_max = 3;   z_max = 4;
   x_min = 0;   y_min = 0;   z_min = 0;

   lhs = Integral(f, (x, x_min, x_max),
                     (y, y_min, y_max),
                     (z, z_min, z_max))                   # py(lhs.01,lhs)

   f = x*y + y*sin(z) + cos(x+y)

   rhs = Integral(f, (x, x_min, x_max),
                     (y, y_min, y_max),
                     (z, z_min, z_max))                   # py(rhs.01,rhs)
   rhs = Integral(Integral(f, (x, x_min, x_max)).doit(),
                              (y, y_min, y_max),
                              (z, z_min, z_max))          # py(rhs.02,rhs)
   rhs = Integral(Integral(f, (x, x_min, x_max),
                              (y, y_min, y_max)).doit(),
                              (z, z_min, z_max))          # py(rhs.03,rhs)
   rhs = Integral(f, (x, x_min, x_max),
                     (y, y_min, y_max),
                     (z, z_min, z_max)).doit()            # py(rhs.04,rhs)

   # And now, a numerical approximation
   rhs = N(rhs)                                           # py(rhs.05,rhs)

\end{python}

\begin{minipage}[t]{0.65\textwidth}
\begin{align*}
   \py{lhs.01} &= \py{rhs.01}\\
               &= \py{rhs.02}\\
               &= \py{rhs.03}\\
               &= \py{rhs.04}\\[8pt]
               &\approx \py{rhs.05}
\end{align*}
\end{minipage}
\hskip 1cm
\lower16pt\hbox{%
\begin{minipage}[t]{0.35\textwidth}
\begin{latex}
   \begin{align*}
      \py{lhs.01} &= \py{rhs.01}\\
                  &= \py{rhs.02}\\
                  &= \py{rhs.03}\\
                  &= \py{rhs.04}\\[8pt]
                  &\approx \py{rhs.05}
   \end{align*}
\end{latex}
\end{minipage}}

\end{document}


\section{Algebra}
Absolute Value Inequalities:\\
$ |f(x)| < a  \Rightarrow  -a < f(x) < a$\\
$|f(x)| > a  \Rightarrow f(x) > a$ or $f(x) < -a$\\
\section{Matrixalgebra}

$\displaystyle  \displaystyle \left\lVert \mathbf{A}\mathbf{x}\right\rVert ^2 = \displaystyle (\mathbf{A}\mathbf{x})^ T(\mathbf{A}\mathbf{x})\, =\, \mathbf{x}^ T\mathbf{A}^ T\mathbf{A}\mathbf{x} = \displaystyle \mathbf{x}^ T\mathbf{A}^ T\mathbf{A}\mathbf{x}\qquad$

\section{Calculus}

Differentiation under the integral sign\\
$\frac{\text{d}}{\text{d}x}\left( \int_{a(x)}^{b(x)}f(x,t)\text{d}t \right ) = f(x,b(x))b'(x)-f(x,a(x))a'(x)+\int_{a(x)}^{b(x)}f_x(x,t)\text{d}t.$

\subsection*{Concavity in 1 dimension}
If $g:I \to \mathbb {R}$ is twice differentiable in the interval $I$:

concave:\\
if and only if $g^{\prime \prime }(x) {\color{blue}{\leq }}  0$ for all $x \in I$\\

strictly concave:\\
if $g^{\prime \prime }(x) {\color{blue}{<}}  0$ for all $x \in I$\\

convex:\\
if and only if $g^{\prime \prime }(x) {\color{blue}{\geq }}  0$ for all $x \in I$\\

strictly convex if:\\
$g^{\prime \prime }(x) {\color{blue}{>}}  0$ for all $x \in I$\\

\subsection*{Multivariate Calculus}
The Gradient $\nabla$ of a twice differntiable function $f$ is defined as:

$\nabla f:\mathbb {R}^ d \rightarrow \mathbb {R}^ d $\\
$\displaystyle \theta =\begin{pmatrix} \theta _1\\ \theta _2\\ \vdots \\ \theta _ d\end{pmatrix} \displaystyle \mapsto \displaystyle \left.\begin{pmatrix}  \frac{\partial f }{\partial \theta _1}\\ \frac{\partial f }{\partial \theta _2}\\ \vdots \\ \frac{\partial f }{\partial \theta _ d}\end{pmatrix}\right|_{\theta }$\\
\textbf{Hessian}\\

The Hessian of $f$ is a symmetric matrix of second partial derivatives of $f$\\

$\mathbf{H} h(\theta )= \nabla ^{2} h(\theta )= \\
\left(\begin{array}{ c c c }
\frac{\partial ^{2} h}{\partial \theta _{1} \partial \theta _{1}} (\theta ) & \cdots  & \frac{\partial ^{2} h}{\partial \theta _{1} \partial \theta _{d}} (\theta )\\
 & \vdots  & \\
\frac{\partial ^{2} h}{\partial \theta _{d} \partial \theta _{1}} (\theta ) & \cdots  & \frac{\partial ^{2} h}{\partial \theta _{d} \partial \theta _{d}} (\theta )
\end{array}\right) \in \mathbb{R}^{d\times d}$\\

A symmetric (real-valued) $d\times d$  matrix $\mathbf{A}$ is:\\

Positive semi-definite:\\
$\mathbf{x}^ T \, \mathbf{A}\, \mathbf{x} \geq 0\qquad \text {for all }\,  \mathbf{x}\in \mathbb {R}^ d.$\\

Positive definite:\\
$\mathbf{x}^ T \, \mathbf{A}\, \mathbf{x}> 0$ for all non-zero vectors $\mathbf{x}\in \mathbb {R}^ d$\\

Negative semi-definite (resp. negative definite):\\

$\mathbf{x}^ T \, \mathbf{A}\, \mathbf{x}$ is negative for all $\mathbf{x}\in \mathbb {R}^ d-\{ \mathbf{0}\}$.\\

Positive (or negative) definiteness implies positive (or negative) semi-definiteness.\\

If the Hessian is positive definite then $f$ attains a local minimum at $a$ (convex).\\

If the Hessian is negative definite at $a$, then f attains a local maximum at $a$ (concave).\\

If the Hessian has both positive and negative eigenvalues then $a$ is a saddle point for $f$.

%\subsection*{Lagrange Multiplier}


\subsection{Writing first-order logic}

\subsubsection{Existential quantifier}

We introduce a shorthand for “at least one term satisfies a predicate”, that is:

\(P(x_0)\lor P(x_1)\lor P(x_2)\lor P(x_2)\lor P(x_3)...\)

The short hand is:

\(\exists x P(x)\)

\subsubsection{niversal quantifier}

We introduce another shorthand, this time for:

\(P(x_0)\land P(x_1)\land P(x_2)\land P(x_2)\land P(x_3)...\)

The shorthand is

\(\forall x P(x)\)

\subsubsection{Free and bound variables}

A bound variable is one which is quantified in the formula. A free variable is one which is not. Consider:

\(\forall x P(x,y)\)

In this, \(x\) is bound while \(y\) is free.

Free variables can be interpreted differently, while bound variables cannot.

We can also bind a specific variable to a value. For example \(0\) can be defined to be bound.

\subsubsection{Ground terms}

A ground term does not contain any free variables. A ground formula is one which only includes ground terms.

\(\forall x\ x\) is a ground term.

\(\forall x P(x)\) is a ground formula.


\input{../header.tex}
\title{\vspace{-2cm}INF3490/INF4490 Exercise Solutions - Simple search algorithms}
\author{Ole Herman S. Elgesem\input{../author_footnote.tex}}
\date{}

% Removing paragraph indents is sometimes useful:
\setlength\parindent{0pt}
% ==============================================================================

% ================================= DOCUMENT ===================================
\begin{document}
    \renewcommand\marginsymbol[1][0pt]{%
  \tabto*{0cm}\makebox[-1cm][c]{$\mathbb{P}$}\tabto*{\TabPrevPos}}

\maketitle
\input{../intro.tex}

\section{Simple search algorithms}

Given the function \(f(x) = -x^4 + 2x^3 + 2x^2 - x\):

\subsection{Derivative}
What is its derivative \(f'(x)\) ?\\

\textit{Answer:}
\[f'(x) = -4x^3 + 6x^2 + 4x - 1\]

\subsection{Plotting \marginsymbol}
Plot the function, and its gradient(derivative) from \(x=-2\) to \(x=3\).
Use python, wolfram alpha or another plotting tool of your choice.\\

\textit{Plot:}
\begin{figure}[H]
\begin{center}
\includegraphics[width=0.6\textwidth]{eps/w1e1b.eps}
\caption{\(f(x)\) and it's derivative.}
\label{fig:w1e1b}
\end{center}
\end{figure}

\textit{Source code (Python 3):}
\inputminted{Python}{py/w1e1b.py}

\subsection{Gradient Ascent \marginsymbol}
\label{subsec:grada}
Maximize using gradient ascent.
You can try step size 0.1 and start somewhere in the range [-2, 3].
How does the choice of starting point and step size affect the algorithm's performance?
Is there a starting point where the algorithm would not even be able to find a local maximum?\\

\textit{Answer:}

Both starting position and step size affects where the algorithm ends:
\begin{itemize}
    \item Starting Position
    \begin{itemize}
        \item \textit{Left side:} Should converge on left maximum
        \item \textit{Center:} Stops immediately, gradient is zero.
        \item \textit{Right side:} Should converge on right maximum
    \end{itemize}
    \item Step Size
    \begin{itemize}
        \item \textit{Too low:} Converges slowly (poor performance)
        \item \textit{Too high:} Overshoot, bounce over solutions. Doesn't converge, might not terminate.
    \end{itemize}
\end{itemize}

\textit{Plot:}
\begin{figure}[H]
\begin{center}
\includegraphics[width=0.6\textwidth]{eps/w1e1c.eps}
\caption{Result of gradient ascent}
\label{fig:w1e1c}
\end{center}
\end{figure}

\textit{Source code (Python 3):}
\inputminted{Python}{py/w1e1c.py}

\subsection{Exhaustive Search \marginsymbol}
\label{subsec:exhaust}
Assume that we are only interested in maxima of \(f(x)\) where
\(-2\leq x \leq 3\), and \(x\) increases in steps of length 0.5.
(\(\Delta x = 0.5\)). Perform an exhaustive search to
maximize \(f(x)\) and plot the result.\\

\textit{Plot:}
\begin{figure}[H]
\begin{center}
\includegraphics[width=0.6\textwidth]{eps/w1e1d.eps}
\caption{Result of exhaustive search}
\label{fig:w1e1c}
\end{center}
\end{figure}

\textit{Source code (Python 3):}
\inputminted{Python}{py/w1e1d.py}

\subsection{Greedy Search and Hill Climbing}
In what way would greedy search and hill climbing differ for the maximization problem in Problem \ref{subsec:grada}?
Can you identify a starting position where the two algorithms might give different results?\\

\textit{Answer:}

Greedy search will check all neighbors and always go in most promising direction.
Hill climber will randomly find a decent neighbor and walk there.
If both algorithms started in center ( \(x = 0.5\) ),
greedy search always go right,
while hill climber could go right and left ( 50/50 ) as both are \emph{up}.

\subsection{Possible improvements}
Gradient ascent, greedy search and hill climbing are quite similar,
and are all based almost exclusively on exploitation.
Can you think of any additions to these algorithms in order to do more exploration?\\

\textit{Answer:}

Run the algorithm several times with random starting positions,
this will \emph{explore} the solution space and find several local optima.
Another option is to add more random movement to either algorithm.
This can be done after a solution is found, or at a probability while searching.
Could also do backtrack + random jump after a solution is found.

\subsection{Exhaustive search vs. simulated annealing}
Which algorithm do you think is the most efficient at maximizing \(f(x)\) under the conditions in Problem \ref{subsec:exhaust}: exhaustive search or simulated annealing?
Explain.\\

\textit{Answer:}

Exhaustive search is better for this case.
It takes few iterations to go through the one dimensional solution space.
For problems in higher dimensions or more complex functions with smaller step size exhaustive search becomes impractical and simulated annealing will outperform.

\input{../contact.tex}
\end{document}
% ==============================================================================


\section{Linear Regression}
Linear Regression seeks to approximate a real valued label $y$ as a
linear function of $x$:
\begin{equation}
  h_{\theta}(x) = \theta_0 + \theta_1 \cdot x_1 + \cdots + \theta_n \cdot x_n
\end{equation}
The $\theta_i$'s are the parameters, or weights. If we include the
intercept term via $x_0=1$, we can write our model more compactly as:
\begin{equation}
  h(x) = \sum_{i=0}^{n} \theta_i \cdot x_i = \theta^T x
\end{equation}
Here $n$ is the number of input variables, or features. In Linear Regression,
we seek to make $h(x)$ as close to $y$ for a set of training examples. We
define the cost function as:
\begin{equation}
  J(\theta) = \frac{1}{2} \sum_{i=1}^{m} \left( h \left(x^{(i)}\right) - y^{(i)}\right)^2
\end{equation}
\subsection{LMS Algorithm}
We seek to find a set of $\theta$ such that we minimize $J(\theta)$ via a
search algorithm that starts at some initial guess for our parameters and takes
incremental steps to make $J(\theta)$ smaller until convergence. This is know
as gradient descent:
\begin{equation}
  \theta_j := \theta_j - \alpha \frac{\partial}{\partial \theta_j}J(\theta)
\end{equation}
Here, $\alpha$ is the learning rate. We can derive the partial derivative as:
\begin{equation}
  \begin{split}
    \frac{\partial}{\partial \theta_{j}}J(\theta) \quad =& \quad \frac{\partial}{\partial \theta_{j}}\frac{1}{2} \left( h(x)-y \right)^{2} \\
    =& \quad 2\cdot \frac{1}{2} \left(h(x) - y \right) \cdot \frac{\partial}{\partial \theta_{j}} (h(x) - y) \\
    =& \quad \left(h(x) - y \right) \cdot \frac{\partial}{\partial \theta_{j}} \left(\sum_{i=0}^{n} \theta_{i} x_{i} - y \right) \\
    =& \quad \left( h ( x ) - y \right) x _ { j } \\
\end{split}
\end{equation}
Hence, for a single example (stochastic gradient descent):
\begin{equation}
  \theta _ { j } : = \theta _ { j } + \alpha \left( y ^ { ( i ) } - h \left( x ^ { ( i ) } \right) \right) x _ { j } ^ { ( i ) }
\end{equation}
This is called the LMS update rule. For a batched version, we can evaluate the
gradient on a set of examples (batch gradient descent), or the full set
(gradient descent).
\begin{equation}
  \theta _ { j } : = \theta _ { j } + \alpha \sum _ { i = 1 } ^ { m } \left( y ^ { ( i ) } - h \left( x ^ { ( i ) } \right) \right) x _ { j } ^ { ( i ) }
\end{equation}
\subsection{The Normal Equations}
We can also directly minimize $J$ without using an iterative algorithm. We
define $X$ as the matrix of all samples of size $m$ by $n$.
We let $\vec{y}$ be a $m$ dimensional vector of all target values. We can
define our cost function $J$ as:
\begin{equation}
  J(\theta) = \frac { 1 } { 2 } ( X \theta - \vec { y } ) ^ { T } ( X \theta - \vec { y } ) = \frac { 1 } { 2 } \sum _ { i = 1 } ^ { m } \left( h \left( x ^ { ( i ) } \right) - y ^ { ( i ) } \right) ^ { 2 }
\end{equation}
We then take the derivative and find its roots.

\begin{equation}
  \begin{split}
    \nabla _ { \theta } J ( \theta ) \quad =& \quad \nabla _ { \theta } \frac { 1 } { 2 } ( X \theta - \vec { y } ) ^ { T } ( X \theta - \vec { y } ) \\
    =& \quad \frac { 1 } { 2 } \nabla _ { \theta } \left( \theta ^ { T } X ^ { T } X \theta - \theta ^ { T } X ^ { T } \vec { y } - \vec { y } ^ { T } X \theta + \vec { y } ^ { T } \vec { y } \right) \\
    =& \quad \frac { 1 } { 2 } \nabla _ { \theta } \left( \operatorname { tr } \theta ^ { T } X ^ { T } X \theta - 2 \operatorname { tr } \vec { y } ^ { T } X \theta \right) \\
    =& \quad \frac { 1 } { 2 } \left( X ^ { T } X \theta + X ^ { T } X \theta - 2 X ^ { T } \vec { y } \right) \\
    =& \quad X ^ { T } X \theta - X ^ { T } \vec { y } \\
  \end{split}
\end{equation}
To minimize $J$, we set its derivatives to zero, and obtain the normal
equations:
\begin{equation}
  X ^ { T } X \theta = X ^ { T } \vec { y }
\end{equation}
Which solves $\theta$ for a value that minimizes $J(\theta)$ in closed form:
\begin{equation}
  \theta = \left( X ^ { T } X \right) ^ { - 1 } X ^ { T } \vec { y }
\end{equation}
\subsection{Probabilistic Interpretation}
Why does linear regression use the least-squares cost function? Assume that
the target variables and inputs are related via:
\begin{equation}
  y ^ { ( i ) } = \theta ^ { T } x ^ { ( i ) } + \epsilon ^ { ( i ) }
\end{equation}
Here, $\epsilon^{(i)}$ is an error term for noise. We assume each
$\epsilon^{(i)}$ is independently and identically distributed according to a
Gaussian distribution with mean zero and some variance $\sigma^2$. Hence,
$\epsilon^{(i)} \sim \mathcal { N } \left( 0 , \sigma ^ { 2 } \right)$, so
the density for any sample $x^{(i)}$ with label $y^{(i)}$ is
$y^{(i)} | x^{(i)}; \theta \sim \mathcal{N} \left( \theta^T x^{(i)}, \sigma^{2} \right)$.
This implies:
\begin{equation}
  p \left( y ^ { ( i ) } | x ^ { ( i ) } ; \theta \right) = \frac { 1 } { \sqrt { 2 \pi } \sigma } \exp \left( - \frac { \left( y ^ { ( i ) } - \theta ^ { T } x ^ { ( i ) } \right) ^ { 2 } } { 2 \sigma ^ { 2 } } \right)
\end{equation}
The probability of a dataset $X$ is quantified by a likelihood function:
\begin{equation}
  L ( \theta ) = L ( \theta ; X , \vec { y } ) = p ( \vec { y } | X ; \theta )
\end{equation}
Since we assume independence on each noise term (and samples), we can write
the likelihood function as:
\begin{equation}
  \begin{aligned}
    L ( \theta ) & = \prod _ { i = 1 } ^ { m } p \left( y ^ { ( i ) } | x ^ { ( i ) } ; \theta \right) \\
    & = \prod _ { i = 1 } ^ { m } \frac { 1 } { \sqrt { 2 \pi } \sigma } \exp \left( - \frac { \left( y ^ { ( i ) } - \theta ^ { T } x ^ { ( i ) } \right) ^ { 2 } } { 2 \sigma ^ { 2 } } \right)
  \end{aligned}
\end{equation}
To get the best choice of parameters $\theta$, we perform maximum likelihood
estimation such that $L(\theta)$ is maximized. Usually we take the negative log
and minimize:
\begin{equation}
  \begin{aligned}
    \ell ( \theta ) & = - \log L ( \theta ) \\
    & = - \log \prod _ { i = 1 } ^ { m } \frac { 1 } { \sqrt { 2 \pi } \sigma } \exp \left( - \frac { \left( y ^ { ( i ) } - \theta ^ { T } x ^ { ( i ) } \right) ^ { 2 } } { 2 \sigma ^ { 2 } } \right) \\
    & = - \sum _ { i = 1 } ^ { m } \log \frac { 1 } { \sqrt { 2 \pi } \sigma } \exp \left( - \frac { \left( y ^ { ( i ) } - \theta ^ { T } x ^ { ( i ) } \right) ^ { 2 } } { 2 \sigma ^ { 2 } } \right) \\
    & = - m \log \frac { 1 } { \sqrt { 2 \pi } \sigma } + \frac { 1 } { \sigma ^ { 2 } } \cdot \frac { 1 } { 2 } \sum _ { i = 1 } ^ { m } \left( y ^ { ( i ) } - \theta ^ { T } x ^ { ( i ) } \right) ^ { 2 }
  \end{aligned}
\end{equation}
Hence, maximizing $L(\theta$) is the same as minimizing the negative log
likelihood $\ell(\theta)$, which for linear regression is the least squares
cost function:
\begin{equation}
  \frac { 1 } { 2 } \sum _ { i = 1 } ^ { m } \left( y ^ { ( i ) } - \theta ^ { T } x ^ { ( i ) } \right) ^ { 2 }
\end{equation}
Under the previous probabilistic assumptions on the data, least-squares
regression corresponds to finding the maximum likelihood estimate of $\theta$.
This is thus one set of assumptions under which least-squares regression
can be justified as performing maximum likelihood estimation. Note that $\theta$
is independent of $\sigma^2$.
\subsection{Locally Weighted Linear Regression}
Locally Weighted Regression, also known as LWR, is a variant of
linear regression that weights each training example in its cost function by
$w^{(i)}(x)$, which is defined with parameter $\tau \in \mathbb{R}$ as:
\begin{equation}
  w^{(i)}(x)=\exp\left(-\frac{(x^{(i)}-x)^2}{2\tau^2}\right)
\end{equation}
Hence, in LWR, we do the following:
\begin{enumerate}
  \item Fit $\theta$ to minimize $\sum_i w^{(i)} \left( y^{(i)} - \theta^{T} x^{(i)} \right)^{2}$
  \item Output $\theta^T x$
\end{enumerate}
This is a non-parametric algorithm, where non-parametric refers to the fact
that the amount of information we need to represent the hypothesis $h$ grows
linearly with the size of the training set.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS624: Analysis of Algorithms
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 3}

In a max-heap of size $n$, represented as discussed in class, in what index(es) can the smallest element reside? Explain carefully.
Assume all the $n$ numbers are different.

\subsection*{Solution}

Parent elements in a in a max-heap data structure are always larger than their children.
Thus, smallest element resides in indices of leaves, i.e. nodes of height $0$.
We know some leaves are at height $H$ and some at height $H-1$.
Since row $(H-1)$ is inevitably filled out, number of leaves in height $H$ is $n - (2^H - 1)$ and number of leaves in height $(H-1)$ is $(2^H-1)-\lfloor \frac{n}{2} \rfloor$.
Therefore, there are $\lceil \frac{n}{2} \rceil$ leaves in total so indices in which smallest element may reside are $A[n - \lceil \frac{n}{2} \rceil]$ to $A[n]$.


\subsection{Motivation}

Consider a system of linear equations in two variables, such as

\begin{align*}
y_1 &= ax_1 + bx_2 \\
y_2 &= cx_1 + dx_2.
\end{align*}

We would like to solve for $x_1$ and $x_2$ in terms of $y_1$ and $y_2$. Presumably, these solutions will have expressions involving $a$, $b$, $c$, and $d$. To solve this equation for $x_1$, we first eliminate the second variable by multiplying the first and second equations by appropriate constants, respectively:

\begin{align*}
dy_1 &= adx_1 + bdx_2 \\
by_2 &= bcx_1 + bdx_2.
\end{align*}

Subtracting the second equation from the first, and solving for $x_1$, we get
\[x_1 = \frac{dy_1 - by_2}{(ad-bc)}.\]
In a similar manner we can solve for $x_2$ as well.

Now consider an alternative perspective. We can rewrite the system of linear equations that we are considering as a matrix vector equation
\[\begin{bmatrix}y_1 \\ y_2 \end{bmatrix} = \begin{bmatrix}a & b \\ c & d\end{bmatrix}\begin{bmatrix}x_1 \\ x_2\end{bmatrix}.\]

Then in the language of determinants, we find that
\[x_1 = \frac{\det{ \begin{bmatrix}y_1 & b \\ y_2 & d\end{bmatrix}}}{\det \begin{bmatrix}a & b \\ c & d\end{bmatrix}}\] and
\[x_2 = \frac{\det{ \begin{bmatrix}a & y_1 \\ c& y_2\end{bmatrix}}}{\det \begin{bmatrix}a & b \\ c & d\end{bmatrix}}.\]

This is the $2$ dimensional case of \textit{Cramer's Rule}.


\subsection{Gradient Descent}
\noindent
Remember that if we have a multidimensional function, taking a step in the direction of the gradient results in the maximum possible increase of the function, and taking a step in the opposite direction of the gradient results in the maximum possible decrease of the function.
Gradient descent is a method to find minima of functions.\\

\noindent
Let's say we're trying to minimize $J(\vec{x})$ with gradient descent. Here are the steps we would take:
\begin{enumerate}
	\item Pick (or guess) a starting point $\vec{x_0}$ and a learning rate (step size) $\delta$.
	\item $\overrightarrow{x_{n+1}} = \overrightarrow{x_n} - \delta J(\overrightarrow{x_n})$
	\item Repeat step 2 until some stopping criteria is met, like $\norm{\delta \nabla J(\overrightarrow{x_n}) - \delta \nabla J(\overrightarrow{x_{n+1}})} < \epsilon$.
\end{enumerate}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.5\textwidth]{./differentialMultivariableCalculus/gradient_descent.png}
	\caption{Path of several iterations of gradient descent}
\end{figure}

\noindent
This method will lead you arbitrarily close to a local minimum, but does not guarantee finding the global minimum.
More advanced versions of gradient descent exists that try to help with this, like giving the point ``momentum'' to be able to move out of local mins.
This method also has a trade off between speed and accuracy.
Although increasing $\delta$ means fewer iterations of gradient descent are needed to narrow in on a local minimum, one is more likely to be stuck in a local min than if they had used a smaller $\delta$.\\

\noindent
In the real world, the function you are trying to minimize will likely not be well defined enough to take its partial derivatives to find the gradient, so they too are approximated by doing something like
\begin{equation*}
	J_{k} = \frac{J(k+.0001, \ldots) - J(k, \ldots)}{.0001}.
\end{equation*}


\subsection{Autocovariance and autocorrelation}

\subsubsection{Autocovariance}

\(AC(a,b)=cov(X_a, X_b)\)

\subsubsection{Autocorrelation}

The autocorrelation between two time periods is their covariance, normlised by their variances

\(AC(a,b)=\dfrac{E[(X_a-\mu_a)(X_b-\mu_b)]}{\sigma_a \sigma_b}\)

This is also called serial correlation.


% Principal Component Analysis
\section{Principal Component Analysis (PCA)}
\label{sec:pca}

Principal Component Analysis (PCA) is a decomposition of data into linearly uncorrelated components, ordered by their explained variance. It turns out that these axes are the right-singular eigenvectors $\mathbf{V}$ that are found from SVD. SVD and PCA have correspondence (cf. section \ref{sec:pcasvd}). My guess is that the advantage of PCA is that its results are interpretable in terms of commonly used summary statistics of the distribution (namely the variances or Pearson correlations of the data).
\\

More elaborately, for a data matrix $\mathbf{X}\in\mathbb{R}^{N\times p}$, PCA extracts the ordered, rank-$p$, orthonormal basis in which the $p\times p$ covariance matrix of $\mathbf{X}$ is diagonal. The basis vectors are called the principal axes or principal directions of the data, and their ordering is by the magnitude of the variance that they explain.
\\

The property that the covariance matrix is diagonal in the principal component basis means that projecting the data onto any of the basis vectors extracts a linearly uncorrelated component of the data that has variance corresponding to the corresponding eigenvalue of the covariance matrix.
\\

Just like with SVD, the ordering of the principal component basis in terms of their explained variance allows for lower-rank approximations of the data matrix to be constructed (section \ref{sec:svd}). Just like SVD, the lower rank approximations $\mathbf{X}$ are the best possible approximations with respect to the Frobenius Norm (cf. section \ref{sec:frobenius}).
\\

The $p\times p$ covariance matrix  $\mathbf{C}$ of $\mathbf{X}$ is:

\begin{equation}
\mathbf{C} = \frac{\left(\mathbf{X}-\left<\mathbf{X}\right>\right)^T\left(\mathbf{X}-\left< \mathbf{X}\right>\right)}{n-1}
\end{equation}

Where $\left(\mathbf{X}-\left<\mathbf{X}\right>\right)^T$ is often referred to as the \textit{centered} data matrix. The covariance matrix is a symmetric, positive-definite matrix that can be diagonalized with orthonormal eigenvectors $\mathbf{V}$ and positive (or vanishing) eigenvalues $\lambda_i$:

\begin{equation}
\mathbf{C} = \mathbf{V}\mathbf{\Lambda}\mathbf{V}^T
\end{equation}

Where the eigenvectors in $V$ are ordered so that the eigenvectors along the diagonal of $\mathbf{\Lambda}$ have decreasing magnitude. The eigenvectors are the principal axes or principal directions of the data.
\\
That PCA corresponds to the diagonalization of the covariance matrix implies a description of the underlying data in terms of a multivariate Gaussian distribution. In other words, it assumes normally distributed noise, and the components describe best-fit linear subspaces in the data under the $L^2$ norm. Whenever the Gaussian assumption is unjustified, it makes sense to look for best-fit linear subspaces under different norms. For example, $L^1$-PCA is more robust to outliers (cf. section \ref{sec:l1pca}).

% Relationship between PCA and SVD
\subsection{Relationship between PCA and SVD}
\label{sec:pcasvd}
This is based on a great Stack Exchange answer \cite{amoeba2015svdpca}.

Let the singular value decomposition of the centered data matrix be:

\begin{equation}
\left(\mathbf{X}-\left<\mathbf{X}\right>\right) = \mathbf{U}\mathbf{\Sigma}\mathbf{V}^T
\end{equation}

Then:

\begin{equation}
\mathbf{C} = \frac{\mathbf{V\Sigma U}^T\mathbf{U\Sigma V}^T}{n-1} = \mathbf{V}\frac{\Sigma^2}{n-1}\mathbf{V}^T
\end{equation}

That means that:

\begin{itemize}
\item The principal axes are the right-singular vectors $\mathbf{V}$ that are obtained during SVD.
\item The singular values and the eigenvalues of the covariance matrix are related via $\lambda_i = \frac{\sigma_i^2}{n-1}$.
\end{itemize}

\subsection{Tracking Principal Components over Time}
Given the direct parallel between PCA and SVD, the identical issue with sign flips emerges with the principal component basis, and the remedy is the same. See section \ref{sec:svd_tracking}.

\subsection{$L^1$-Norm Principal Component Analysis ($L^1$ PCA)}
\label{sec:l1pca}

$L1$-Norm Principal Component Analysis is an alternative to conventional PCA that aims to provide better robustness to outliers. While L2-PCA is uniquely defined, a cursory search reveals a zoo of approaches to what $L^1$ PCA is supposed to be \cite{brooks2012pcal1}. $L^1$-PCA may also be a natural choice in situations in which the $L^1$ norm is the proper way of measuring distances in a space, for example in cellular automata models or perhaps even in case of categorical data.

\citeasnoun{brooks2014pure} establishes that the PCA under $L^1$ norm is found by successive fitting of hyperplanes under $L^1$ error in progressively smaller subspaces. Much like the $L^2$ distance of a point to a hyperplane is given by the radius of a circle around that point, the $L^1$ distance of a point to a hyperplane is given by its intersection of a rhombus. This has the (kind of quirky) consequence that the distance of all points to a hyperplane is always measured directly along one of the coordinate axes (and it is the same coordinate axes for each of the points). This, in turn, implies, that in some subspace $\mathbb{R}^m$, the $L^1$ best-fit hyperplane can be found by performing an $L^1$ regression with each of the $m$ dimensions serving as the explanatory variable, and then selecting the regression result that had the smallest residual.

It is worth noting that this approach is within the spirit of minimizing the taxicab-like distances of a point to some sub-space, which assigns great significance to the coordinate grid. This goes against the spirit of PCA, in that it typically aims to extract some kind of natural coordinate system from the data, irrespective of the coordinate system in which the raw data was expressed. \possessivecite{brooks2014pure} method is within keeping of the definition of the $L^1$ norm, but its different from simply avoiding assigning quadratic importance to far-away points to achieve greater robustness.

\begin{figure}
\centering
    \includegraphics[width=0.4\textwidth]{l1pca.png}
    \caption{Distance of a point to a hyperplane in $L^1$ and $L^2$. The $L^1$ distance in this case is just the distance along the $x$ axis. If the hyperplane was cutting a shallower angle, then the distance would be measured along the $y$ axis. The best fit hyperplane to many points in $\mathbb{R}^2$ would be found by regressing the data points once with $y = \beta x + \epsilon$ and once with $x = \beta y + \epsilon$ and selecting the result with the smaller $||\epsilon||_1$. (The figure is from J. P. Brooks (2014).}
    \label{fig:l1pca}
\end{figure}


\section{Runge-Kutta Methods}
\begin{theo}
Suppose that $f(t,y)$ and all its partial derivatives of order less or equal to $n+1$ are continuous on $D=\left\{(t,y)|a\leq t\leq b,c\leq y\leq d\right\}$ ($D=[a,b]\times [c,d]$) and let $(t_0,y_0)\in D$. For every $(t,y)\in D$, there exists $\xi$ between $t$ and $t_0$ and $\mu$ between $y$ and $y_0$ with
\[
f(t,y) = P_n(t,y)+R_n(t,y)
\]
where
\begin{align*}
  P_n(t,y) &= f(t_0,y_0)+\left[(t-t_0)\partialD{f}{t}(t_0,y_0)+(y-y_0)\partialD{f}{y}(t_0,y_0)\right] \\
  &+ \left[\frac{(t-t_0)^2}{2}\partialD{^2f}{t^2}(t_0,y_0)+(t-t_0)(y-y_0)\partialD{^2f}{t\partial y}(t_0,y_0)+\frac{(y-y_0)^2}{2}\partialD{^2f}{y^2}(t_0,y_0)\right] \\
  &+ \left[\frac{1}{n!}\sum_{j=0}^{n+1}\binom{n}{j}(t-t_0)^{n-j}(y-y_0)^j\partialD{^{n+1}f}{t^{n+1}\partial y^i}(t_0,y_0)\right]
\end{align*}
\[
and\quad R_n(t,y)=\frac{1}{(n+1)!}\sum_{j=0}^{n+1}\binom{n+1}{j}(t-t_0)^{n+1-j}(y-y_0)\partialD{^{n+1}f}{t^{n+1}\partial y^i}(\xi,\mu)
\]
The function $P_n(t,y)$ is called the \emph{$n$th Taylor polynomial in two variables} for the function $f$ about $(t_0,y_0)$, and $R_n(t,y)$ is the remainder term associated with $P_n(t,y)$.
\end{theo}
\subsection{Runge-Kutta Methods of Order Two}
\begin{equation*}
  \begin{cases}
    y_{n+1}=y_n+h(c_1k_1+c_2k_2) &\\
    k1 = f(x_n,y_n) &\\
    k1 = f(x_n+\lambda_2h,y_n+\mu_{21}hk_1)
  \end{cases}
\end{equation*}
\begin{align*}
  T_{n+1} &= y(x_{n+1})-y(x_n)-h\left[c_1f(x_n,y_n)+c_2f(x_n+\lambda_2h,y_n+\mu_{21}hf_n)\right] \\
  &= hf_n+\frac{h^2}{2}\left[f_x'(x_n,y_n)+f_y'(x_n,y_n)f_n\right] \\
  &\quad -h\left[c_1f_n+c_2\left(f_n+\lambda_2f_x'(x_n,y_n)h+\mu_{21}f_y'(x_n,y_n)f_nh\right)\right]+O(h^3) \\
  &= (1-c_1-c_2)f_nh+\left(\frac{1}{2}-c_2\lambda_2\right)f_x'(x_n,y_n)h^2 \\
  &\quad +\left(\frac{1}{2}-c_2\mu_{21}\right)f_y'(x_n,y_n)f_nh^2+O(h^3) \\
  \Rightarrow & y_{n+1}=y_n+hf\left(x_n+\frac{h}{2},y_n+\frac{h}{2}f(x_n,y_n)\right)
\end{align*}

\subsection{Midpoint Method}
\begin{equation*}
  \begin{cases}
    \omega_0=\alpha & \\
    \omega_{i+1}=\omega_i+hf\left(t_i+\frac{h}{2},\omega_i+\frac{h}{2}f(t_i,\omega_i)\right) & \text{for $i=0,\cdots,N-1$}
  \end{cases}
\end{equation*}
Local truncation error: $O(h^2)$.

\subsection{Modified Euler Method} \begin{equation*}
  \begin{cases}
    \omega_0=\alpha & \\
    \omega_{i+1}=\omega_i+\frac{h}{2}\left[f(t_i,\omega_i),f\left(t_{i+1},\omega_i+hf(t_i,\omega_i)\right)\right] & \text{for $i=0,\cdots,N-1$}
  \end{cases}
\end{equation*}

\subsection{Higher-Order Runge-Kutta Methods}
Runge-Kutta Order Three:
\begin{equation*}
  \begin{cases}
    \omega_0=\alpha &\\
    k_1=hf(t_i,\omega_i) &\\
    k_2=hf(t_i+\frac{h}{2},\omega_i+\frac{1}{2}k_1) &\\
    k_3=hf(t_i+h,\omega-k_1+2k_2) &\\
    \omega_{i+1}=\omega_i+\frac{1}{6}(k_1+4k_2+k_3) &\\
  \end{cases}
\end{equation*}

Runge-Kutta Order Four:
\begin{equation*}
  \begin{cases}
    \omega_0=\alpha &\\
    k_1=hf(t_i,\omega_i) &\\
    k_2=hf(t_i+\frac{h}{2},\omega_i+\frac{1}{2}k_1) &\\
    k_3=hf(t_i+\frac{h}{2},\omega-k_1+\frac{1}{2}k_2) &\\
    k_4=hf(t_i+h,\omega_i+k_3) &\\
    \omega_{i+1}=\omega_i+\frac{1}{6}(k_1+2k_2+2k_3+k_4) &\\
  \end{cases}
\end{equation*}

\subsection{Computational Comparisons}
\begin{table}[H]
\begin{tabular}{|c|c|c|c|c|}
\hline
Evaluations per step & $n\in[2,4]$ & $n\in[5,7]$ & $n\in[8,9]$ & $n\in[10,\infty]$ \\ \hline
Best possible local truncation error & $O(h^n)$ & $O(h^{n-1})$ & $O(h^{n-2})$ & $O(h^3)$ \\ \hline
\end{tabular}
\end{table}


\chapter{Partitions}
\label{chapter:partitions}
The main question we study in this chapter is as follows:
``how many ways to put $n$ objects into $k$ boxes''.
Note that there are four modes for this question:
\begin{enumerate}
  \item the objects and boxes are identical,
  \item the objects are identical but boxes are different,
  \item the objects are different but boxes are identical,
  \item the objects and boxes are different.
\end{enumerate}
We are going to study the question in all these modes.
The Table~\ref{table:partitions} summarizes the results we are going
to prove for the cases when all the boxes are not empty.
\begin{table}[h!]
  \centering
  \begin{tabular}{l l l}
    \toprule
    Object's name & Parameters & Formula \\
    \midrule
    \multirow{4}{*}{Surjections}
                       & $n$ distinct objects &
                       \multirow{2}{*}{$\stirlingTwo{n}{k} k!$} \\
                       & $k$ distinct boxes   &  \\
    \rule{0pt}{2ex}
                       & $n$ distinct objects & \multirow{2}{*}{$\sum_{k = i}^n
                       \stirlingTwo{n}{k} k!$} \\
                       & any number of boxes  & \\
    \rule{0pt}{4ex}
    \multirow{4}{*}{Compositions}
                       & $n$ identical objects & \multirow{2}{*}{$\binom{n - 1}{k - 1}$} \\
                       & $k$ distinct boxes   &  \\
   \rule{0pt}{2ex}
                       & $n$ identical objects & \multirow{2}{*}{$2^{n - 1}$} \\
                       & any number of boxes  & \\
   \rule{0pt}{4ex}
   \multirow{4}{*}{Set partitions}
                      & $n$ distinct objects &
                      \multirow{2}{*}{$\stirlingTwo{n}{k}$} \\
                      & $k$ identical boxes   &  \\
    \rule{0pt}{2ex}
                      & $n$ distinct objects & \multirow{2}{*}{$B(n)$} \\
                      & any number of boxes  & \\
    \rule{0pt}{4ex}
    \multirow{4}{*}{Integer partitions}
                       & $n$ identical objects & \multirow{2}{*}{$p_k(n)$} \\
                       & $k$ identical boxes   &  \\
    \rule{0pt}{2ex}
                       & $n$ identical objects & \multirow{2}{*}{$p(n)$} \\
                       & any number of boxes  & \\
    \bottomrule
  \end{tabular}
  \caption{Formulas for the numbers of ways to put $n$ objects into $k$ boxes
  so that the boxes are not empty}
  \label{table:partitions}
\end{table}
\section{Set Partitions}
This section considers the case when objects are not identical.

First, we define a notion that allows us to compute the answer in case when
all the boxes are the same.
\begin{definition}
  A partition of the set $[n]$ is a collection of non-empty
  blocks so that each element of $[n]$ belongs to exactly
  one of these blocks. The number of partitions of $[n]$
  into $k$ nonempty blocks is denoted by $\stirlingTwo{n}{k}$.
  The numbers $\stirlingTwo{n}{k}$ are called the \emph{Stirling numbers
  of the second kind}.
\end{definition}
\nomenclature[C]{$\stirlingTwo{n}{k}$}{denotes the Stirling number
of the second kind; i.e. the number of partitions
of $[n]$ into $k$ nonempty blocks}

It is easy to see that $\stirlingTwo{n}{1} = 1$ and $\stirlingTwo{n}{n} = 1$.
Moreover, $\stirlingTwo{n}{k} = 0$ if $k > n$ or $k \le 0$.

Let us find the value in a more complicated setting, we
claim that $\stirlingTwo{n}{n - 1} = \binom{n}{2}$. Indeed, any
partition of $[n]$ into $n - 1$ blocks consists of
$n - 1$ singletons and one set with two elements, thus we
just need to select these two elements.

Using double counting, one may prove a recursive formula
for Stirling numbers of the second kind.
\begin{theorem}
  For any $n > k > 0$,
  \[
    \stirlingTwo{n}{k} = \stirlingTwo{n - 1}{k - 1} + k \cdot \stirlingTwo{n - 1}{k}.
  \]
\end{theorem}
\begin{proof}
  Let us consider $n$, note that there are two cases either $n$ forms a
  singleton in a partition or it is not the only element in the part.

  It is easy to see that there are $\stirlingTwo{n - 1}{k - 1}$ partitions where $n$ is
  a singleton and $k \cdot \stirlingTwo{n - 1}{k}$ partitions where $n$ is not a singleton
  (we multiply by $k$ since there are $k$ possible ways to add $n$ to a
  partition of $\range{n - 1}$).
\end{proof}

Using this notation, we can express the number of surjections.
\begin{lemma}
  There are exactly $k! \stirlingTwo{n}{k}$ surjective functions from
  $\range{n}$ to $\range{k}$.
\end{lemma}
\begin{proof}
  Let $\mathcal{S}(n, k)$ be the set of surjections from $\range{n}$ to $\range{k}$,
  $\mathcal{P}(n, k)$ be the set of partitions with non-empty blocks, and
  $F : \mathcal{S}(n, k) \to \mathcal{P}(n, k)$ such that
  $F(f) = \set{f^{-1}(1), \dots, f^{-1}(k)}$.

  It is easy to see that $F(f) = F(g)$ iff there is $h : \range{k} \to \range{k}$
  such that $f \circ h = g$. Hence, $F^{-1}(f) = k!$ for any $f \in \mathcal{S}(n, k)$.
  Thus $\cardinality{\mathcal{S}(n, k)} = k! \cardinality{\mathcal{P}(n, k)}$.
\end{proof}
Note that the number of surjections from $\range{n}$ to $\range{k}$ is equal to the
number of ways to put $n$ different objects into $k$ different boxes.

Using this equality, we can prove a surprising result.
\begin{theorem}
\label{theorem:stirling-numbers-and-polynomials}
  For any real $x$ and positive integer $n$,
  \[
    x^n = \sum_{k = 0}^n \stirlingTwo{n}{k} \numberOfPermutations[k]{x},
  \]
  where $\numberOfPermutations[k]{x} = \prod_{i = 0}^{k - 1} (x - i)$.
\end{theorem}

To prove the statement we need the following statement.
\begin{theorem}
  Let $p$ and $q$ be real polynomials. If $p(\ell) = q(\ell)$ for all
  natural numbers $\ell$, then $p(x) = q(x)$ for all real numbers $x$.
\end{theorem}

\begin{proof}[Proof of Theorem~\ref{theorem:stirling-numbers-and-polynomials}]
  Using the previous result, it is enough to prove that
  for any integer $\ell > 0$,
  \[
    \ell^n = \sum_{k = 0}^n \stirlingTwo{\ell}{k} \numberOfPermutations[k]{\ell}.
  \]

  Clearly $\ell^n$ denotes the number of ways to put $n$ different
  objects into $\ell$ different boxes. Note that if we have $k$ nonempty
  boxes, then there are $\binom{n}{k}$ ways to select these boxes and
  $k! \stirlingTwo{\ell}{k}$ ways to put objects in these $k$ boxes. Thus formula in
  the left is equal to the formula on the right.
\end{proof}

\begin{definition}
  The number of all set partitions of $\range{n}$ into nonempty parts
  is denoted by $B(n)$, and is called the \emph{$n$th Bell number}.
  (We define $B(0) = 0$).
\end{definition}
\nomenclature[C]{$B(n)$}{denotes the $n$th Bell number; i.e.
the number of partitions of $\range{n}$ into nonempty blocks}

It is easy to see that the following theorem holds.
\begin{theorem}
  For any $n \ge 0$,
  \[
    B(n) = \sum_{k = 0}^n \stirlingTwo{n}{k}.
  \]
\end{theorem}

However, it is also possible to express the Bells numbers in terms
of themselves.
\begin{theorem}
  For any $n \ge 0$,
  \[
    B(n + 1) = \sum_{i = 0}^n \binom{n}{i} B(i).
  \]
\end{theorem}
\begin{proof}
  Note that there are $B(n + 1)$ ways to split $\range{n + 1}$ into
  non-empty blocks. At the same time there are $\binom{n}{n - i}$
  ways to select elements to put with $n + 1$ in the same block
  (if we know that there are $n - i$ elements with $n + 1$ in the block)
  and $B(i)$ ways to split the rest into blocks. As a result,
  there are $\sum_{i = 0}^n \binom{n}{i} B(i)$ to split $\range{n + 1}$
  into nonempty blocks.
\end{proof}

\section{Composition}
This section answers the question in the case when the objects
are the same but boxes are different. Since all the objects
are identical, only the number of objects in each box matters.

\begin{definition}
  A sequence $(a_1, \dots, a_k)$ of nonnegative integers such
  that $a_1 + \dots + a_k = n$ is called a \emph{weak composition}
  of $n$ into $k$. If, in addition, all the numbers are positive,
  the sequence is called a \emph{composition}.
\end{definition}

Using the binomial coefficients we can find the number of weak
compositions.
\begin{theorem}
  For all positive integers $n$ and $k$,
  the number of weak compositions of $n$ into $k$ is equal to
  $\binom{n + k - 1}{n}$.
\end{theorem}
\begin{proof}
  Let us consider $k$ boxes in line one after each other. Note that if we
  put balls inside of the boxes we see a line consisting of $n$ balls and
  $k - 1$ walls separating the $k$ boxes from each other. Note that simply
  knowing in which order the n identical balls and $k - 1$ separating walls
  follow each other is the same as knowing the number of balls in each box.
  So our problem is equivalent to counting the number of ways to put $k - 1$
  walls on one of $n + k - 1$ positions.
\end{proof}

As a result, we can count the number of compositions.
\begin{corollary}
  For all positive integers $n$ and $k$,
  the number of compositions of $n$ into $k$ is equal to
  $\binom{n - 1}{k - 1}$.
\end{corollary}

\begin{exercise}
  Let $\ell_1$, \dots, $\ell_k$ be some nonnegative numbers
  such that $\ell_1 + \dots + \ell_k = \ell$.
  Find the number of weak compositions (in terms of $\ell$, $k$, and $n$)
  $(a_1, \dots, a_k)$ of $n$ into $k$ such that $a_i \ge \ell_i$.
\end{exercise}

\begin{corollary}
  The number of all compositions of $n$ is equal to $2^{n - 1}$.
\end{corollary}

\section{Integer Partitions}
Now consider the case when both objects and boxes are identical. In this case,
as in the previous we are only interested in numbers of objects in boxes, but
in addition, we are not interested in an order of these numbers.

\begin{definition}
  Let $n$ and $a_1 \ge a_2 \ge \dots \ge a_k \ge 1$ be integers so that
  $a_1 + \dots a_k = n$. Then the sequence $(a_1, \dots, a_k)$ is called
  a \emph{partition}\footnote[][-3cm]{%
    Note that we used the word partition in two different meanings: one to
    denote a partition of a set $[n]$ and another to denote the partition of
    an integer $n$. In most of the cases the meaning is clear from the context;
    however, if it is necessary to emphasize that we mean partition of a set,
    we say set-partition. Note that in some languages there are two different
    words for these two notions; e.g in French ``partition'' is used for
    set-partitions, and ``partage'' for partitions of the integer n).
  } of the integer $n$ into $k$ parts.

  The number of all the partitions is denoted by $p(n)$ and the number of
  partitions of $n$ into $k$ parts is denoted by $p_k(n)$.
\end{definition}

\nomenclature[C]{$p_k(n)$}{denotes the number of all the partitions of $n$ into
$k$ blocks}
\nomenclature[C]{$p(n)$}{denotes the number of all the partitions of $n$}

There is no good formula allowing to find the value of $p(n)$. Nevertheless, we will
prove some properties of $p(n)$. The main tool to explain proofs we are going to
discuss are Young diagrams\footnote{%
  A small variation of these diagrams is called Ferrers shapes after an
  American mathematician Norman Macleod Ferrers.
}.
A Young diagram for a partition $(a_1, \dots, a_k)$ consists of $k$ columns of
squares called ``boxes'' such that in the $i$th column there are $a_i$ boxes
(an example of such a diagram is depicted on \ref{figure:young-diagram-example}).
\begin{figure}
  \centering
  \subfloat[The Young diagram for the partition $(4, 3, 1, 1)$.]{
    \qquad\qquad$\yng(4,2,2,1)$\qquad\qquad
  }
  \qquad\qquad
  \subfloat[The conjugate of the Young diagram for the partition $(4, 3, 1, 1)$.]{
    \qquad\qquad$\yng(4,3,1,1)$\qquad\qquad
  }
  \caption[][1cm]{Young diagrams.}
  \label{figure:young-diagram-example}
\end{figure}
We can reflect a Young diagram of a partition of $n$ with respect to its main
diagonal, we get another shape, representing the \emph{conjugate} partition of
$n$ (an example of such transformation is also depicted on
\ref{figure:young-diagram-example}).

Using these diagrams, it is easy to show the following theorem.
\begin{theorem}
  The number of partitions of $n$ into at most $k$ parts is equal to that of
  partitions of $n$ into parts not larger than $k$.
\end{theorem}
\begin{proof}
  Note that if a partition has at most $k$ parts, then the conjugate of this
  partition has all the parts of size at most $k$. As, a result, the number
  of partitions of $n$ into at most $k$ parts is equal to that of
  partitions of $n$ into parts not larger than $k$.
\end{proof}

\begin{chapterendexercises}
  \exercise How many sets $S \subseteq \range{n}$ such that $S$ does not contain
    two successive integers (i.e., if $x \in S$, then $x + 1 \not\in S$)?
  \exercise How many numbers in $\range{999999}$ have the sum of digits at most
    $47$?
  \exercise Let $k \in \N$. How many sets $S \subseteq \range{n}$ of size $k$
    such that $S$ does not contain two successive integers (i.e., if $x \in S$,
    then $x + 1 \not\in S$)?
  \exercise
    Let $q(n)$ be the number of partitions of $n$ in which each part is at least
    two. Then $q(n) = p(n) - p(n - 1)$, for all positive integers $n \ge 2$.
  \exercise[recommended] Find a formula for $\stirlingTwo{n}{2}$.
  \exercise Find a formula for $\stirlingTwo{n}{3}$.
    \begin{solution}
      In this problem it is easier to find the number of surjections from
      $\range{n}$ to $\range{3}$. There are $3^n$ functions from $\range{n}$ to
      $\range{3}$, there are $3 \cdot 2^n$ functions from $\range{n}$ to
      $\range{3}$ such that the image has $2$ elements, and there are $3$
      functions such that their image has $1$ element. Therefore, by the
      inclusion-exclusion principle, there are $3^n - 3 \cdot 2^n + 3$
      surjections from $\range{n}$ to $\range{3}$. As a result,
      $\stirlingTwo{n}{3} = \frac{1}{6}(3^n - 3 \cdot 2^n + 3)$.
    \end{solution}
  \exercise Find a formula for $\stirlingTwo{n}{n - 2}$.
    \begin{solution}
      Note that there two variants how we can split $n$ elements into $n - 2$ subsets:
      \begin{enumerate}
        \item all subsets except one are singletons, there are $\binom{n}{3}$
          ways to do this;
        \item all subsets except two are singletons, there are $\binom{n}{4} \cdot
          \binom{4}{2} \frac{1}{2}$ (we divide by two since the order ot these two
          sets of size $2$ is not important).
      \end{enumerate}
      Hence, the answer is $\binom{n}{3} + \binom{n}{4} \cdot \binom{4}{2} \frac{1}{2}$.
    \end{solution}
  \exercise[recommended] Show that $B(n) \le n!$.
  \exercise Let $m \ge n$ be positive integers. Show that
    \[
      \stirlingTwo{m}{n} = \sum_{i = 1}^m \stirlingTwo{m - i}{n - 1} n^{i - 1}.
    \]
  \exercise Prove that the number of partitions of $n$ into exactly $k$
    parts is equal to the number of partitions of $n$ in which the
    largest part is exactly $k$.
  \exercise[recommended] Prove that the number of partitions of $n$ into at most
    $k$ parts is equal to that of partitions of $n + k$ into exactly $k$ parts.
  \exercise[recommended] We call a partition $\set{P_1, \dots, P_k}$ of
    $\range{n}$ nice iff $(j + 1) \not\in P_i$ for every $i \in \range{k}$ and
    $j \in P_i$.
    Prove that number of nice partitions is equal to $B(n - 1)$.
    \begin{solution}
      Let $\mathcal{P}_n$ be the set of all partitions of $\range{n}$ and let
      $\mathcal{M}_n$ be the set of all nice partitions of $\range{n}$. We are
      going to construct a bijection
      \[
        f : \mathcal{P}_{n - 1} \to \mathcal{M}_n.
      \]

      Notice that we can obtain every nice partition of $\range{n}$ by:
      \begin{enumerate}
        \item First adding a singleton block $\set{n}$ to any partition of
          $\range{n}$.
        \item Then, in each block of that partition of $\range{n - 1}$, we locate
          consecutive integers $i, i + 1, \dots, i + j$ and if $j$ is odd place
          every other integer, i.e $i, i + 2, i + 4, \dots, i + j - 1$, into
          the block with $n$, if $j$ is even we put $i + 1, \dots, i + j - 1$ into
          the block with $n$. We do this for each consecutive sequence of each
          block of each partition of $\range{n - 1}$ to obtain all possible nice
          partitions of $\range{n}$.
      \end{enumerate}
      Note that the resulting partition is nice, since $i + j \le n - 1$, hence
      $i + j - 1 < n - 1$.

      For each nice partition of $\range{n}$ obtained this way we can have an
      inverse transformation $f^{-1}$ that takes every nice partition of
      $\range{n}$ and gives the corresponding original partition of
      $\range{n - 1}$ by:
      \begin{enumerate}
        \item Taking every element except $n$ in the block that contains
          $n$ and placing each element $i$, in order, into a block that contains
          $i - 1$.
        \item We then remove the block of $n$ from our partition resulting in a
          partition of $\range{n - 1}$.
      \end{enumerate}

      It can be seen that applying $f^{-1}$ to every partition of $\mathcal{M}_n$ gives us
      every partition of $\mathcal{P}_n$. Thus $f$ forms a bijection between
      $\mathcal{M}_n$ and $\mathcal{P}_{n - 1}$.
    \end{solution}
\end{chapterendexercises}


\lab{Algorithms}{Givens Rotations}{Givens Rotations}
\label{Ch:Givens Rotations}

\objective{Learn how to use Givens rotations in numerical computing.}
\section*{Givens rotations}

The matrix $\begin{pmatrix} cos(\theta) & -sin(\theta) \\ sin(\theta) & cos(\theta) \end{pmatrix}$ rotates a vector counterclockwise by $\theta$. Given a vector $x = \begin{pmatrix} a \\ b \end{pmatrix}$, we can rotate $x$ into the range of $e_1$ by choosing the correct $\theta$. The problem is equivalent to solving for $c = cos(\theta)$ and $s = sin(\theta)$ in the system
\begin{equation}
\label{eq:Givens rotation system}
\begin{pmatrix} c & -s \\ s & c \end{pmatrix} \begin{pmatrix}a\\b\end{pmatrix}
= \begin{pmatrix}r\\0\end{pmatrix}
\end{equation}
 In fact, it's not necessary to compute $\theta$; we solve for $c$ and $s$ directly. An obvious solution is $c = \frac{a}{\sqrt{a^2 + b^2}}$ and $s = \frac{b}{\sqrt{a^2+b^2}}$. However, a more numerically stable method is

 \begin{pseudo}{Givens rotation solver}{a,b}
 \label{Alg:Givens rotation solver}
\IF b=0 \THEN c \GETS copysign(1,a); s \GETS 0
\ELSEIF a=0 \THEN c \GETS 0;s \GETS -copysign(1,b)
\ELSEIF abs(b)>abs(a) \THEN
\BEGIN
	t \GETS a/b\\
	u \GETS copysign(sqrt(1+t*t),b)\\
	s \GETS -1/u\\
	c \GETS -s*t
\END
\ELSE
\BEGIN
	t \GETS b/a \\
	u \GETS copysign(sqrt(1+t*t),a)\\
	c \GETS 1/u\\
	s \GETS -c*t
\END
\end{pseudo}\\
where \li{copysign(a,b)} returns \li{a} with the sign of \li{b} (if this function is not available, use \li{a*sign(b)}).

\subsection*{Givens triangularization}

Like Householder, the Givens $QR$ algorithm, \emph{triangularizes} a matrix $A$ using \emph{orthogonal transformations}. The Householder $QR$ algorithm worked one column at a time; Givens works one element at a time. Each nonzero below the main diagonal can be zeroed out with a Givens rotation. If we solve for $c$ and $s$ as above, the matrix
\begin{equation*}
G(i,j,c,s) =
\begin{pmatrix}
1   & \cdots & 0 & \cdots & 0 & \cdots & 0 \\
 \vdots & \ddots & \vdots &  & \vdots & & \vdots \\
0   & \cdots &    c   & \cdots &    -s   & \cdots &    0   \\
 \vdots &        & \vdots & \ddots & \vdots &        & \vdots \\
 0   & \cdots &   s   & \cdots &    c   & \cdots &    0   \\
\vdots &        & \vdots &        & \vdots & \ddots & \vdots \\
0   & \cdots &    0   & \cdots &    0   & \cdots &    1
\end{pmatrix}
\end{equation*}
where
\[ \begin{array}{ll}
g_{i\, i} = c  & g_{j\, i}= -s   \\
g_{j\, j} = c &  g_{i\, j}= s  \\
\end{array}\]
rotates by $\theta$ in the $i,j$ plane. Also, it's easy to check that $G(i,j,c,s)$ is orthogonal. Left-multiplying $A$ by $G(i,j,c,s)$ zeroes out $A_{i\, j}$. For example,

\[
\begin{array}{ccccccc}
\begin{pmatrix}
*&*&*\\
*&*&*\\
*&*&*
\end{pmatrix}
&
\underrightarrow{G(3,1)}
&\begin{pmatrix}
*&*&*\\
*&*&*\\
0&*&*
\end{pmatrix}
&
\underrightarrow{G(2,1)}
&\begin{pmatrix}
*&*&*\\
0&*&*\\
0&*&*
\end{pmatrix}
&
\underrightarrow{G(3,2)}
&\begin{pmatrix}
*&*&*\\
0&*&*\\
0&0&*
\end{pmatrix}
\end{array}
\]

The Givens QR method is often slower than Householder, because it works one element at a time. But it is faster than Householder when $A$ is sparse, and it is very parallelizable.

\begin{problem}
Write a script that uses Givens rotations to do QR decomposition. By performing successive Givens rotations, triangularize $A$ to find $R$. You can find $Q$ using the chain of rotations.

To find each rotation, you will have to solve \eqref{eq:Givens rotation system} for $c$ and $s$. Let $b$ be the element you want to zero out, say $A_{i\,j}$. Then $a$ is in the same \emph{column} as $b$, and in the \emph{row} you want to rotate into (it's like we're squishing the whole vector into $a$'s spot.) You can choose $a$ to be the diagonal element above $b$.
\end{problem}

\begin{problem}
Compare the MGS, Householder, and Givens algorithms for $QR$ decomposition on various matrices. Try different sizes and different levels of sparsity. Which is the fastest? Which is the most stable?
\end{problem}


\section{Signal-to-noise maximisation}
\label{sec:GEVD_SNR}

Suppose all training samples $\z\train_t$ are gathered and divided over two
data matrices $\Sm \in \reals^{C \cross N_s}$ and $\Nm \in \reals^{C \cross
N_n}$, where $\Sm$ (for `signal') contains all $N_s$ samples of $\z\train_t$
where an SWR was present, and $\Nm$ (for `noise') contains all $N_n$ other
samples. (These matrices can be constructed by simply concatenating all
segments in $\z\train_t$ where an SWR was present, and all segments where
there was no SWR present).

As described in the previous section, we project the data onto the weight
vector $\w$. For the data matrices, this becomes, in vector notation:
%
\begin{align*}
\y_s &= \w^T \Sm \\
\y_n &= \w^T \Nm,
\end{align*}
where each element of the row vectors $\y_s$ and $\y_n$ corresponds to one
(multichannel) sample of $\Sm$ or $\Nm$. \Cref{fig:GEVD_principle} (right)
shows the distribution of the values in two example data vectors $\y_s$ and
$\y_n$.

We want to find the weight vector $\what$ that maximises the variance of
$\y_s$ versus the variance of $\y_n$, i.e.
%
\begin{align}
\what &= \argmax_{\w} \frac{\Var(\y_s)}
                           {\Var(\y_n)}  \nonumber \\[1em]
      &= \argmax_{\w}
         \frac{\frac{1}{N_s} \y_s \y_s^T}
              {\frac{1}{N_n} \y_n \y_n^T}   \label{eq:argmax_SNR} \\[1em]
      &= \argmax_{\w}
         \frac{\frac{1}{N_s} \w^T \Sm \Sm^T \w}
              {\frac{1}{N_n} \w^T \Nm \Nm^T \w}  \nonumber
\end{align}

The signal and noise covariance matrices $\Rss \in \reals^{C \cross C}$ and
$\Rnn \in \reals^{C \cross C}$ are defined as:\footnotemark{}
%
\begin{align}
\Rss &= \frac{1}{N_s} \Sm \Sm^T \label{eq:covariance_s}\\
\Rnn &= \frac{1}{N_n} \Nm \Nm^T \label{eq:covariance_n}
\end{align}
%
($\Rss$ and $\Rnn$ are symmetric. The diagonal elements yield the variance of
each channel, and the off-diagonal elements yield the covariance between
pairs of channels).

\footnotetext{To be precise, these are only estimates of the `true'
covariance matrices of two random, \emph{ergodic} variables $\z_{s,t}$ and
$\z_{n,t}$, of which the data in $\Sm$ and $\Nm$ are some samples.
\Cref{eq:covariance_s,eq:covariance_n} are then correct definitions only in
the limit for $N_s \to \infty$ and $N_n \to
\infty$.}

The expression for the optimal weight vector (\cref{eq:argmax_SNR}) then
becomes:
%
\begin{equation}
\label{eq:argmax_R}
\what = \argmax_{\w}
       \frac{\w^T \Rss \w}
            {\w^T \Rnn \w}
\end{equation}
%
This quotient of quadratic forms is the `generalised Rayleigh quotient' of
$\Rss$ and $\Rnn$. It reaches its maximum when $\w$ is the first so caled
`generalised eigenvector' of $\Rss$ and $\Rnn$. The following section defines
these terms, and proves the ``$\argmax$'' --- ``first generalised
eigenvector'' correspondence.


\subsection{Right-handed systems of vectors}

We begin with a discussion of right-handed systems of vectors in
$3$-dimensional space.%
\index{right-handed system of vectors}%
\index{vector!right-handed system of}

\begin{definition}{Right-handed system of vectors}{right-hand}
  Three vectors, $\vect{u},\vect{v},\vect{w}$ form a right-handed
  system if when you extend the thumb of your right hand in the
  direction of $\vect{u}$ and your index finger in the direction of
  $\vect{v}$, your relaxed middle finger points roughly in the
  direction of $\vect{w}$.
  \begin{center}
    \raisebox{0.25in}{
      \begin{tikzpicture}
        \draw[->](0,0,0) -- node[right] {$\vect{u}$} (0,2,0);
        \draw[->](0,0,0) -- node[above] {$\vect{v}$} (-2,0,0);
        \draw[->](0,0,0) -- node[right] {$\vect{w}$} (0,0,3);
      \end{tikzpicture}
    }
    \hspace{1in}
    \includegraphics[height=1.8in]{figures/right-handed}
  \end{center}
\end{definition}

You should consider how a right-handed system would differ from a
left-handed system. Try using your left hand and you will see that the
vector $\vect{w}$ would need to point in the opposite direction.

Recall the special vectors $\vect{i}=\mat{1,0,0}^T$,
$\vect{j}=\mat{0,1,0}^T$, and $\vect{k}=\mat{0,0,1}^T$ we saw in
Section~\ref{sec:linear-combinations-rn}. We always assume that our
coordinate system is drawn in such a way that the vectors $\vect{i}$,
$\vect{j}$, $\vect{k}$ form a right-handed system. Thus, if the thumb
of your right hand points along the $x$-axis and your index finger
points along the $y$-axis, your middle finger should point along the
$z$-axis.

\begin{center}
  \begin{tikzpicture}
    \draw[->, thick] (0,0,0)--(2,0,0);
    \draw[->, thick] (0,0,0)--(0,2,0);
    \draw[->, thick] (0,0,0)--(0,0,2);
    \node[below right] at (2,0,0){$\vect{j}$};
    \node[below left] at (0,0,2){$\vect{i}$};
    \node[above right] at (0,2,0){$\vect{k}$};
  \end{tikzpicture}
\end{center}

\noindent
When all three vectors lie in a plane, then we say that the vectors
are \textbf{coplanar}%
\index{coplanar vectors}%
\index{vector!coplanar}. In this case, the system is neither
right-handed nor left-handed.


\section{Identication Trees}

\paragraph{Identication Tree} An identification tree is a
represntation, that is a decision tree in which:
\begin{itemize}
  \item Each set of possible conclusion is established implicitly
    by a list of samples of known class
\end{itemize}

\paragraph{Average disorder}
\begin{math}
  Average(disorder) =
  \sum_{b}{\frac{n_b}{n_t} \sum_{c}{-\frac{n_{bc}}{n_b}\log_{2}\frac{n_{bc}}{n_b}}}
\end{math}
where:
\begin{itemize}
  \item $n_b$ is the number of samples in brench $b$
  \item $n_t$ is the total number of samples in all branches
  \item $n_bc$ is the total of samples in branch $b$ of class $c$
\end{itemize}

To generate an identification tree using SPROUTER:
\begin{itemize}
  \item Until each leaf node is populated by as homogeneous a
    sample set as possible:
    \begin{itemize}
      \item Select a leaf node with an inhomogeneous sample set
      \item Replace that leaf node by a test node that divides the
        inhomogeneous sample set into minimally inhomogeneous
        subsets, according to some measure of disorder
    \end{itemize}
\end{itemize}

To convers an identification tree into a rule set, execute the
following procedure - PRUNER:
\begin{itemize}
  \item Create one rule for each root-to-leaf path in the
    identification tree
  \item Simplify each rule by discarding antecedents that have no
    effect on the conclusion reached by the rule
  \item Replace those rules that share the most common consequent
    by a default rule that is triggered when no other rule is
    triggered. In the eventi of a tie, use some heuristic tie
    breaker to choose a default rule
\end{itemize}