\def\fac{{\left(k^*/k\right)^\alpha}}

\subsection{Convergence}

Following \cite{Dunkley2008}, we model the power spectrum of each variate as:

\begin{equation}
P(k) = P_0 {{\fac\over{\fac + 1}}}
\end{equation}

$P_0$ is an estimate of the sample variance, and $k^*$ is the
wavenumber where the power spectrum turns over to white noise.

{\bf Implementation Notes}

We fit the power spectrum to this function, for which we require the
derivatives w.r.t. $P_0$, $k^*$ and $\alpha$.  Letting

\begin{equation}
f = \fac,
\end{equation}

we have:

\begin{eqnarray}\nonumber
{df}\over{dk^*} &=& f\,{{\alpha}\over{k^*}}\\\nonumber
{df}\over{d\alpha} &=& f\,\ln{(k^*/k)}
\end{eqnarray}

and

\begin{eqnarray}\nonumber
{{dP}\over{dP_0}} &=& {{f}\over{f+1}}\\\nonumber
{{dP}\over{dk^*}} = {{dP}\over{df}}{{df}\over{dk^*}} &=& {{f}\over{f+1}}\times P_0\left[1 - {{f}\over{f+1}}\right]{{\alpha}\over{k^*}}\\\nonumber
{{dP}\over{d\alpha}} = {{dP}\over{df}}{{df}\over{d\alpha}} &=& {{f}\over{f+1}}\times P_0\left[1 - {{f}\over{f+1}}\right]{\ln{(k^*/k)}}
\end{eqnarray}

Note that the convergence test relies on using the power spectrum of
the chain $\tilde{P}(k)$ to estimate the true sample variance $P_0$.
Thus the power spectrum should be normalized so that $P(k=0)$ is an
estimate of the sample variance.  For real transforms in FFTW, the
discrete Fourier transform is unnormalized, or

\begin{equation}
Y_k = \sum^{N-1}_{j=0}{X_j e^{-2\pi jk\sqrt{-1}/N}}
\end{equation}

For $k=0$, we see that

\begin{equation}
Y_0 = \sum^{N-1}_{j=0}{X_j},
\end{equation}

and

\begin{eqnarray}
Y^2_0 &=& \left(\sum^{N-1}_{j=0}{X_j}\right)^2\\
 &=& \sum_j{X^2_j} + \sum_i\sum_{j\neq i}{X_iX_j}\\
 &=& N \left<X^2\right> + 0,
\end{eqnarray}

assuming the $X_i$ and $X_j$ are uncorrelated.  Therefore, in order
for the power spectrum to reflect the variance, we instead compute the
power spectrum of the normalized samples $X_j^\prime = X_j/\sqrt{N}$.

Once $P_0$


\section{Linear Classifier}
Feature vectors $x$, labels $y$
\begin{align*}
x \in \mathbb{R}^d\\
y \in \{-1,1\}
\end{align*}
Training set
\begin{align*}
S_n = \{(x^{(i)}, y^{(i)}), i=1,...,n\}
\end{align*}

Classifier
\begin{align*}
h: \mathbb{R}^d \rightarrow \{-1,1\}\\
\chi^{+} = \{x \in \mathbb{R}^d: h(x) =1\}\\
\chi^{-} = \{x \in \mathbb{R}^d: h(x) =-1\}
\end{align*}

Training error
\begin{align*}
\varepsilon_n(h)= \frac{1}{n} \sum_{i=1}^n \textbf{1}\{h(x^{(i)}) \neq y^{(i)} \}
\end{align*}

Test error (over disjoint set of examples)

\begin{align*}
\varepsilon(h)
\end{align*}

Set of classifiers
\begin{align*}
h \in H
\end{align*}

\subsection{Linear classifiers through origin}

Set of all points that satisfies a line through the origin.

\begin{align*}
\theta &= \begin{bmatrix}
           \theta_{1} \\
           \theta_{2}
         \end{bmatrix}\\
X &= \begin{bmatrix}
           x_{1} \\
           x_{2}
         \end{bmatrix}\\
\end{align*}

Decision Boundary
\begin{align*}
\{x&: \theta_1 x_1 + \theta_2 x_2 = 0\}\\
\{x&: \theta \cdot X=0\}
\end{align*}

Linear Classifier through origin
\begin{align*}
h(x,\theta)=sign(\theta \cdot X)\\
\Theta \in \mathbb{R}^d
\end{align*}

\subsection{Linear classifiers}
General linear Classifier (with Intercept)
\begin{align*}
\theta &= \begin{bmatrix}
           \theta_{1} \\
           \theta_{2}
         \end{bmatrix}\\
X &= \begin{bmatrix}
           x_{1} \\
           x_{2}
         \end{bmatrix}\\
\end{align*}

Decision Boundary
\begin{align*}
\{x&: \theta \cdot X + \theta_0 = 0\}
\end{align*}

Linear Classifier through origin
\begin{align*}
h(x,\Theta, \theta_0)=sign(\theta \cdot X + \theta_0)\\
\theta \in \mathbb{R}^d\\
\theta_0 \in \mathbb{R}
\end{align*}

\subsection{Linear Separation}

Traning examples $S_n = \{(x^{(i)}, y^{(i)}), i=1,...,n\}$ are linear separable if there exists a parameter vector $\hat{\theta}$ and offset parameter $\hat{\theta}_0$ such that $y^{(i)}(\hat{\theta} \cdot x^{(i)} + \hat{\theta}_0 )>0$ for all $i=1,\cdots,n$.

\begin{align*}
(\hat{\theta} \cdot x^{(i)})>0
	\begin{cases}
		 y^{(i)}>0 \text{ and } \theta \cdot x^{(i)} >0\\
		 y^{(i)}<0 \text{ and } \theta \cdot x^{(i)} <0\\
	\end{cases}
\end{align*}

$y^{(i)}(\theta \cdot x^{(i)})>0$ if label and classified result match. This leads to a new definition of the \textbf{Training error}:

\begin{align*}
\varepsilon_n(\theta)&= \frac{1}{n} \sum_{i=1}^n \textbf{1}\{y^{(i)}(\theta \cdot x^{(i)}) \leq 0\}\\
\varepsilon_n(\theta,\theta_0)&= \frac{1}{n} \sum_{i=1}^n \textbf{1}\{y^{(i)}(\theta \cdot x^{(i)} + \theta_0) \leq 0\}
\end{align*}

\subsection{Perceptron through Origin}
\textbf{Perceptron}$\displaystyle \left(\big \{ (x^{(i)}, y^{(i)}), i=1,...,n\big \} , T \right):$
\begin{enumerate}[\indent {}]
	\item initialize  $\theta=0$ (vector);
	\begin{enumerate}[\indent {}]
		\item for $t=1,\cdots,T$ do
		\begin{enumerate}[\indent {}]
			\item for $i=1,\cdots,n$ do
			\begin{enumerate}[\indent {}]
				\item if $y^{(i)}(\theta \cdot x^{(i)})\leq 0$ then
				\item update $\theta = \theta + y^{(i)}x^{(i)}$
			\end{enumerate}
		\end{enumerate}
	\end{enumerate}
\end{enumerate}

\subsection{Perceptron with Offset}

\textbf{Perceptron}$\displaystyle \left(\big \{ (x^{(i)}, y^{(i)}), i=1,...,n\big \} , T \right):$
\begin{enumerate}[\indent {}]
	\item initialize  $\theta=0$ (vector); $\theta_0=0$ (scalar)
	\begin{enumerate}[\indent {}]
		\item for $t=1,\cdots,T$ do
		\begin{enumerate}[\indent {}]
			\item for $i=1,\cdots,n$ do
			\begin{enumerate}[\indent {}]
				\item if $y^{(i)}(\theta \cdot x^{(i)} + \theta_0)\leq 0$ then
				\item update $\theta = \theta + y^{(i)}x^{(i)}$
				\item update $\theta_0=\theta_0 + y^{(i)}$
			\end{enumerate}
		\end{enumerate}
	\end{enumerate}
\end{enumerate}

\subsection{Margin Boundary}

The Margin Boundary is the set of points  $x$  which satisfy $\theta \cdot x + \theta _0= \pm 1$. So, the distance (with signed direction) from the decision boundary to the margin boundary is $\displaystyle \frac{1}{\mid \mid \theta \mid \mid }$.

$\frac{y^{(i)}(\theta \cdot x^{(i)} + \theta _0)}{\mid \mid \theta \mid \mid }=\frac{1}{\mid \mid \theta \mid \mid }.$

\textbf{Hinge Loss (agreement)}
\begin{align*}
Agreement &= z = y^{(i)}(\theta \cdot x^{(i)} + \theta _0)\\
Loss_h(z) &=
\begin{cases}
		 0 \text{ if } z \geq 1\\
		 1-z \text{ if } z < 1
\end{cases}
\end{align*}

\textbf{Regularization} means pushing out the margin boundaries by adding $max(\frac{1}{\mid \mid \theta \mid \mid })$ or $min(\frac{1}{2}\mid \mid \theta \mid \mid^2)$ to the objective function.

Alternatively, the sum of the hinge losses can be calculated by
$\sum _{i=1}^{n} \max \{ 0, 1 - y^{(i)} (\theta \cdot x^{(i)} + \theta _0)\}$

\textbf{Objective Function}

Objective function = average loss + regularization\\

Objective function is minimized, learning becomes an optimization problem. Using hinge loss and margin boundaries is called \textbf{Support Vector Machine} or \textbf{Large margin linear classification}:

\begin{align*}
J(\theta , \theta _0) = \frac{1}{n} \sum _{i=1}^{n} \text {Loss}_ h (z) + \frac{\lambda }{2} \mid \mid \theta \mid \mid ^2.
\end{align*}

Where $\lambda > 0$ is called the regularization parameter that regulates how important the margin boundaries are in comparison to the average hinge loss.

\textbf{Cost:} is an averaged loss.

\subsection{Gradient Descent}

Assume $\theta \in \mathbb{R}$ the goal is to find $\theta$ that minimizes $J(\theta , \theta _0) = \frac{1}{n} \sum _{i=1}^{n} \text {Loss}_ h (y^{(i)} (\theta \cdot x^{(i)} + \theta _0 )) + \frac{\lambda }{2} \mid \mid \theta \mid \mid ^2$ through gradient descent.\\

In other words, we will
\begin{itemize}
\item Start $\theta$ at an arbitrary location: $\theta \leftarrow \theta _{start}$
\item Update $\theta$ repeatedly with $\theta \leftarrow \theta - \eta \frac{\partial J(\theta , \theta _0)}{\partial \theta }$ until $\theta$ does not change significantly.
\end{itemize}

Where $\eta >0$ is called the stepsize or \textbf{learning parameter}.

\subsection{Stochastic Gradient Descent}

\begin{align*}
J(\theta , \theta _0) &= \frac{1}{n} \sum _{i=1}^{n} \text {Loss}_ h (z) + \frac{\lambda }{2} \mid \mid \theta \mid \mid ^2\\
&= \frac{1}{n} \sum _{i=1}^{n}\big [ \text {Loss}_ h (z) + \frac{\lambda }{2} \mid \mid \theta \mid \mid ^2 \big ]
\end{align*}

With stochastic gradient descent, we choose $i \in \big \{ 1,...,n \big \}$ at random and update $\theta$ such that\\

\begin{align*}
\theta \leftarrow \theta - \eta \nabla _{\theta } \big [\text {Loss}_ h(z) + \frac{\lambda }{2}\mid \mid \theta \mid \mid ^2 \big ]
\end{align*}


\subsubsection{Real Distinct Eigenvalues}
\noindent
Real, distinct eigenvalues are the simplest case, similar to real, distinct roots of an auxiliary equation.
\begin{theorem}
	Let $\left\{\lambda_1, \ldots, \lambda_n\right\}$ be the set of unique eigenvalues and $\left\{\vec{v_1}, \ldots, \vec{v_n}\right\}$ be the corresponding set of non-zero, unique, eigenvectors for an $n \times n$ matrix $A$. Then the set of fundamental solutions to the system $\vec{x}' = A\vec{x}$ is $\left\{e^{\lambda_1 t}\vec{v_1}, \ldots, e^{\lambda_n t}\vec{v_n} \right\}$
\end{theorem}

\begin{example}
	Find the general solution to the system
	\begin{equation*}
		\vec{x}' = \begin{bmatrix}
			3 & 0 & 0 \\
			-5 & -2 & 0 \\
			1 & 1 & -1
		\end{bmatrix} \vec{x}.
	\end{equation*}
\end{example}
\noindent
For a triangular matrix, the eigenvalues are simply the diagonal entries.
\begin{equation*}
	\lambda = 3, -2, -1.
\end{equation*}
Finding the eigenvector for $\lambda = 3$,
\begin{equation*}
	(A - 3I)\vec{v} = \vec{0} \implies \vec{v} = C_1\begin{bmatrix}
		-1 \\
		1 \\
		0
	\end{bmatrix}.
\end{equation*}
Finding the eigenvector for $\lambda = -2$,
\begin{equation*}
	(A + 2I)\vec{v} = \vec{0} \implies \vec{v} = C_2\begin{bmatrix}
		0 \\
		-1 \\
		1
	\end{bmatrix}.
\end{equation*}
Finding the eigenvector for $\lambda = -1$,
\begin{equation*}
	(A + I)\vec{v} = \vec{0} \implies \vec{v} = C_3\begin{bmatrix}
		0 \\
		0 \\
		1
	\end{bmatrix}.
\end{equation*}
So, our general solution is
\begin{equation*}
	\vec{x} = C_1e^{3t}\begin{bmatrix}
		-1 \\
		1 \\
		0
	\end{bmatrix} + C_2e^{-2t}\begin{bmatrix}
		0 \\
		-1 \\
		1
	\end{bmatrix} + C_3e^{-t}\begin{bmatrix}
		0 \\
		0 \\
		1
	\end{bmatrix}.
\end{equation*}


The $\Delta{}t$ component represents the time that it takes for the sound emitted by source $S$ to reach the position of virtual microphone $m$. Following from (\ref{distance}), the value of $d_{\vec{mS}}$ when combined with the speed of sound constant $c$ can be used to find the time-domain component.

\subsubsection{Calculating the speed of sound}

The speed of sound in real spaces is generally considered to be ~\SI[per-mode=fraction]{343}{\m\per\s}. However, it is not a constant. The speed of sound can change based on a number of environmental parameters. To account for this, the speed of sound can be adjusted by some value, $\Delta{}c$, which is implemented in this version as a number $\Delta{}c = [-10, 10]$; so that:

\begin{equation}\label{speedofsound}
c = 343 + \Delta{}c
\end{equation}

Following the usual formula for time:

\begin{equation}\label{timeFormula}
t = \frac{d}{c}
\end{equation}

Substitutions can be made based on (\ref{distance}) and (\ref{speedofsound}) to give the expansion of $\Delta{}t$:

\begin{equation}\label{deltat}
\Delta{}t = \frac{d_{\vec{v}}}{343 + \Delta{}c}
\end{equation}


\chapter{Description of the Problem}
Let $f: D \rightarrow \mdr$ with $D \subseteq \mdr$ be a polynomial function and $P \in \mdr^2$
be a point. Let $d_{P,f}: \mdr \rightarrow \mdr_0^+$
be the Euklidean distance of $P$ to a point $\left (x, f(x) \right )$
on the graph of $f$:
\[d_{P,f} (x) := \sqrt{(x - x_P)^2 + (f(x) - y_P)^2}\]

Now there is finite set $M = \Set{x_1, \dots, x_n} \subseteq D$ of minima for given $f$ and $P$:
\[M = \Set{x \in D | d_{P,f}(x) = \min_{\overline{x} \in D} d_{P,f}(\overline{x})}\]

But minimizing $d_{P,f}$ is the same as minimizing
$d_{P,f}^2 = (x_p^2 - 2x_p x + x^2) + (y_p^2 - 2y_p f(x) + f(x)^2)$.

In order to solve the minimal distance problem, Fermat's theorem
about stationary points will be tremendously usefull:

\begin{theorem}[Fermat's theorem about stationary points]\label{thm:fermats-theorem}
    Let $x_0$ be a local extremum of a differentiable function $f: \mathbb{R} \rightarrow \mathbb{R}$.

    Then: $f'(x_0) = 0$.
\end{theorem}

So in fact you can calculate the roots of $(d_{P,f}(x))'$ or $(d_{P,f}(x)^2)'$ to get
candidates for minimal distance.
$(d_{P,f}(x)^2)'$ is a polynomial if $f$ is a polynomial. So if $f$
is a polynomial, we can always get a finite number of candidates by
finding roots of $(d_{P,f}(x)^2)'$. But this gets difficult when $f$
has degree 3 or higher as explained in Theorem~\ref{thm:no-finite-solution}.
Another problem one has to bear in mind is that these candidates
include all points with minimal distance, but might also contain
more. Example~\ref{ex:false-positive} shows such a situation.

Let $S_n$ be the function that returns the set of solutions for a
polynomial $f$ of degree $n$ and a point $P$:

\[S_n: \Set{\text{Polynomials of degree } n \text{ defined on } \mdr} \times \mdr^2 \rightarrow \mathcal{P}({\mdr})\]
\[S_n(f, P) := \underset{x\in\mdr}{\arg \min d_{P,f}(x)} = M\]

If possible, I will explicitly give this function.


\subsection{Defing sine and cosine using Euler's formula}

\subsubsection{Euler's formula}

Previously we showed that:

\(e^x=\sum_{i=0}^\infty \dfrac{x^i}{i!}\)

Consider:

\(e^{i\theta }\)

\(e^{i\theta }=\sum_{j=0}^\infty \dfrac{(i\theta )^j}{j!}\)

\(e^{i\theta }=[\sum_{j=0}^\infty \dfrac{(\theta )^{4j}}{(4j)!}-\sum_{j=0}^\infty \dfrac{(\theta )^{4j+2}}{(4j+2)!}]+i[\sum_{j=0}^\infty \dfrac{(\theta )^{4j+1}}{(4j+1)!}-\sum_{j=0}^\infty \dfrac{(\theta )^{4j+3}}{(4j+3)!}]\)

We then use this to define \(\sin \) and \(\cos \) functions.

\(\cos (\theta ):=\sum_{j=0}^\infty \dfrac{(\theta )^{4j}}{(4j)!}-\sum_{j=0}^\infty \dfrac{(\theta )^{4j+2}}{(4j+2)!}\)

\(\sin (\theta ):=\sum_{j=0}^\infty \dfrac{(\theta )^{4j+1}}{(4j+1)!}-\sum_{j=0}^\infty \dfrac{(\theta )^{4j+3}}{(4j+3)!}\)

So:

\(e^{i\theta }=\cos (\theta )+i\sin (\theta )\)

\subsubsection{Alternative formulae for sine and cosine}

We know

\(e^{i\theta }=\cos (\theta )+i\sin (\theta )\)

\(e^{-i\theta }=\cos (\theta )-i\sin (\theta )\)

So

\(e^{i\theta }+e^{-i\theta }=cos (\theta )+i\sin (\theta )+\cos (\theta )-i\sin (\theta )\)

\(\cos (\theta )=\dfrac{e^{i\theta }+e^{-i\theta }}{2}\)

And

\(e^{i\theta }-e^{-i\theta }=cos (\theta )+i\sin (\theta )-\cos (\theta )+i\sin (\theta )\)

\(\sin (\theta )=\dfrac{e^{i\theta }-e^{-i\theta }}{2i}\)

\subsubsection{Sine and cosine are odd and even functions}

Sine is an odd function.

\(\sin (-\theta )=-\sin (\theta )\)

Cosine is an even function.

\(\cos (-\theta )=\cos (\theta )\)


\section*{Exercises}

\begin{ex}
  Find the matrix for the linear transformation that
  rotates every vector in $\R^2$ by an angle of $\pi /3$.
  \begin{sol}
    $\begin{mymatrix}{cc}
      \cos \paren{
        \frac{\pi}{3}} & -\sin \paren{\frac{\pi}{3}} \\
      \sin \paren{\frac{\pi}{3}} & \cos \paren{\frac{\pi}{3}}%
    \end{mymatrix} = \begin{mymatrix}{cc}
      \frac{1}{2} & -\frac{1}{2}\sqrt{3} \\
      \frac{1}{2}\sqrt{3} & \frac{1}{2}
    \end{mymatrix}$
  \end{sol}
\end{ex}

\begin{ex}
  Find the matrix for the linear transformation that reflects every
  vector in $\R^2$ about the $x$-axis.
\end{ex}

\begin{ex}
  Find the matrix for the linear transformation that reflects every
  vector in $\R^2$ about the line $y=-x$.
\end{ex}

\begin{ex}
  Find the matrix for the linear transformation that stretches $\R^2$
  by a factor of $3$ in the vertical direction.
\end{ex}

\begin{ex}
  Find the matrix of the linear transformation that reflects every
  vector in $\R^3$ about the $xy$-plane.
\end{ex}

\begin{ex}
  Find the matrix of the linear transformation that reflects every
  vector in $\R^3$ about plane $x=z$.
\end{ex}

\begin{ex}
  Describe the linear transformation that is given by each of the
  following matrices. Draw a before-and-after picture for each.
  \begin{equation*}
    (a)\quad
    A = \begin{mymatrix}{rr}
      1 & -1 \\
      1 &  1 \\
    \end{mymatrix},\quad
    (b)\quad
    B = \begin{mymatrix}{rr}
      0 & 2 \\
      1 & 0 \\
    \end{mymatrix},\quad
    (c)\quad
    C = \begin{mymatrix}{rr}
      1 & 0 \\
      -1 & 1 \\
    \end{mymatrix},\quad
    (d)\quad
    D = \begin{mymatrix}{rr}
      1 & 0 \\
      0 & 0 \\
    \end{mymatrix}.
  \end{equation*}
\end{ex}

\begin{ex}
  Let $\vect{u}=\begin{mymatrix}{r} a \\ b \end{mymatrix}$ be a unit
  vector in $\R^2$. Find the matrix%
  \index{reflection!about a given vector} that reflects all vectors
  about this vector, as shown in the following picture.
  \begin{center}
    \begin{tikzpicture}[rotate=45]
      \draw[->](0,0)--(2,1.5);
      \draw[blue, ->](0,0) -- (3,0) node [right] {$\vect{u}$};
      \draw[->](0,0)--(2,-1.5);
    \end{tikzpicture}
  \end{center}
  \begin{sol}
    First, we compute $\vect{v}'$, the projection of $\vect{v}$ onto
    $\vect{u}$.
    \begin{center}
      \begin{tikzpicture}[rotate=45]
        \draw[->](0,0) -- node[left] {$\vect{v}$} (2,1.5);
        \draw[blue, ->](0,0) -- (3,0) node [right] {$\vect{u}$};
        \draw[red, ->](0,0) --  node [above] {$\vect{v}'$} (2,0);
        \draw[->](0,0) -- node[below] {$T(\vect{v})$} (2,-1.5);
        \draw[dashed,gray] (2,1.5) -- (2,-1.5);
      \end{tikzpicture}
    \end{center}
    We have
    \begin{equation*}
      \vect{v}'
      = \proj_{\vect{u}} \vect{v}
      = \frac{\vect{u}\dotprod\vect{v}}{\norm{\vect{u}}^2}\,\vect{u}.
    \end{equation*}
    But since $\vect{u}$ is a unit vector, this simplifies to
    $\vect{v}' = (\vect{u}\dotprod\vect{v})\vect{u}$.
    From the above picture, we see that $T(\vect{v}) = \vect{v} +
    2(\vect{v}'-\vect{v}) = 2\vect{v}' - \vect{v} =
    2(\vect{u}\dotprod\vect{v})\vect{u} - \vect{v}$.
    To get the matrix of this linear transformation, we must compute
    the image of the standard basis vectors:
    \begin{equation*}
      T(\vect{e}_1) = 2(\vect{u}\dotprod\vect{e}_1)\vect{u} - \vect{e}_1
      = 2a\begin{mymatrix}{c} a\\b \end{mymatrix} - \begin{mymatrix}{c} 1\\0 \end{mymatrix}
      = \begin{mymatrix}{c} 2a^2-1\\2ab \end{mymatrix}
    \end{equation*}
    \begin{equation*}
      T(\vect{e}_2) = 2(\vect{u}\dotprod\vect{e}_2)\vect{u} - \vect{e}_2
      = 2b\begin{mymatrix}{c} a\\b \end{mymatrix} - \begin{mymatrix}{c} 0\\1 \end{mymatrix}
      = \begin{mymatrix}{c} 2ab\\2b^2-1 \end{mymatrix}.
    \end{equation*}
    Therefore, the matrix is
    \begin{equation*}
      A = \begin{mymatrix}{cc}
        2a^2-1 & 2ab \\
        2ab & 2b^2-1 \\
      \end{mymatrix}.
    \end{equation*}
  \end{sol}
\end{ex}


\chapter{Combinatorics}
\section{Basics}
\subsection{Considerations}
\begin{enumerate}
\item Does \textbf{order} matter?
\item Are the objects \textbf{repeatable}?
\item Are the objects partially \textbf{duplicated}?
\end{enumerate}}
If order does not matter, you can pre-set the order.
\subsection{Basic formula}
\begin{eqnarray*}
&& {n \choose k} = \frac{n!}{k!(n-k)!} \\
&& {n \choose k} = {n \choose n-k} \\
&& {n\choose k} = {n-1\choose k} + {n-1 \choose k-1}
\end{eqnarray*}

\subsection{N objects, K ceils}
When $N=10, K=3$:
$$
x_1 + x_2 + x_3 = 10
$$
is equivalent to
$$
*****|**|***
$$

, notice that $*$ are non-order, and it is possible to have
$$
*****||*****
$$
\\
then the formula is:
$$
{n+r \choose r}
$$

,where $r=k-1$.
\\
Intuitively, the meaning is to choose $r$ objects from $n+r$ objects to become the $|$.

\runinhead{Unique paths.} Given a $m \times n$ matrix, starting from $(0, 0)$, ending at $(m-1, n-1)$, can only goes down or right. What is the number of unique paths?

There are total

\subsection{N objects, K types} \label{N_objects_K_types}
What is the number of permutation of $N$ objects with $K$ different types:
\begin{align*}
ret &= \frac{A_N^N}{\prod_{k=1}^K{A_{sz(k)}^{sz(k)}}} \\
&= \frac{N!}{\prod_{k} sz[k]!}
\end{align*}

\subsection{Inclusion–Exclusion Principle}
\begin{figure}[hbtp]
\centering
\subfloat{\includegraphics[scale=0.50]{500px-Inclusion-exclusion}}
\caption{Inclusion–exclusion principl}
\label{fig:500px-Inclusion-exclusion}
\end{figure}
\begin{eqnarray*}
|A \cup B \cup C| = |A| + |B| + |C| \\ - |A \cap B| - |A \cap C| - |B \cap C| \\ + |A \cap B \cap C|
\end{eqnarray*}
Generally,
$$
\Biggl|\bigcup_{i=1}^n A_i\Biggr| = \sum_{k = 1}^{n} (-1)^{k+1} \left( \sum_{1 \leq i_{1} < \cdots < i_{k} \leq n} \left| A_{i_{1}} \cap \cdots \cap A_{i_{k}} \right| \right)
$$
\section{Combinations with Duplicated Objects}
Determine the number of combinations of 10 letters (order does not matter) that can be formed from 3A, 4B, 5C.

\subsection{Basic Solution}
If there are no restrictions on the number of any of the letter, it is ${10+2 \choose 2}$; then we get the universal set,
$$
|U|={10+2 \choose 2}
$$

Let $P_A$ be the set that a 10-combination has more than 3A. $P_B$...4B. $P_C$...5C.

The result is:
\begin{align*}
|3A \cap 4B \cap 5C| = & |U|\\
& - sum(|P_i|\cdot \forall i) \\
& + sum(|P_i \cap P_j|\cdot \forall i,j)\\
& - sum(|P_i \cap P_j \cap P_k|\cdot \forall i,jk)
\end{align*}

To calculate $|P_i|$, take $|P_A|$ as an example. \textbf{Pre-set} 4A -- if we take any one of these 10-combinations in $P_A$ and remove 4A we are left with a 6-combination with unlimited on the numbers of letters; thus,
$$
|P_A|={6+2 \choose 2}
$$

Similarly, we can get $P_B, P_C$.

To calculate $|P_i \cap P_j}|$, take $|P_A \cap P_B|$ as an example. \textbf{Pre-set} 4A and 5B; thus,
$$
|P_A \cap P_B| = {1+2 \choose 2}
$$

Similarly, we can get other $|P_i \cap P_j|$.

Similarly, we can get other $|P_i \cap P_j \cap P_k|$.
\subsection{Algebra Solution}
The number of 10-combinations that can be made from 3A, 4B, 5C is found from the coefficient of $x^{10}$ in the expansion of:
$$
(1+x+x^2+x^3)(1+x+x^2+x^3+x^4)(1+x+x^2+x^3+x^4+x^5)
$$

And we know:
\begin{eqnarray*}
1+x+x^2+x^3         = (1-x^4)/(1-x)  \\
1+x+x^2+x^3+x^4     = (1-x^5)/(1-x)  \\
1+x+x^2+x^3+x^4+x^5 = (1-x^6)/(1-x)  \\
\end{eqnarray*}

We expand the formula, although the  naive way of getting the coefficient of $x^{10}$ is tedious.

\section{Permutation}

\subsection{$k$-th permutation}
Given $n$ and $k$, return the $k$-th permutation sequence. $k\in [1, n!]$. $O(nk)$ in time complexity is easy, can you do it in $O(n^2)$ or less?

Reversed Cantor Expansion

Core clues:
\begin{enumerate}
\item \pyinline{A = [1, 2, ..., n]}

Suppose for $n$ element, the $k$-th permutation is:

\pyinline{ret = [a0, a1, a2, ..., an-1]}
\item \rih{Basic case.} Since \pyinline{[a1, a3, ..., an-1]} has $(n-1)!$ permutations,
if $k < (n-1)!, a_0 = A_0$ (first element in array), else $a_0 = A_{k/(n-1)!}$

\item Recursively, (or iteratively), calculate the values at each position. Similar to Radix.
\begin{enumerate}
\item $a_0 = A_{k_0/(n-1)!}$, where $k_0 = k$
\item $a_1 = A_{k_1/(n-2)!}$, where $k_1 = k_0\%(n-1)!$ in the remaining array $A$
\item $a_2 = A_{k_2/(n-3)!}$, where $k_2 = k_1\%(n-2)!$ in the remaining array $A$
\end{enumerate}
\end{enumerate}
\begin{python}
def getPermutation(self, n, k):
    k -= 1  # start from 0

    A = range(1, n+1)
    k %= math.factorial(n)
    ret = []
    for i in xrange(n-1, -1, -1):
        idx, k = divmod(k, math.factorial(i))
        ret.append(A.pop(idx))

    return "".join(map(str, ret))
\end{python}

\subsection{Numbers counting}
\runinhead{Count numbers with unique digit.} Given a non-negative integer n, count all numbers with unique digits, $x$, where $0 \leq x < 10^n$.

Digit by digit:
\begin{enumerate}
\item The 1st digit has 10 possibilities. The 2nd digit has 9 possibilities. Therefore it seems to be $A_{10}^n$.
\item Exception: The first digit cannot be 0. Therefore it is $9\times 9\times 8\times ...\times (10-i)$
\end{enumerate}

\section{Catalan Number}\label{section:catalanNumber}
\subsection{Math}
\runinhead{Definition.}
$$
C_n = {2n\choose n} - {2n\choose n+1} = {1\over n+1}{2n\choose n} \quad\text{ for }n\ge 0
$$
\runinhead{Proof.} Proof of Calatan Number $C_n ={2n\choose n} - {2n\choose n+1}$. Objective: count the number of paths in $n\times n$ grid without exceeding the main diagonal.
\begin{itemize}
\begin{figure}[]
    \centerline{\includegraphics[height = 1.6in]{catalan_proof}}
    \caption{Monotonic Paths}
  \label{fig:catalanProof}
\end{figure}
\item monotonic paths - $n$ right, $n$ up
$$
{2n\choose n}
$$
\item flip at the line just above the diagonal line - $n-1$ right, $n+1$ up
$$
{n-1+n+1\choose n-1}
$$
\item thus, the number of path without \textit{exceedance} (i.e. passing the diagonal line) is:
\begin{align*}
C_n &= {2n\choose n} - {2n\choose n-1}\\
&={2n\choose n} - {2n\choose n+1}
\end{align*}
\end{itemize}

\subsection{Applications}
The paths in Figure \ref{fig:catalanProof} can be abstracted to anything that at any time \#right $\geq$ \#up.
\runinhead{\#Parentheses.}Number of different ways of adding parentheses. At any time, \#\pyinline{(} $\geq$ \#\pyinline{)}.
\runinhead{\#BSTs.}Number of different BSTs. Consider it as a set of same binary operators with their operands. Reduce this problem to \#Parentheses.
\begin{figure}[hbtp]
\centering
\subfloat{\includegraphics[scale=2.00]{Catalan_number_binary_tree_example}}
\caption{\#BSTs. Circles are operators; crescents are operands.}
\label{fig:NumberOfBSTs}
\end{figure}

\section{Stirling Number}
a Stirling number of the second kind (or Stirling partition number) is the number of ways to partition a set of n objects into k non-empty subsets and is denoted by $S(n,k)$ or  $\lbrace{n\atop k}\rbrace$.

$$
\left\{ {n \atop k}\right\} = \frac{1}{k!}\sum_{j=0}^{k} (-1)^{k-j} \binom{k}{j} j^n.
$$


\subsection{Bounds on eigenvalues}

\begin{Lemma}{bound-by-norm}
  Let $\norm{\cdot}$ be a vector norm and denote by the same symbol
  a consistent norm for matrices. Then, for any matrix $\mata\in\Cnn$
  and for any eigenvalue $\lambda\in\sigma(\mata)$ there holds
  \begin{gather}
    \abs{\lambda} \le \norm{\mata}.
  \end{gather}
\end{Lemma}

\begin{Lemma}{pre-gershgorin}
  Let $\mata,\matb\in\Cnn$ and let $\norm{\cdot}$ be an operator norm
  on the space of matrices corresponding to a vector norm denoted by
  $\norm{\cdot}$ as well. Then, for any eigenvalue
  $\lambda\in\sigma(\mata)$ such that $\lambda\not\in\sigma(\matb)$
  there holds
  \begin{gather}
    \norm*{(\lambda\id-\matb)^{-1}(A-B)} \ge 1.
  \end{gather}
\end{Lemma}

\begin{Theorem*}{gershgorin}{Gershgorin circle theorem}
  All eigenvalues of a matrix $\mata\in\Cnn$ are contained in the
  union of the \define{Gershgorin Circle}s
  \begin{gather}
    G_j = \left\{ z\in \C \middle| \abs{z-a_{jj}} \le \sum_{k\neq j} \abs{a_{jk}}\right\}.
  \end{gather}
  Furthermore, if there is a subset of $m$ circles disjoint from the
  other circles, then this subset contains $m$ eigenvalues.
\end{Theorem*}

\subsection{The Rayleigh quotient}

\begin{Definition}{rayleigh-quotient}
  For a matrix $\mata\in\Cnn$ and a vector $\vx\in\C^n$, the
  \define{Rayleigh quotient} is defined as
  \begin{gather}
    R_\mata(\vx) = \frac{\scal(\mata\vx,\vx)}{\scal(\vx,\vx)}.
  \end{gather}
\end{Definition}

\begin{Theorem*}{minmax}{Courant-Fischer min-max theorem}
  Let $\mata\in\Cnn$ be Hermitian with eigenvalues
  $\lambda_1 \le \lambda_2\le\dots\le \lambda_n$. Then, for $k=1,\dots,n$
  \begin{align}
    \lambda_k
    &= \min_{\substack{V \subset \C^n\\\dim V = k}} \max_{\vx\in V} R_\mata(\vx),\\
    &= \max_{\substack{V \subset \C^n\\\dim V = n-k+1}} \min_{\vx\in V} R_\mata(\vx).
  \end{align}
  In particular,
  \begin{gather}
    \lambda_{\min}(\mata) = \min_{\vx\in\C^n} R_\mata(\vx),
    \qquad
    \lambda_{\max}(\mata) = \max_{\vx\in\C^n} R_\mata(\vx).
  \end{gather}
\end{Theorem*}

\subsection{Conditioning of the eigenvalue problem}
In this section, we study the conditioning of finding eigenvalues and
eigenvectors. While we will not cover the full theory, we will provide
examples for ill-posed problems as well as exemplary proofs for
well-posedness.

In all cases, we will investigate the change of eigenvalues or
eigenvectors when the matrix $\mata$ is perturbed by a small matrix
$\mate$ of norm $\epsilon$.

\begin{Example}{characteristic-polynomial}
  Take a matrix of dimension 20 with eigenvalues $1,2,\ldots,20$. Its
  characteristic polynomial is
  \begin{gather}
    \chi(\lambda) = (\lambda-1)\dots(\lambda-20).
  \end{gather}
  The coefficient in front of $\lambda^{20}$ is one, the constant term is $20! > 10^{19}$.
  We perturbe it in the form
  \begin{gather}
    \tilde \chi(\lambda) = \chi(\lambda) - 10^{-23}\lambda^{19}.
  \end{gather}
  Their greatest roots are
  \begin{gather}
    \begin{array}{l@{\qquad}l@{\,}c@{\,}l}
      \multicolumn{1}{c}{\chi}&
      \multicolumn{3}{c}{\tilde \chi}\\
      20&20.847\\
      19,18&19.502&\pm&1.940i\\
      17,16&16.731&\pm&2.813i\\
      15,14&13.992&\pm&2.519i\\
    \end{array}
  \end{gather}
  {\tiny Source: \cite{DeuflhardHohmann08}}
\end{Example}

\begin{Example}{conditioning-Jordan-block}
  Consider the matrix
  \begin{gather}
  \mata_\epsilon =
      \begin{pmatrix}
        0&1\\
        &0&1\\
        &&\ddots&\ddots\\
        &&&0&1\\
        \epsilon &&&&0
      \end{pmatrix}
      \in\C^{n\times n},
  \end{gather}
  For $\epsilon=0$, it has a single eigenvalue of geometric multiplicity one and algebraic multiplicity $n$.

  For $\epsilon>0$, it has $n$ simple eigenvalues
  \begin{gather}
      \lambda_j = \sqrt[n]{\epsilon} \,e^{2\frac jni\pi}
  \end{gather}
\end{Example}

\begin{proof}
  For $\epsilon=0$, the matrix is the generic Jordan-block of an eigenvalue which is not semi-simple, thus the ill-posedness of this example implies the ill-posedness for not semi-simple eigenvalues in the general case. Note that this statement holds notwithstanding that special perturbations may be benign.

  The characteristic polynomial of this matrix is
  \begin{gather}
      \chi(\lambda) = \det(\mata-\lambda\id)
      = \det\begin{pmatrix}
      -\lambda&1\\
        &\ddots&\ddots\\
        &&-\lambda&1\\
        \epsilon &&&-\lambda
      \end{pmatrix}.
  \end{gather}
  Applying Laplace expansion to the first column yields
  \begin{gather}
      \chi(\lambda)
      = -\lambda \det\begin{pmatrix}
        -\lambda&1\\
        &\ddots&\ddots\\
        &&-\lambda&1\\
        &&&-\lambda
      \end{pmatrix}
      + (-1)^{n+1} \epsilon\det\begin{pmatrix}
        1 \\
        -\lambda &1\\
        &\ddots&\ddots\\
        &&-\lambda&1
      \end{pmatrix},
  \end{gather}
  where both matrices are of dimension $n-1$. Since they are triangular, recursion of Laplace expansion is particularly simple and yields the product of the diagonal elements. Thus
  \begin{gather}
      \chi(\lambda) = (-1)^n \lambda^n
      + (-1)^{n+1} \epsilon.
  \end{gather}
  Its roots are determined by the condition
  \begin{gather}
      \lambda^n = \epsilon.
  \end{gather}
  Thus, $\lambda$ can be computed as an $n$th root of unity times the (real) $n$th root of $\epsilon$.
\end{proof}

\begin{Theorem}{Jordan-block-ill-conditioned}
  The eigenvalue problem for eigenvalues which are not semi-simple is
  in general ill-posed.
\end{Theorem}

\begin{proof}
  The analysis in \slideref{Example}{conditioning-Jordan-block} is
  generic in the sense that it applies to nonzero eigenvalues and also
  to matrices which are similar to such a block. Thus, we can conclude
  that for every matrix $\mata$ which is similar to a matrix with a
  nontrivial Jordan block for eigenvalue $\lambda$, there is a
  perturbation $\mate$ such that the derivative of the function
  $\lambda(\epsilon) = \lambda(A+\epsilon\mate)$ at zero is unbounded.
\end{proof}

\begin{Theorem*}{bauer-fike}{Bauer-Fike}
  Let $\mata\in \Cnn$ be diagonalizable with matrix of eigenvectors
  $\matv \in \Cnn$ and diagonal matrix
  $\matlambda = \diag(\lambda_1\dots,\lambda_n)$. Let $\mata+\mate$ be
  a perturbation of $\mata$. Then, for any eigenvalue $\mu$ of
  $\mata+\mate$, there is an eigenvalue $\lambda_i$ of $\mata$ such
  that
  \begin{gather}
    \abs{\mu-\lambda_i} \le \cond_2(\matv) \norm{\mate}_2.
  \end{gather}
\end{Theorem*}

\begin{proof}
  Wikipedia
\end{proof}

\begin{Corollary}{conditioning-eigenvalues-normal}
  The eigenvalue problem of a normal matrix $\mata\in\Cnn$ is
  well-conditioned in the sense that for every eigenvalue $\mu$ of the
  perturbed matrix $\mata+\mate$, there is an eigenvalue $\lambda$ of
  $\mata$ such that
  \begin{gather}
    \abs{\mu-\lambda} \le \norm{E}_2.
  \end{gather}
\end{Corollary}

The Bauer-Fike theorem provides a general estimate for diagonalizable
matrices in terms of the condition number of the matrix of
eigenvectors. The following theorem is less general, since it only
applies to simple eigenvalues, but it provides geometric intuition of
the issue.

\begin{Theorem}{conditioning-eigenvalue-single}
  Let $\mata_\epsilon = \mata+\epsilon\mate\in\Cnn$ be a perturbation
  of $\mata\in\Cnn$. Let $\lambda(0)$ and let $\lambda(0)$ be a single
  eigenvalue of $\mata$. Then, there exists a uniquely defined
  differentiable continuation $\lambda(\epsilon)$ for small $\epsilon$
  such that $\lambda(\epsilon) \in \sigma(\mata_\epsilon)$ and with
  its left and right eigenvectors $u$, and $w$, respectively, there
  holds
  \begin{gather}
    \abs*{\tfrac{d}{d\epsilon} \lambda(0)}
    \le \norm{E}_2\frac{\norm{u}_2\norm{w}_2}{\abs{\scal(u,w)}}
    = \norm{E}_2 \frac1{\cos\angle(u,w)}.
  \end{gather}
\end{Theorem}

\subsection{Conditioning of eigenvectors and eigenspaces}

\begin{intro}
  Positive results on the conditioning of eigenvectors require
  additional tools which go beyond the exposition planned for this
  class. We will thus only discuss this question at hand of an example
  and conclude a rule of thumb.
\end{intro}

\begin{Example}{conditioning-eigenvectors}
  Consider the two matrices
  \begin{gather}
    A =
    \begin{pmatrix}
      1-\epsilon & 0\\ 0 & 1+\epsilon
    \end{pmatrix},
    \qquad
    B =
    \begin{pmatrix}
      1&\epsilon\\\epsilon&1
    \end{pmatrix}.
  \end{gather}
  Their eigenvalues are $1-\epsilon$ and $1+\epsilon$, but their
  eigenvectors differ by an angle of $\pi/4$ independent of
  $\epsilon$.
\end{Example}

\begin{Remark}{conditioning-eigenvectors}
  The problem of finding eigenvectors for tight clusters of
  eigenvalues is ill-posed. Nevertheless, finding the invariant
  subspace associated to all eigenvalues in such a cluster is
  well-posed.

  Conditioning of the eigenvector problem depends on the separation of
  eigenvalues.
\end{Remark}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End:


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS624: Analysis of Algorithms
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 9}

Let us guess that perhaps $T(n)$ can be expressed as an affine function of $n$:
\begin{equation}
T(n) = \alpha n + \beta
\end{equation}
We have already seen that this is a reasonable guess.
Substitute equation 3 in the equation 1 and 2 and solve the two resulting equations $\alpha$ and $\beta$.

\subsection*{Solution}

To obtain an intuition, we begin by formulating $T(n)$ for $n = 1$ and $n=2$.

If $n = 1$, there are no nodes in left and right subtrees.
\begin{equation}
\begin{aligned}
T(1) &= c + T(0) + T(0) + v\\
&= (2c + v) \times 1 + c
\end{aligned}
\end{equation}
If $n = 2$, there is 1 node in one subtree and no node in the other subtree.
\begin{equation}
\begin{aligned}
T(2) &= c + T(1) + T(0) + v\\
&= (2c + v) \times 2 + c
\end{aligned}
\end{equation}
Therefore, we prove the statement by induction on number of nodes $n$.
We form the inductive hypothesis as for a tree with $k$ nodes ($k \leq k_t$), $T(n)$ is as described as in Equation \ref{eq93}.
\begin{equation}
T(k) = (2c + v) \times k + c
\label{eq93}
\end{equation}
Now we show that the statement holds true for $k = k_t + 1$.
In this case, we assume there are $l$ nodes in the left subtree and $r = k_t - l - 1$ nodes in the right subtree.
Since $l, r \leq k$, Equation \ref{eq94} would be valid.
\begin{equation}
\begin{aligned}
T(k) &= c + T(l) + T(r) + v\\
&= c + (2c + v) \times l + c + (2c + v) \times r + c + v\\
&= (2c + v) + (2c + v)(l + r) + c\\
&= (2c + v) \times k + c
\end{aligned}\label{eq94}
\end{equation}
Therefore,
\begin{align}
\alpha &= 2c + v\\
\beta &= c
\end{align}\label{eq95}


\subsection{Chain Rule}
\begin{lemma}
	Let $f$ and $g$ be differentiable functions. Then
	\begin{equation}
		\dd{}{x}f(g(x)) = f^\prime(g(x))g^\prime(x).
	\end{equation}
	Equivalently, if $f$ is a function of $g$ and $g$ is a function of $x$,
	\begin{equation}
		\dd{f}{x} = \dd{f}{g}\hspace{3pt}\dd{g}{x}.
	\end{equation}
\end{lemma}
\begin{proof}
	Applying the limit definitions of the derivative,
	\begin{align*}
		\dd{f}{x} &= \lim_{h \to 0}{\frac{f(g(x+h))-f(g(x))}{h}} \\
		&= \lim_{h\to 0}{\frac{f(g(x+h))-f(g(x))}{g(x+h)-g(x)}\hspace{3pt}\frac{g(x+h)-g(x)}{h}} \\
		&= \lim_{h\to 0}{\frac{f(g(x+h))-f(g(x))}{g(x+h)-g(x)}} \hspace{3pt} \lim_{h\to 0}{\frac{g(x+h)-g(x)}{h}} \\
		&= \left(\lim_{h\to 0}{\frac{f(g(x+h))-f(g(x))}{g(x+h)-g(x)}}\right) \hspace{3pt} \dd{g}{x} \\
		&= \dd{f}{g}\hspace{3pt}\dd{g}{x}.
	\end{align*}
\end{proof}

\begin{example}
	Find the derivative of $y = (x^2 + 1)^5$ using the chain rule.
\end{example}
\begin{answer}
	$y$ is a composition of the two functions $x^5$ and $x^2 + 1$.
	Applying the chain rule,
	\begin{equation*}
		y^\prime = \dd{}{(x^2+1)}(x^2+1)^5 \hspace{3pt} \dd{}{x}(x^2+1)
	\end{equation*}

	Making the substitution $u = x^2 + 1$,
	\begin{align*}
		y^\prime &= \dd{}{u}u^5 \hspace{3pt} \dd{}{x}(x^2+1) \\
		&= 5u^{4}2x
	\end{align*}

	Substituting back,
	\begin{align*}
		y^\prime &= 5(x^2+1)^{4}2x \\
		&= 10x(x^2+1)^4.
	\end{align*}
\end{answer}

\subsubsection{$u$ Substitutions}
As we did in the example, we can substitute a variable, usually called $u$ when applying the chain rule.
\begin{example}
	Given that the derivative of $\sin{x}$ is $\cos{x}$, find the derivative of $f(x)=\sin^5{x}$.
\end{example}
\begin{answer}
	$f$ is a composition of $x^5$ and $\sin$.
	Substituting $u=\sin{x}$, we can rewrite $f$ as $u^5$.
	Applying the chain rule,
	\begin{align*}
		\dd{f}{x} &= \dd{f}{u} \hspace{3pt} \dd{u}{x} \\
		&= 5u^4 \hspace{3pt} \cos{x} \\
		&= 5\sin^4{x}\cos{x}.
	\end{align*}
\end{answer}


\subsection{Gradient Descent}
\noindent
Remember that if we have a multidimensional function, taking a step in the direction of the gradient results in the maximum possible increase of the function, and taking a step in the opposite direction of the gradient results in the maximum possible decrease of the function. Gradient descent is a method to find minima of functions.\\

\noindent
Let's say we're trying to minimize $J(\vec{x})$ with gradient descent. Here are the steps we would take:
\begin{enumerate}
	\item Pick (or guess) a starting point $\vec{x_0}$ and a learning rate (step size) $\delta$.
	\item $\vec{x_{n+1}} = \vec{x_n} - \delta J(\vec{x_n})$
	\item Repeat step 2 until some stopping criteria is met, like $\norm{\delta \nabla J(\vec{x_n}) - \delta \nabla J(\vec{x_{n+1}})}<.01$.
\end{enumerate}

[INSERT IMAGE]

\noindent
This method will lead you arbitrarily close to a local minimum, but does not guarantee finding the global minimum. More advanced versions of gradient descent exists that try to help with this, like giving the point “momentum” to be able to move out of local mins.
This method also has a trade off between speed and accuracy. Although increasing $\delta$ means fewer iterations of gradient descent are needed to narrow in on a local minimum, one is more likely to be stuck in a local min than they had used a smaller $\delta$.\\

\noindent
In the real world, the function you are trying to minimize will likely not be well defined enough to take its partial derivatives to find the gradient, so they too are approximated by doing something like
\begin{equation*}
	J_{k} = \frac{J(k+.0001, \ldots) - J(k, \ldots)}{.0001}
\end{equation*}


\subsection{Cartesian product}

The cartesian product takes two sets, and creates a set containing all ordered pairs of \(a\) and \(b\).

\(a\times b\)


\section{Adam algorithm}
Adam (Kingma and Ba, 2014) is yet another adaptive learning rate optimization algorithm. Adam optimization is an extension to stochastic gradient decent and can be used in place of classical stochastic gradient descent to update network weights more efficiently. The name ``Adam'' derives from the phrase \emph{Adaptive Moment Estimation}, which adapts both learning rate $\eta$ and subgradient $g$.

Let $g$ be the gradient of the stochastic objective $f$ with parameters $x$. We wish to estimate its  first order moment vector (the mean of gradients), and second order moment vector (the uncentered variance of gradients)  using an exponential moving average of the squared gradient, with decay rate $\beta_1$ and $\beta_2$, respectively. Let $g_1$, $g_2$, $\cdots$, $g_T$ be the gradients at subsequent timesteps.
Use moving average of both $g$ and $g^2$ (here $g^2$ indicates the element-wise square% $g \bigodot g$
) to compute first order moment vector
\begin{equation}
m_t  =\sum_{i=1}^{t} w_i g_i,\quad w_i={\beta_1^{t-i} \over \sum_{i=1}^t\beta_1^{t-i}},
\end{equation}
and second order moment vector
\begin{equation}
v_t  =\sum_{i=1}^{t}r_i g^2_i,\quad r_i={\beta_2^{t-i} \over \sum_{i=1}^t\beta_2^{t-i}}.
\end{equation}
Here $\beta_1 = 0.9$, $\beta_2 = 0.99$.

The iterate can be written as
\begin{equation}\label{Adam}
x_{t+1}=x_t-\frac{\eta}{\sqrt{v_t}+\delta} m_t,
\end{equation}
where $\eta$ is stepsize, $\delta$ is some small constant and $x$ is the parameter vector. Another way to represent the first and second order moment vector  is
\begin{align}
m_t=\frac{\tilde m_t}{1-\beta_1^t} &\mbox{ with }\ \tilde m_t  =\beta_1 \tilde m_{t-1} + (1-\beta_1) g_t,\\
v_t=\frac{\tilde v_t}{1-\beta_2^t}&\mbox{ with }\ \tilde v_t  =\beta_2 \tilde v_{t-1} + (1-\beta_2) g_t^2,
\end{align}
which is usually used in implementation as in the following algorithm.

\begin{algorithm}[H]
\caption{Adam Optimization}
\label{alg:Adam}
{\bf Input}:
\begin{enumerate}
\item small constant $\delta$ (can be $10^{-8}$),
\item decay rate for moment estimates $\beta_1$ and $\beta_2$ (suggested defaults: 0.9 and 0.999 respectively)
\item initial first moment vector $m_0=0$ and initial second moment vector $v_0=0$
\item initial timestep $t=0$, initial parameter vector $x_0$ and stepsize $\eta$
\end{enumerate}
{\bf Steps}:

Compute the gradient on $B_{i_t}$:
\begin{equation}
g_t = \nabla_{x} \frac{1}{m} \sum_{i \in B_{i_t}} f_i(x_{t}),\quad B_{i_t} \subset \{1, 2, \cdots, N \},\ \#B_{i_t} =m
\end{equation}
Update biased first moment estimate:
\begin{equation}
m_t = \beta_1 m_{t-1} + (1-\beta_1)g_t.
\end{equation}
Update biased second moment estimate:
\begin{equation}
v_t = \beta_2 v_{t-1} + (1-\beta_2)g_t \circ g_t.
\end{equation}
Correct bias in first moment:
\begin{equation}
\tilde m_t = \frac{m_t}{1 - \beta_1^{t}}.
\end{equation}
Correct bias in second moment:
\begin{equation}
\tilde v_t = \frac{v_t}{1 - \beta_2^{t}}.
\end{equation}
Compute update:
\begin{equation}
\Delta x_t =  -\frac{\eta}{\sqrt{\tilde v_t} + \delta} \otimes \tilde m_t
\end{equation}
Update $w$:
\begin{equation}
x_{t+1} = x_t + \Delta x_t.
\end{equation}
\end{algorithm}


Now we prove from the Well Ordering principle the Principle of Mathematical Induction. The idea will be as follows: we will consider the \textit{smallest} element such that a proposition is false, and then we will use the assumptions of the Priniciple of Mathematical Induction to derive contradictions. Hence the Principle of Mathematical Induction is true as stated.

In fact, if we assume the Principle of Mathematical Induction as a given statement, we can use it to show the well ordering principle of the natural numbers. But to give a rigorous proof we need the Principle of Strong Induction, which we will prove from the Principle of Weak Induction later.

\begin{theorem}
Suppose that $P: \mathbb{N} \to \{T, F\}$ is a boolean function with the following properties:
\begin{itemize}
	\item $P(1) \equiv T$
	\item $P(n) \equiv T \implies P(n + 1) \equiv T$ for all $n \in \mathbb{N}$.
\end{itemize}
Then $P(n) \equiv T$ for all $n \in \mathbb{N}$.
\end{theorem}

\begin{proof}
Assume for the sake of contradiction that there is some $s \in \mathbb{N}$ such that $P(s) \equiv F$. Let $S$ be the set of all $s$ where $P(s) \equiv F$. By assumption, $S \neq \varnothing$, and hence by the well ordering principle there is a minimal element $r \in S$. We see that $r \neq 1$ because $P(1) \equiv T$ by assumption. So $r - 1 \in \mathbb{N}$. But since $r \in S$ is minimal, we must have that $P(r - 1) \equiv T$ (or else our assumption about $r$ being the minimal element in $S$ would be wrong). But since $P(r - 1) \equiv T$, this implies that $P(r) \equiv T$ (by modus ponens). This is a contradiction and the conclusion follows.
\end{proof}

To prove the other direction (namely, that the Principle of Mathematical Induction implies the Well Ordering Principle), we need the stronger principle of Strong Induction. So what we will do is use the Principle of Mathematical Induction to prove the Principle of Strong Induction first.

\begin{theorem}
Suppose that the Principle of Mathematical Induction is true. Suppose that $P: \mathbb{N} \to \{T, F\}$ is a boolean function with the following properties:
\begin{itemize}
	\item $P(1) \equiv T$
	\item If $P(k) \equiv T$ for all $1 \leq k \leq n$ for $n \geq 1$, then $P(n + 1) \equiv T$.
\end{itemize}

Then $P(n) \equiv T$ for all $n \in \mathbb{N}$.
\end{theorem}

\begin{proof}
Define an auxiliary function $Q: \mathbb{N} \to \{T, F\}$ by the following:
\[Q(n) \equiv \begin{cases} T & \text{$P(k) \equiv T$ for $1 \leq k \leq n$}\\ F & \text{otherwise.}\end{cases}\]
Then by assumption, $Q(1) \equiv T$ and $Q(k) \equiv T$ implies that $P(n + 1)$ is true, which together with $Q(n)$ implies that $Q(n + 1) \equiv T$. Hence by the Principle of Mathematical Induction we have that $Q(n)$ is true for all $n \in \mathbb{N}$ which implies that $P(n)$ is true for all $n \in \mathbb{N}$, as desired.
\end{proof}

The converse statement is also true, this will be left as an exercise. Now that we have the principle of strong induction, we can use it now to prove the well ordering principle. This will show that the Well Ordering Principle and the Principle of Mathematical Induction are equivalent.

\begin{theorem}
The Strong Induction Principle implies the Well Ordering Principle. Hence the Well Ordering Principle and the Principle of Mathematical Induction are equivalent.
\end{theorem}
\begin{proof}
Suppose that $S \subset \mathbb{N}$ is a non-empty set with no least element. Let $P$ be the following proposition:
\[P(n) = \begin{cases}T & n \not\in S \\ F & n \in S\end{cases}.\]
So the proposition $P(n)$ reflects whether or not $n$ is in the set $S$. Since $1$ is the least element in $\mathbb{N}$, we know that $1 \not\in S$ or else $S$ would have a least element. So $P(1) \equiv T$. Now suppose for any $n \in \mathbb{N}$ that $P(k) \equiv T$ for $1 \leq k \leq n$. Then by the definition of $P$ no natural number less than or equal to $n$ is in $S$. It follows that $n + 1 \not\in S$ or else $n + 1$ would be the least element in $S$. It follows that $P(n + 1) \equiv T$.

By the Principle of Strong Induction it follows that $P(n) \equiv T$ for all $n \in \mathbb{N}$. This implies that no element $n \in \mathbb{N}$ is in $S$. So $S = \varnothing$. The Well Ordering Principle follows, as desired.
\end{proof}

\section{Exercises}
\input{ch5/wopIndExercises}


\subsection{Axiom of extensionality}

If two sets contain the same elements, they are equal.

\(\forall x \forall y[\forall z(z\in x \leftrightarrow z\in y)\rightarrow x=y]\)

This is an axiom, not a definition, because equality was defined as part of first-order logic.

Note that this is not bidirectional. \(x=y\) does not imply that \(x\) and \(y\) contain the same elements. This is appropriate as \(\dfrac{1}{2}= \dfrac{2}{4}\) for example, even though they are written differently as sets.

\subsubsection{Reflexivity of equality}

The reflexive property is:

\(\forall x(x=x)\)

We can replace the instance of \(y\)  with \(x\):

\(\forall x [\forall z(z\in x \leftrightarrow z\in x)\rightarrow x=x]\)

We can show that the following is true:

\(\forall z(z\in x \leftrightarrow z\in x)\)

Therefore:

\(\forall x [T \rightarrow x=x]\)

\(x=x\)

\subsubsection{Symmetry of equality}

The symmetry property is:

\(\forall x \forall y[(x=y)\leftrightarrow (y=x)]\)

We know that the following are true:

\(\forall x \forall y[\forall z(z\in x \leftrightarrow z\in y)\rightarrow x=y]\)

\(\forall x \forall y[\forall z(z\in x \leftrightarrow z\in y)\rightarrow y=x]\)

So:

\(\forall x \forall y[\forall z(z\in x \leftrightarrow z\in y)\rightarrow (x=y\land y=x)]\)

\subsubsection{Transitivity of equality}

The transitive property is:

\(\forall x \forall y \forall z[(x=y \land y=z) \rightarrow x=z]\)

\subsubsection{Substitution for functions}

The substitutive property for functions is:

\(\forall x \forall y[(x=y)\rightarrow (f(x)=f(y))]\)

\subsubsection{Substitution for formulae}

The substitutive property for formulae is:

\(\forall x \forall y[((x=y)\land P(x))\rightarrow P(y)]\)

Doesn’t this require iterating over predicates? Is this possible in first order logic??

\subsubsection{Result 1: The empty set is unique}

We can now show the empty set is unique.

\subsubsection{Result 2: Every element of a set exists}

If an element did not exist, the set containing it would be equal to a set which did not contain that element.

\subsubsection{Result 3: Sets are unique}


\section{Formula Sheet}
\subsection{Basics}
\paragraph{Standard Deviation} $$SD(x) = \sqrt{\frac{1}{n-1}\Sigma_i (x_i - \bar{x})^2}$$
\paragraph{Variance} $$Var(x) = \frac{1}{n-1}\Sigma_i (x_i - \bar{x})^2$$
\paragraph{Covariance} $$Cov(x,y) = \frac{1}{n-1}\Sigma_i (x_i - \bar{x})(y_i - \bar{y})$$
\paragraph{Normal Distribution} $$f(x) = \frac{1}{\sigma \sqrt{2\pi}}e^{-\frac{(x-\mu)^2}{2\sigma^2}}$$
standardized: $$Z = \frac{X - \mu}{\sigma}$$
\paragraph{Probability density function (pdf): f(x)} $$f(x) = P(X = x)$$

\paragraph{Cumulative distribution fuction (cdf): F(x)} $$F(x) = P(X \leq x)$$ $$P(X \geq x) = 1 - F(x)$$  $$P(x_1 \leq X \leq x_2) = F(x_2) - F(x_1)$$
\paragraph{Confidence Interval} $$CI = \left[ \bar{X} - z_{(1 - \frac{\alpha}{2})}\cdot \frac{\sigma}{\sqrt{n}} , \bar{X} + z_{(1 - \frac{\alpha}{2})}\cdot \frac{\sigma}{\sqrt{n}} \right] $$
$$CI = \left[ \bar{X} - t_{(1 - \frac{\alpha}{2})}\cdot \frac{s}{\sqrt{n}} , \bar{X} + t_{(1 - \frac{\alpha}{2})}\cdot \frac{s}{\sqrt{n}} \right] $$
\subsection{Linear Regression}
\subsubsection{First Order Linear Model}

$$Y = \beta_0 + \beta_1 X + \varepsilon$$

\paragraph{OLS Estimator}

\begin{align*}
	\hat{\beta}_1 &= \dfrac{\frac{1}{n-1}\Sigma_i (x_i - \bar{x})(y_i - \bar{y})}{\frac{1}{n-1}\Sigma_i (x_i - \bar{x})^2} = \frac{Cov(x,y)}{Var(x)} \\ \ \\
	\hat{\beta}_0 &= \bar{y} - \hat{\beta}_1\cdot \bar{x} \\ \ \\
	\hat{y} &= \hat{\beta}_0 + \hat{\beta}_1 x
\end{align*}

\subsubsection{Multiple Linear Regression Model}
$$Y = \beta_0 + \beta_1 X_1 + \beta_2 X_2 + \dots + \beta_k X_k + \varepsilon$$
Formulation in \textbf{matrix form}:
$$\begin{bmatrix}
y_1 \\ y_2 \\ \vdots \\ y_m
\end{bmatrix} = \begin{bmatrix}
1 & X_11 &X_12 &\dots &X_1k \\
1 & X_21 &X_22 &\dots &X_2k \\
\vdots & \vdots &\vdots &\vdots &\vdots \\
1 & X_m1 &X_m2 &\dots &X_mk
\end{bmatrix} \begin{bmatrix}
\beta_0 \\ \beta_1 \\ \beta_2 \\ \vdots \\ \beta_k
\end{bmatrix} + \begin{bmatrix}
\varepsilon_1 \\ \varepsilon_2 \\ \vdots \\ \varepsilon_m
\end{bmatrix}$$
$$[m \times 1] = [m \times (k+1)] \cdot [(k+1) \times 1] + [m \times 1]$$

\paragraph{OLS Estimator}
\begin{itemize}
	\item Model:
	\begin{align*}
		RSS  &= e^{T} e = (y - X\hat{\beta})^T (y - X\hat{\beta}) \rightarrow \min \\
		\rightarrow  &\frac{\partial RSS}{\partial \beta} = -2X^Ty + 2X^TX\beta= 0
	\end{align*}

	\item Solution:
	\begin{align*}
		\hat{\beta} &= (X^TX)^{-1}X^Ty \\
		\hat{y} &= X(X^TX)^{-1}X^Ty
	\end{align*}

\end{itemize}
\subsubsection{Quality Metrics of Linear Regression Models}
\paragraph{Residual Sum of Squares (RSS)}
An \textbf{unbiased} estimator of RSS of the population is given by
$$RSS = \Sigma_{i=1}^{n} (y_i - \hat{y}_i)^2$$

\paragraph{Mean Squared Error (MSE)}
$$MSE = \frac{RSS}{N}$$
\paragraph{Root Mean Squared Error (RMSE)}
$$RMSE = \sqrt{MSE}$$
\paragraph{Total Sum of Squares (TSS)} sum of Explained Sum of Squares(ESS) and Residual Sum of Squares(RSS).
\begin{align*}
	\Sigma (y - \bar{y})^2 &= \Sigma (\hat{y} - \bar{y})^2 + \Sigma (y - \hat{y})^2 \\
	TSS &= ESS + RSS
\end{align*}
\paragraph{R-squared $\mathbf{R^2}$} the proportional of explained variablity from the model
$$R^2 = \frac{TSS - RSS}{TSS} = 1 - \frac{RSS}{TSS} = \frac{ESS}{TSS}$$

\subsection{Logistic Regression \& Poisson Regression}
\subsubsection{The Logistic Regression Model}
The \textbf{binary} logistic regression model described in \textbf{log odds/logit}:
$$log\_odds  =  \ln(\frac{p(X)}{1 - p(X)}), \quad  \ln(\frac{p(X)}{1 - p(X)}) = \beta_0 + \beta_1 X + \varepsilon$$

The logistic regression model described in \textbf{odds}:
$$odds = \frac{p(X)}{1 - p(X)}, \quad \frac{p(X)}{1 - p(X)} = e^{\beta_0 + \beta_1 X}$$

\subsubsection{The Logistic Function}
$$Pr[Y|X] = p(X) = \dfrac{e^{\beta_0 + \beta_1 X}}{1 + e^{\beta_0 + \beta_1 X}}$$

\subsubsection{Multiple Logistic Regression Model}
$$\ln(\frac{p(X)}{1 - p(X)}) = \beta_0 + \beta_1 X_1 + \beta_2 X_2 + \dots + \beta_k X_k$$

The logistic function can also be described as a \textbf{sigmoid function} in form $S(x) = \frac{e^x}{1 + e^x} = \sigma(x)$
$$P(X) = \sigma(\beta_0 + \beta_1 X)$$
\subsubsection{The Likelihood Function and Maximum Likelihood Estimator}
\paragraph{likelihood function for Logistic Regression Model}
\begin{align*}
	L &= \Pi_{i=1} p^{y_i}(1-p)^{1-y_i} \\
	&= \Pi_{i=1} \sigma(\beta_0 + \beta_1 X)^{y_i} \cdot (1 - \sigma(\beta_0 + \beta_1 X))^{1-y_i}
\end{align*}
\paragraph{Log of Likelihood Function}
$$LL = \ln(L) = \Sigma_{i=1} ( y_{i} \ln(p) + (1 - y_i)\ln(1-p))$$
\paragraph{Maximum Likelihood Estimator}
\begin{align*}
	\beta = \arg\max_{\beta}(LL) &= \arg\max_{\beta} [\Sigma_{i=1} ( y_{i} \ln(p) + (1 - y_i)\ln(1-p))] \\
	&= \arg\max_{\beta} [\Sigma_{i=1} ( y_{i} \ln(\sigma(\beta_0+\beta_1X)) + (1 - y_i)\ln(1-\sigma(\beta_0 + \beta_1X)))]
\end{align*}

\paragraph{Gradient(partial derivatives) of the LL-Function}: \textbf{chain rule}

with $z = \beta_0+\beta_1X$

$$\frac{\partial LL}{\beta_j} = \Sigma_{i=1} \frac{\partial LL}{\partial p} \cdot \frac{\partial p}{\partial z} \cdot \frac{\partial z}{\partial \beta_j}$$
\begin{align*}
	\frac{\partial LL}{\partial p} &= \frac{y_i}{p} - \frac{1 - y_i}{1 - p} \\
	\frac{\partial p}{\partial z} &= \sigma(z) \cdot(1 - \sigma(z)) \\
	\frac{\partial z}{\partial \beta_0} &= 1 \text{, } \frac{\partial z}{\partial \beta_j} = x_j\\
\end{align*}
\begin{align*}
	\frac{\partial LL}{\beta_0} &= \left[ \frac{y_i}{p} - \frac{1 - y_i}{1 - p}\right]  \sigma(z) \cdot(1 - \sigma(z)) = \left[ y_i - \sigma(X\beta) \right] \\
	\frac{\partial LL}{\beta_j} &= \left[ \frac{y_i}{p} - \frac{1 - y_i}{1 - p}\right]  \sigma(z) \cdot(1 - \sigma(z))\cdot x_j = \left[ y_i - \sigma(X\beta) \right] x_j
\end{align*}

\subsubsection{Poisson Regression Model}
$$ln(\mu(x)) = \beta_0 + \beta_1 X_1 + \dots \beta_j X_i$$

\paragraph{random component(dependent variable)}
$$Pr(Y|X) = p(X) = \dfrac{e^{-\mu} \mu^y}{y!} = \dfrac{e^{\beta xy} e^{-e^{\beta x}}}{y!}$$

\paragraph{likelihood function}
$$L(\beta|X,Y) = \Pi_{i=1} p = \Pi_{i=1} \dfrac{e^{\beta x_iy_i} e^{-e^{\beta x}}}{y_i!}$$

\paragraph{The Maximum Likelihood Estimator}
$$\log L(\beta | X,Y) = \Sigma_{i=1} (\beta x_iy_i - e^{\beta x_i} - \log(y_i!))$$

\subsection{Naive Bayes \& Bayesian Network}
\subsubsection{Bayes Theorem}
\begin{itemize}
	\item single evidence: $$Pr(h|e) = \frac{Pr(h \cap e)}{Pr(e)} = \dfrac{Pr(e|h) \cdot Pr(h)}{Pr(e)}$$
	\item multiple evidence:
	\begin{align*}
		Pr(h|e_1, e_2, \dots, e_k) &= \dfrac{Pr(e_1 | h) \cdot Pr(e_2 | h) \dots Pr(e_k | h) \cdot Pr(h)}{Pr(e_1, e_2, \dots, e_k)} \\
		&= \frac{\Pi_{i=1}^k Pr(e_i|h) \cdot Pr(h)}{Pr(e_1,e_2, \dots e_k)}
	\end{align*}
\end{itemize}

\subsubsection{Pr(e)}
\begin{itemize}
	\item If the prior probability $Pr(e_i)$ is \textbf{known}:
	$$Pr(e_1, e_2, \dots, e_k) = Pr(e_1)\cdot Pr(e_2)\dots Pr(e_k)$$
	\item If the prior probability $Pr(e_i)$ is \textbf{unknown}: law of total probability
	$$Pr(e_1, e_2, \dots, e_k) = Pr(e_1, e_2, \dots, e_k|h)\cdot Pr(h) + Pr(e_1, e_2, \dots, e_k | \neg h) \cdot Pr(\neg h)$$
\end{itemize}

\subsubsection{Chain Rule}
According to the \textbf{directed acyclic graph}, derive the \textbf{joint probability distribution}

$$Pr(e_1, e_2, \dots, e_k) = \Pi_{i=1} Pr(e_i| e_{i-1}, \dots, e_1) = \Pi_{i=1} Pr(e_i| \text{Parents}(e_i))$$

\subsubsection{Conditional Independence}
\paragraph{conditional independence between hypothesis and evidence} the hypothesis $h$ is only dependent on $e_1, e_2, e_3$, not on $e_4$ (redundant), then
$$Pr(h | e_1, e_2, e_3, e_4) = Pr(h | e_1, e_2, e_3) $$

\paragraph{conditional independence between hypotheses}

if two hypotheses are \textbf{independent} from each other, then
$$Pr(h_1, h_2 | e_1, e_2) = Pr(h_1|e_1,e_2) \cdot Pr(h_2|e_1,e_2)$$

\subsection{Decision Tree}
\paragraph{Entropy} $\in [0,1]$, measures how much \textbf{additional information required} in \textbf{bits}
	$$\text{entropy}(p_1, \dots, p_n) = - \Sigma_{i=1} ^n p_i \cdot \log_{2} p_i$$

\paragraph{Information of Each Attribute Value}

	$$\text{info}([c_1, \dots, c_n]) = \text{entropy}(\frac{c_1}{C}, \dots, \frac{c_n}{C})$$

\paragraph{Information of the Attribute} the \textbf{weighted average} of the \textbf{information needed} from each attribute value.

Say an attribute has $m$ attribute values/branches,

$$\text{info}([c_1, \dots, c_n]_1, \dots, [c_1, \dots, c_n]_m) = \Sigma_{i=1}^m  \frac{C_m}{N} \cdot \text{info}([c_1, \dots, c_n])_m$$

\paragraph{Information Gain of the Attribute}

$$\text{Information\_Gain(attribute)} = \text{info(before split by attribute)} - \text{info(after split by attribute)}$$

\paragraph{Intrinsic Information of a Attribute} s: size of a leaf from each branch
$$\text{intrinsic\_info}([s_1, \dots, s_n]) = \text{info}([s_1, \dots, s_n])$$

\paragraph{Gain Ratio of a Attribute}

$$\text{Gain\_Ratio(attribute)} = \dfrac{\text{Gain(attribute)}}{\text{Intrinsic\_Info(attribute)}}$$

\subsection{Evaluation}

\paragraph{Accuracy} $$\text{Accuracy} = \frac{TP + TN}{N}$$
\paragraph{Error Rate} $$\text{Error Rate} = 1 - \text{Accuracy} = \frac{FP + FN}{N}$$

\paragraph{True Positive Rate / Recall / Hit Rate} $$\text{True Positive Rate/Recall} = \frac{TP}{TP + FN}$$
\paragraph{True Negative Rate / Specificity} $$\text{True Negative Rate/Specificity} = \frac{TN}{TN + FP}$$
\paragraph{False Positive Rate / False Alarm Rate} $$\text{False Positive Rate/False Alarm Rate} = 1- \text{Specificity} = \frac{FP}{TN + FP}$$

\paragraph{Precision} $$\text{Precision} = \frac{TP}{TP + FP}$$

\subsection{Clustering: Expectation Maximization}
\paragraph{Expectation Step} calculate the probability for \textbf{all instances} in \textbf{each cluster}.
$$Pr(A|x) = \frac{Pr(x|A) \cdot Pr(A)}{Pr(x)} ,\quad Pr(x|A) = \frac{1}{\sqrt{2\pi} \cdot \sigma_A} e^{-\frac{(x - \mu_A)^2}{2\sigma_A^2}}$$
$$Pr(B|x) = \frac{Pr(x|B) \cdot Pr(B)}{Pr(x)} ,\quad Pr(x|B) = \frac{1}{\sqrt{2\pi} \cdot \sigma_B} e^{-\frac{(x - \mu_B)^2}{2\sigma_B^2}}$$

\paragraph{Maximization Step} calculate the weighted mean and weighted variance using \textbf{all instances}.
$$w_{iA} = Pr(A|x), \quad  w_{iB} = Pr(B|x)$$
$$\mu_A = \frac{w_{1A}x_1 + w_{2A}x_2 + \dots + w_{nA}x_n}{w_{1A} + w_{2A} + \dots + w_{nA}}$$
$$\sigma_A = \sqrt{\frac{w_{1A} (x_1 - \mu_A)^2 + \dots + w_{nA} (x_n - \mu_A)^2}{w_{1A} + w_{2A} + \dots + w_{nA}}}$$
analog to $\mu_B$ and $\sigma_B$

$$Pr(A) = \frac{\Sigma w_A}{\Sigma w_A + \Sigma w_B}, \quad Pr(B) = 1 - Pr(A)$$

\subsection{Principal Component Analysis \& Restoring Original Data}
\begin{figure}[H]
	\centering
	\includegraphics[width=\textwidth]{dimreduce.png}
\end{figure}
\newpage
\subsection{Recommendation Systems: Association Rules}
\paragraph{Support of a Rule}
\item support of \textbf{a rule}: the support of all item sets it contains.
$$supp(A,B \Rightarrow C,D) = supp(\{A,B,C,D\})$$

\paragraph{Confidence of a Rule}  the probability that X and Y coexist given that X exists.
$$conf(R: X \Rightarrow Y) = \frac{supp(X \cup Y)}{supp(X)}$$

\paragraph{Lift of a Rule} indicates \textbf{by how much (ratio)} the \textbf{confidence of a rule} surpasses the \textbf{expected value}.
$$Lift(R: X \Rightarrow Y) = \frac{conf(R)}{expConf(R)} = \dfrac{\frac{supp(X \cup Y)}{supp(X)}}{supp(Y)} = \frac{supp(X \cup Y)}{supp(X)\cdot supp(Y)}$$
\subsection{Recommendation Systems: Collaborative Filtering}
\paragraph{Weighted Correlation}
$$w_{a,u} = s_{a,u} \cdot c_{a,u}$$
$$c_{a,u} = \frac{Cov(r_{a}, r_{u})}{\sigma_{r_{a}} \cdot \sigma_{r_{u}}}$$
$$Cov(r_{a}, r_{u}) = \frac{1}{m-1}\cdot \Sigma (r_a - \bar{r}_a) (r_u - \bar{r}_u)$$

\paragraph{Rating Prediction}
$$p_{a,i} = \bar{r}_a + \Sigma_{u = 1}^k \dfrac{w_{a,u} \cdot (r_{u,i} - \bar{r}_u)}{\Sigma_{u=1}^k |w_{a,u}|}$$

\subsection{Recommendation Systems: SVD}
\paragraph{Rating Prediction}
$$r_{u,i} = \bar{r}_u + U(user) \cdot S \cdot V^T(item)$$

\subsection{Neural Network}
\paragraph{Forward Pass}
\begin{align*}
	z^{[1]} &= W^{[1]}\cdot a^{[0]} + b^{[1]} =  W^{[1]}\cdot x + b^{[1]}\\
	a^{[1]} &= g^{[1]}(z^{[1]}) = \sigma(z^{[1]}) \\
	z^{[2]} &= W^{[2]}\cdot a^{[1]} + b^{[2]} \\
	a^{[2]} &= g^{[2]}(z^{[2]}) = \sigma(z^{[2]})
\end{align*}

\paragraph{Loss Function} If we evaluate the model using \textbf{cross-entropy loss}: the calculation for $y \ln\hat{y}$ is a dot product(element-wise multiplication).
$$l(y,\hat{y}) = - [y \ln \hat{y} + (1-y)\ln(1 - \hat{y})]$$
\paragraph{Empirical Risk}  \textbf{average} the loss.
$$\mathcal{L}(y,\hat{y}) = \frac{1}{n} \cdot \Sigma l(y,\hat{y})$$

\paragraph{Backpropagation}

example in updating layer 2:
\begin{align*}
	W^{[2]}_{t+1} &= W^{[2]}_t - \alpha \cdot dW =W^{[2]}_t - \alpha \cdot \frac{\partial L}{\partial W^{[2]}} \\
	b^{[2]}_{t+1} &= b^{[2]}_t - \alpha \cdot db =b^{[2]}_t - \alpha \cdot \frac{\partial L}{\partial b^{[2]}}
\end{align*}

$$\frac{\partial L_n}{\partial W} = \frac{\partial L_n}{\partial a_{n}} \cdot \frac{\partial a_{n}}{\partial z_{n}} \cdot \frac{\partial z_n}{\partial W}$$

\begin{align*}
	L_n &= \frac{1}{2} (y_{n} - g(w_{kl}a_{kn} + b_l))^2 = \frac{1}{2} (y_n - a_{ln})^2, \quad &\frac{\partial L_n}{\partial a_{ln}} &= -(y_n - a_{ln})\\
	a_{ln} &= g(z_{ln}) , \quad &\frac{\partial L_n}{\partial z_{ln}} &= g'(z_{ln}) \\
	z_{ln} &= w_{kl}a_{kn} + b_l, \quad &\frac{\partial L_n}{\partial w_{ln}} &= a_{kn}
\end{align*}

If the activation is a sigmoid activation: $\sigma(x) = \frac{e^x}{1 + e^x}$, $\sigma'(x) = \sigma(x)(1 - \sigma(x))$.
$$dW^{[2]} = -(y- a^{[2]})\cdot a^{[1]^{T}} = (a^{[2]} - y) \cdot a^{[1]^{T}}$$

\paragraph{Gradient Descent}

	\subparagraph{fixed step size}
	$$\begin{bmatrix}
	x_n \\y_n
	\end{bmatrix} = \begin{bmatrix}
	x_{n-1} \\ y_{n-1}
	\end{bmatrix} - \alpha \cdot \nabla f(x_{n-1}, y_{n-1})$$
	\subparagraph{dynamic step size}
	$$\begin{bmatrix}
	x_n \\y_n
	\end{bmatrix} = \begin{bmatrix}
	x_{n-1} \\ y_{n-1}
	\end{bmatrix} - \alpha_{n} \cdot \nabla f(x_{n-1}, y_{n-1})$$

	\subparagraph{Momentum}
	\begin{align*}
		d_n &= \beta \cdot d_{n-1} + \alpha \cdot\nabla f(x_{n-1})\\
		x_n &= x_{n-1} - d_n
	\end{align*}


\subsection{Spearman rank correlation}

For each of \(2\) variables we create a ranking of them.

From \(X\) and \(Y\) we then have \(R_X\) and \(R_Y\).

We then calculate the Pearson correlation coefficient between the rankings.

\(r_S=\dfrac{cov(R_X, R_Y)}{\sigma_{R_X}\sigma_{R_Y}}\)


\chapter{Estimation}

Here a linear discrete-time model is assumed with scalar measurements, to avoid the tedious matrix calculations that are ultimately a distraction to the derivation.

\begin{equation}
x_{k+1} = a x_k + w_k
\end{equation}
\begin{equation}
y_k = x_k + v_k
\end{equation}
\begin{equation}
w_k \sim \gauss{0}{q},
v_k \sim \gauss{0}{r}
\end{equation}

\section{Equivalence of the batch and recursive forms of the Kalman filter}

From Ref. \cite{sorenson1970}, the batch linear estimate for $x_k$ is optimal with respect to the following cost function:

\begin{equation}
J = \frac{1}{2} \quadr{(\vest{x}_0-\vec{\mu})}{P_0^{-1}} +
\frac{1}{2} \sum_{k=1}^N \quadr{(\veas{y}_k - H_k \vest{x}_k)}{R^{-1}} +
\frac{1}{2} \sum_{k=1}^N \quadr{(\vest{x}_k - \Phi_{k-1} \vest{x}_{k-1})}{Q^{-1}}
\end{equation}

For the scalar problem with $N = 2$, the first-order necessary conditions for optimality with respect to $\vest{x}_0,\vest{x}_1,\vest{x}_2$ yield the following linear expression:

\begin{equation}
\begin{bmatrix}
\inv{p_0}+\frac{a^2}{q} & -\frac{a}{q} & 0\\
-\frac{a}{q} & \inv{r} + \inv{q} + \frac{a^2}{q} & -\frac{a}{q} \\
0 & -\frac{a}{q} & \inv{q}+\inv{r}
\end{bmatrix}
\begin{bmatrix}
\est{x}_0 \\ \est{x}_1 \\ \est{x}_2
\end{bmatrix} = \begin{bmatrix}
\frac{\mu}{p_0} \\ \frac{\meas{y}_1}{r} \\ \frac{\meas{y}_2}{r}
\end{bmatrix}
\end{equation}

Solving for $\est{x}_2$ yields the following expression:

\begin{equation}
\est{x}_2 = \inv{p_0a^4r + a^2qr + p_0a^2q + p_0a^2r + q^2 + 2qr + r^2} \begin{bmatrix}
a^2q^2 &
p_0ra^3 + qra &
qr + q^2 + a^2p_0q + a^4p_0r + a^2qr
\end{bmatrix}\begin{bmatrix}
\mu \\ \meas{y}_1 \\ \meas{y}_2
\end{bmatrix}
\label{eq:batchkf}
\end{equation}

But this is exactly the expression obtained from recursion of the standard Kalman filter:

\begin{equation}
\est{x}_0 = \mu
\end{equation}
\begin{equation}
p_1^{-} = a^2 p_0 + q
\end{equation}
\begin{equation}
k_1 = \frac{p_1^{-}}{r+p_1^{-}}
\end{equation}
\begin{equation}
\est{x}_1 = a\est{x}_0 + k_1(\meas{y}_1-a\est{x}_0)
\end{equation}
\begin{equation}
p_1^{+} = (1-k_1)p_1^{-}
\end{equation}
\begin{equation}
p_2^{-} = a^2 p_1^{+} + q
\end{equation}
\begin{equation}
k_2 = \frac{p_2^{-}}{r+p_2^{-}}
\end{equation}
\begin{equation}
\est{x}_2 = a\est{x}_1 + k_2(\meas{y}_2 - a\est{x}_1)
\label{eq:x2_lkf}
\end{equation}

The algebra required to prove the equivalence of Eqs. \ref{eq:batchkf} and \ref{eq:x2_lkf} is tedious even for the scalar case.

\section{MATLAB code for showing the equivalence of the first two recursions}

The following code will compute the first two recursions of the sequential filter and compute the symbolic difference between the sequential value and the batch value for $N = 1$ and $N = 2$.
The code can readily be extended for higher recursions if readers think that is necessary.

\begin{verbatim}
clear variables;
close all;

syms mu p0 q r a y1 y2 real;

%% solution for the recursive KF
x0s = mu;
% propagated covariance at t1-
p1s = a^2*p0 + q;
k1 = p1s/(r+p1s);
x1s = a*x0s + k1*(y1-a*x0s);
% propagated covariance at t1+
p1sp = (1-k1)*p1s;
% propagated covariance at t2-
p2s = a^2*p1sp + q;
k2 = p2s/(r+p2s);
x2s = a*x1s + k2*(y2-a*x1s);

%% solution for the batch KF with 1 measurement
M = [1/p0+a^2/q -a/q;
-a/q 1/r+1/q];
b = [mu/p0;
y1/r];
xsol1 = M\b;

% print the error between the recursive and batch estimate
simplify(xsol1(2) - x1s)

%% solution for the batch KF with 2 measurements
M = [1/p0 + a^2/q -a/q 0;
-a/q 1/r+1/q+a^2/q -a/q;
0 -a/q 1/r+1/q];
b = [mu/p0;y1/r;y2/r];
xsol2 = M\b;

% print the error between the recursive and batch estimate
simplify(xsol2(3) - x2s)
\end{verbatim}


\chapter{Resultant and Discriminant of a Polynomial}
\label{chap:appB}
We begin with some useful definitions regarding polynomials. The reason why we refer resultant is that we can calculate the discriminant of the polynomial through this tool, especially when the degree of polynomial is large. First of all, we give a definition of resultant\citep{healy2002resultants}:
\begin{definition}
Let $R$ be an integral domain, given two polynomials $f(x),g(x)\in R[x]$ with roots $\alpha_1,\dots,\alpha_m$ and $\beta_1,\dots,\beta_n$ respectively, then the resultant $\operatorname{Res}(f,g)$ of $f,g$ is defined to be $$\operatorname{Res}(f,g)=l(f)^nl(g)^m\prod_{i,j}(\alpha_i-\beta_j)$$ which is equivalent to both $$\operatorname{Res}(f,g)=l(f)^n\prod_{i}^m g(\alpha_i)$$ and $$\operatorname{Res}(f,g)=(-1)^{n m}l(g)^m\prod_{i}^n f(\beta_i)$$
\end{definition}
From this definition, on can easily see that $f,g$ have a common root in some if and only if $\operatorname{Res}(f,g)=0$.
An important proposition shows the relationship of this definition to the \textbf{Sylvester's matrix} (Some books take that as definition): Let $S$ be the Sylvester's matrix of polynomials $f(x)$ and $g(x)$, then $\operatorname{Res}(f,g)=\det(S)$.

Also for convenience, I choose a definition of normalized discriminant \citep{janson2007resultant} of polynomial as follows in this paper.

\begin{definition}
Let $f$ be a polynomial of degree $n\geq1$ with coefficients in a field $F$. Let $F_1$ be an extension of $F$ where $f$ splits, and let $r_1,\dots,r_n$ be the roots of $f$ in $F_1$. Then the discriminant of $f$ is $$\operatorname{Disc}(f):=\prod_{1\leq i<j\leq n}(r_i-r_j)^2$$
\end{definition}

Note further that $\operatorname{Disc}(cf)=\operatorname{Disc}(f)$ for any nonzero constant. The following theorem give the relation between the discriminant and resultant.

\begin{proposition}\label{prop:disctri}
Let $f=a_nx^n+\cdots+a_0$ be polynomial of degree $n\geq1$ with coefficients in a field $F$. The the discriminant of $f$ is given by
$$\operatorname{Disc}(f)=(-1)^{n(n-1)/2}a^{-(2n-1)}_n \operatorname{Res}(f,f')$$
\end{proposition}

\begin{example}\label{for:nxpxq}
Let $f(x)=x^n+px+q$ for $n\geq2$, then $f'(x)=nx^{n-1}+p$, then from the Proposition \ref{prop:disctri}, we have $$\operatorname{Disc}(f)=(-1)^{(n-1)(n-2)/2}(n-1)^{n-1p^n}+(-1)^{n(n-1/2)}n^nq^{n-1}$$
\end{example}


\subsection{Sample mean}

We previously defined the population mean is defined as \(\mu=E[X]\).

The sample mean is defined as \(\bar x = \dfrac{1}{n}\sum_i x_i\).

\subsubsection{Centred mean}

We can subtract the mean from each entry in the sample. This will leave a new mean of \(0\). This is convenient for many calculations.


\section{Built-in functions}

\section*{abs}
abs($x$) returns the absolute value or vector length of $x$.
The mag function should be used for complex $x$.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
P = (x,y)
abs(P)
\end{Verbatim}

$\displaystyle (x^2+y^2)^{1/2}$

\section*{adj}
adj($m$) returns the adjunct of matrix $m$.

\section*{and}
and($a,b,\ldots$) returns the logical ``and'' of predicate expressions.

\section*{arccos}
arccos($x$) returns the inverse cosine of $x$.

\section*{arccosh}
arccosh($x$) returns the inverse hyperbolic cosine of $x$.

\section*{arcsin}
arcsin($x$) returns the inverse sine of $x$.

\section*{arcsinh}
arcsinh($x$) returns the inverse hyperbolic sine of $x$.

\section*{arctan}
arcttan($x$) returns the inverse tangent of $x$.

\section*{arctanh}
arctanh($x$) returns the inverse hyperbolic tangent of $x$.

\section*{arg}
arg($z$) returns the angle of complex $z$.

\section*{ceiling}
ceiling($x$) returns the smallest integer not less than $x$.

\section*{check}
check($x$) In a script, if the predicate $x$ is true then continue, else stop.

\section*{choose}
choose($n,k$) returns $\displaystyle\binom{n}{k}$

\section*{circexp}
circexp($x$) returns expression $x$ with circular functions converted
to exponential forms.
Sometimes this will simplify an expression.

\section*{coeff}
coeff($p,x,n$) returns the coefficient of $x^n$ in polynomial $p$.

\section*{cofactor}
cofactor($m,i,j$) returns of the cofactor of matrix $m$ with respect to row $i$ and column $j$.

\section*{conj}
conj($z$) returns the complex conjugate of $z$.

\section*{contract}
\index{trace}
contract($a,i,j$) returns tensor $a$ summed over indices $i$ and $j$.
If $i$ and $j$ are omitted then indices 1 and 2 are used.
contract($m$) is equivalent to the trace of matrix $m$.

\section*{cos}
cos($x$) returns the cosine of $x$.
%If $x$ is a floating point number then $\cos(x)$ is evaluated numerically.

\section*{cosh}
cosh($x$) returns the hyperbolic cosine of $x$.

\section*{cross}
cross($u,v$) returns the cross product of vectors $u$ and $v$.

\section*{curl}
curl($u$) returns the curl of vector $u$.

\section*{d}
d($f,x$) returns the derivative of $f$ with respect to $x$.

\section*{defint}
defint($f,x,a,b,\ldots$)
returns the definite integral of $f$ with respect to $x$ evaluated from $a$ to $b$.
The argument list can be extended for multiple integrals.
For example, $d(f,x,a,b,y,c,d)$.

\section*{deg}
deg($p,x$) returns the degree of polynomial $p$ in $x$.

\section*{denominator}
denominator($x$) returns the denominator of expression $x$.

\section*{det}
det($m$) returns the determinant of matrix $m$.

\section*{do}
do($a,b,\ldots$) evaluates the argument list from left to right.
Returns the result of the last argument.

\section*{dot}
dot($a,b,\ldots$) returns the dot product of tensors.

\section*{draw}
draw($f,x$) draws the function $f$ with respect to $x$.

\section*{erf}
erf($x$) returns the error function of $x$.

\section*{erfc}
erf($x$) returns the complementary error function of $x$.

\section*{eval}
eval($f,x,n$) returns $f$ evaluated at $x=n$.

\section*{exp}
exp($x$) returns $e^x$.

\section*{expand}
expand($r,x$) returns the partial fraction expansion of the ratio of
polynomials $r$ in $x$.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
expand(1/(x^3+x^2),x)
\end{Verbatim}

$\displaystyle \frac{1}{x^2}-\frac{1}{x}+\frac{1}{x+1}$

\section*{expcos}
expcos($x$) returns the cosine of $x$ in exponential form.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
expcos(x)
\end{Verbatim}

$\displaystyle \frac{1}{2}\exp(-ix)+\frac{1}{2}\exp(ix)$

\section*{expsin}
expsin($x$) returns the sine of $x$ in exponential form.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
expsin(x)
\end{Verbatim}

$\displaystyle \frac{1}{2}i\exp(-ix)-\frac{1}{2}i\exp(ix)$

\section*{factor}
factor($n$) factors the integer $n$.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
factor(12345)
\end{Verbatim}

$\displaystyle 3\times 5\times 823$

$factor(p,x)$ factors polynomial $p$ in $x$.
The last argument can be omitted for polynomials in $x$.
The argument list can be extended for multivariate polynomials.
For example, factor($p,x,y$) factors $p$ over $x$ and then over $y$.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
factor(125*x^3-1)
\end{Verbatim}

$\displaystyle (5x-1)(25x^2+5x+1)$

\section*{factorial}
Example:

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
10!
\end{Verbatim}

$\displaystyle 3628800$

\section*{filter}
filter($f,a,b,\ldots$) returns $f$ with terms involving $a$, $b$, etc. removed.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
1/a+1/b+1/c
\end{Verbatim}

$\displaystyle \frac{1}{a}+\frac{1}{b}+\frac{1}{c}$

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
filter(last,a)
\end{Verbatim}

$\displaystyle \frac{1}{b}+\frac{1}{c}$

\section*{float}
float($x$) converts $x$ to a floating point value.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
sum(n,0,20,(-1/2)^n)
\end{Verbatim}

$\displaystyle \frac{699051}{1048576}$

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
float(last)
\end{Verbatim}

$\displaystyle 0.666667$

\section*{floor}
floor($x$) returns the largest integer not greater than $x$.

\section*{for}
for($i,j,k,a,b,\ldots$) For $i$ equals $j$ through $k$ evaluate $a$, $b$, etc.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
x = 0
y = 2
for(k,1,9,x=sqrt(2+x),y=2*y/x)
float(y)
\end{Verbatim}

$\displaystyle 3.14159$

\section*{gcd}
gcd($a,b,\ldots$) returns the greatest common divisor.

\section*{hermite}
hermite($x,n$) returns the $n$th Hermite polynomial in $x$.

\section*{hilbert}
hilbert($n$) returns a Hilbert matrix of order $n$.

\section*{imag}
imag($z$) returns the imaginary part of complex $z$.

\section*{inner}
inner($a,b,\ldots$) returns the inner product of tensors.
Same as the dot product.

\section*{integral}
integral($f,x$) returns the integral of $f$ with respect to $x$.

\section*{inv}
inv($m$) returns the inverse of matrix $m$.

\section*{isprime}
isprime($n$) returns 1 if $n$ is prime, zero otherwise.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
isprime(2^53-111)
\end{Verbatim}

$\displaystyle 1$

\section*{laguerre}
laguerre($x,n,a$) returns the $n$th Laguerre polynomial in $x$.
If $a$ is omitted then $a=0$ is used.

\section*{lcm}
lcm($a,b,\ldots$) returns the least common multiple.

\section*{leading}
leading($p,x$) returns the leading coefficient of polynomial $p$ in $x$.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
leading(5x^2+x+1,x)
\end{Verbatim}

$\displaystyle 5$

\section*{legendre}
legendre($x,n,m$) returns the $n$th Legendre polynomial in $x$.
If $m$ is omitted then $m=0$ is used.

\section*{log}
log($x$) returns the natural logarithm of $x$.

\section*{mag}
mag($z$) returns the magnitude of complex $z$.

\section*{mod}
mod($a,b$) returns the remainder of $a$ divided by $b$.

\section*{not}
not($x$) negates the result of predicate expression $x$.

\section*{nroots}
nroots($p,x$) returns all of the roots, both real and complex, of
polynomial $p$ in $x$.
The roots are computed numerically.
The coefficients of $p$ can be real or complex.

\section*{numerator}
numerator($x$) returns the numerator of expression $x$.
%\begin{itemize}
%\item[$\scriptstyle1$]{\tt numerator(a/b+b/a)}
%\item[$\scriptstyle2$]\hspace{50pt} $a^2+b^2$
%\end{itemize}

\section*{or}
or($a,b,\ldots$) returns the logical ``or'' of predicate expressions.

\section*{outer}
outer($a,b,\ldots$) returns the outer product of tensors.

Example 1.
\[
\begin{pmatrix}
1\\
0
\end{pmatrix}
\otimes
\begin{pmatrix}
1\\
0
\end{pmatrix}
=\begin{pmatrix}
1 & 0\\
0 & 0
\end{pmatrix}
\]

\begin{Verbatim}[formatcom=\color{blue}]
outer((1,0),(1,0))
\end{Verbatim}

$\displaystyle
\begin{pmatrix}
1 & 0\\
0 & 0
\end{pmatrix}
$

\bigskip
Example 2. From the identity
\[
|A\rangle=\sum_i|i\rangle\langle i|A\rangle
\]
it follows that
\[
\sum_i|i\rangle\langle i|=\bf I
\]
For a two-state system with basis vectors
\[
|1\rangle=\begin{pmatrix}1/\sqrt2\\i/\sqrt2\end{pmatrix},
\hbox{\quad and\quad}
|2\rangle=\begin{pmatrix}1/\sqrt2\\-i/\sqrt2\end{pmatrix}
\]
the following code computes $\sum|i\rangle\langle i|$.
\begin{Verbatim}[formatcom=\color{blue}]
X1 = (1/sqrt(2),i/sqrt(2))
X2 = (1/sqrt(2),-i/sqrt(2))
outer(conj(X1),X1) + outer(conj(X2),X2)
\end{Verbatim}
$\displaystyle
\begin{pmatrix}
1 & 0\\
0 & 1
\end{pmatrix}
$

\bigskip
The following code uses a different approach.
First, tensor $T$ is computed such that
\[
T=\begin{pmatrix}
|1\rangle\langle1| & |1\rangle\langle2| \\
|2\rangle\langle1| & |2\rangle\langle2|
\end{pmatrix}
\]
Then contraction is used to sum the diagonal elements.
\begin{Verbatim}[formatcom=\color{blue}]
X1 = (1/sqrt(2),i/sqrt(2))
X2 = (1/sqrt(2),-i/sqrt(2))
X = (X1,X2)
T = outer(conj(X),X)
T = transpose(T,2,3)
contract(T)
\end{Verbatim}
$\displaystyle
\begin{pmatrix}
1 & 0\\
0 & 1
\end{pmatrix}
$
\begin{Verbatim}[formatcom=\color{blue}]
-- verify components of T
T[1,1] == outer(conj(X1),X1)
T[1,2] == outer(conj(X1),X2)
T[2,1] == outer(conj(X2),X1)
T[2,2] == outer(conj(X2),X2)
\end{Verbatim}
$1$\\
$1$\\
$1$\\
$1$

\section*{polar}
polar($z$) converts complex $z$ to polar form.

\section*{prime}
prime($n$) returns the $n$th prime number, $1\le n\le10{,}000$.

\section*{print}
print($a,b,\ldots$) evaluates expressions and prints the results..
Useful for printing from inside a ``for'' loop.

\section*{product}
product($i,j,k,f$) returns $\displaystyle\prod_{i=j}^k f$

\section*{quote}
quote($x$) returns expression $x$ unevaluated.

\section*{quotient}
quotient($p,q,x$) returns the quotient of polynomials in $x$.

\section*{rank}
rank($a$) returns the number of indices that tensor $a$ has.
A scalar has no indices so its rank is zero.

\section*{rationalize}
rationalize($x$) puts everything over a common denominator.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
rationalize(a/b+b/a)
\end{Verbatim}

$\displaystyle \frac{a^2+b^2}{ab}$

\section*{real}
real($z$) returns the real part of complex $z$.

\section*{rect}
rect($z$) returns complex $z$ in rectangular form.

\section*{roots}
roots($p,x$) returns the values of $x$ such that the polynomial $p(x)=0$.
The polynomial should be factorable over integers.

\section*{simplify}
simplify($x$) returns $x$ in a simpler form.

\section*{sin}
sin($x$) returns the sine of $x$.

\section*{sinh}
sinh($x$) returns the hyperbolic sine of $x$.

\section*{sqrt}
sqrt($x$) returns the square root of $x$.

\section*{stop}
In a script, it does what it says.

\section*{subst}
subst($a,b,c$) substitutes $a$ for $b$ in $c$ and returns the result.

\section*{sum}
sum($i,j,k,f$) returns $\displaystyle\sum_{i=j}^k f$

\section*{tan}
tan($x$) returns the tangent of $x$.

\section*{tanh}
tanh($x$) returns the hyperbolic tangent of $x$.

\section*{taylor}
taylor($f,x,n,a$) returns the Taylor expansion of $f$ of $x$ at $a$.
The argument $n$ is the degree of the expansion.
If $a$ is omitted then $a=0$ is used.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
taylor(1/cos(x),x,4)
\end{Verbatim}

$\displaystyle \frac{5}{24}x^4+\frac{1}{2}x^2+1$

\section*{test}
test($a,b,c,d,\ldots$)
If $a$ is true then $b$ is returned else if $c$ is true then $d$ is returned, etc.
If the number of arguments is odd then the last argument is returned when all else fails.

\section*{transpose}
transpose($a,i,j$) returns the transpose of tensor $a$ with respect to indices $i$ and $j$.
If $i$ and $j$ are omitted then 1 and 2 are used.
Hence a matrix can be transposed with a single argument.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
A = ((a,b),(c,d))
transpose(A)
\end{Verbatim}

$\displaystyle \begin{bmatrix}a & c\\ b & d\end{bmatrix}$

\section*{unit}
unit($n$) returns an $n\times n$ identity matrix.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
unit(2)
\end{Verbatim}

$\displaystyle \begin{bmatrix}1&0\\0&1\end{bmatrix}$

\section*{zero}
zero($i,j,\ldots$) returns a null tensor with dimensions $i$, $j$, etc.
Useful for creating a tensor and then setting the component values.


\subsection{Hermitian (self-adjoint) matrices}

A matrix where \(M=M^*\)

For matrices over the real numbers, these are the same as symmetric matrices.

\subsubsection{Sesqulinear forms on Hermitian matrices}

\(\phi (u,v)=u^*Mv\)

\((u^*Mv)^*=v^*M^*u=v^*Mu\)

\(\phi (u,v)=\overline {\phi (v,u)}\)

\subsubsection{The forms on the same vector are always real}

\((v^*Mv)^*=v^*M^*v=v^*Mv\)

So we have:

\((v^*Mv)^*=v^*Mv\)

Which is only satisfied for reals.

\subsubsection{If \(A\) and \(B\) are Hermitian}

If \(A\) and \(B\) are Hermitian, \(AB\) is Hermitian if and only if \(AB\) commutes.

\((AB)^*=B^*A^*=BA\)

If it commutes then

\((AB)^*=AB\)

\subsubsection{Real eigenvalues}

Hermitian matrices have real eigenvalues.

\(Hv=\lambda v\)

\(v^*Hv=\lambda v^*v\)

\(v^*Hv=\lambda \)

\subsubsection{Skew-Hermitian matrices}

These are also known as anti-Hermitian matrices.

\(M^*=-M\)

\subsubsection{If eigenvalues are different, eigenvectors are orthogonal}


\section{Linear Algebra}

\subsection{Plane equation}
Assume that we know point, which belongs to plane $P_0=(x_0,y_0,z_0)$ and
we have a normal vector to the plane $\vec{n}=(a,b,c)$. Now assume that $P=(x,y,z)$
is any point in the plane. Define position vectors $\vec{r_0}$ and $\vec{r}$
from our coord. system to points $P_0$ and $P$ on the plane respectively. Vector $\vec{r_0} - \vec{r}$
lies in our plane. Then $\vec{n} \cdot (\vec{r_0} - \vec{r}) = 0$ is our plane equation.

\subsection{Distance}

Consider a line  $L$  in  $\mathbb{R}^2$  given by the equation $L:\theta \cdot x + \theta_0=$ where $\theta$ is a vector normal to the line $L$. Let the point $P$ be the endpoint of a vector $x_0$(so the coordinates of $P$  equal the components of $x_0$).\\

The shortest distance $d$ between the line $L$ and the point $P$ is:
\begin{align*}
d= \displaystyle \frac{\mid \theta \cdot x_0+ \theta _0\mid }{\mid \mid \theta \mid \mid }
\end{align*}


%!TEX root = TTK4150-Summary.tex
\section{Math}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Metric norms}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{General $p$-norm}
\begin{equation}
	\norm{x}_p
	=
	\left(
		\sum_{i=1}^{n} \abs{x_i}^p
	\right)
	^{1/p}
\end{equation}

\paragraph{Taxicab norm ($p=1$)}
\begin{equation}
	\norm{x}_1
	=
	\sum_{i=1}^{n} \abs{x_i}
\end{equation}

\paragraph{Euclidean norm ($p=2$)}
\begin{equation}
	\norm{x}_2
	=
	\sqrt{x_1^2 + \cdots + x_n^2}
\end{equation}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection[\texorpdfstring{Boundedness and $\mathcal{L}_p$-norms}
	{Boundedness and Lp-norms}]
	{Boundedness and $\mathcal{L}_p$-norms}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{$\mathcal{L}_p$-norm}
\begin{equation}
	\norm{f}_p
	=
	\left(
		\int_a^b \abs{f(\tau)}^p \dif \tau
	\right)^{1/p}
\end{equation}

\paragraph{$\mathcal{L}_\infty$-norm}
\begin{equation}
	\norm{f}_\infty
	=
	\sup_{a \leq t \leq b} \abs{f(t)}
\end{equation}

\paragraph{Boundedness}
\begin{equation}
	f \in \mathcal{L}_p \Leftrightarrow \norm{f}_p < \infty
\end{equation}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Properties of norms}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Hölder's inequality}
\begin{equation}
	\norm{fg}_1 \leq \norm{f}_p \norm{g}_q
\end{equation}
with $\frac{1}{p} + \frac{1}{q} = 1$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Matrix properties}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\paragraph{Singular} A matrix is \emph{singular} iff its determinant is zero.

\paragraph{Skew-symmetry} A matrix $A$ is \emph{skew-symmetric} iff
\begin{equation}
	-A = A\T
	.
\end{equation}

\paragraph{Jacobian} The \emph{Jacobian} matrix is defined by
\begin{equation}
	J
	=
	\od{f}{x}
	=
	\begin{bmatrix}
		\pd{f_1}{x_1} & \cdots & \pd{f_1}{x_n} \\
		\vdots        & \ddots & \vdots        \\
		\pd{f_m}{x_1} & \cdots & \pd{f_m}{x_n}
	\end{bmatrix}
	.
\end{equation}

\paragraph{Hurwitz}
A matrix $A$ is \emph{Hurwitz} if all eigenvalues of $A$ satisfy $\Re \lambda_i < 0$.

\paragraph{Positive definite} A matrix being \emph{positive definite} is equivalent to
\begin{itemize}
	\item all its eigenvalues being positive,
	\item all its leading principal minors being positive.
\end{itemize}
In addition, we have
\begin{equation}
	\lambda\sub{min}(H) x\T x \leq x\T H x \leq \lambda\sub{max}(H) x\T x
\end{equation}
for a positive definite $x\T H x$.

TTK4150 Nonlinear control systems/sec-math.tex


\chapter{The Metric Tensor}
We've created this idea of the inner product $(\cdot,\cdot)$, which maps from $V \times V$ to $\mathbb{F}$.
There is a striking similarity between this function and rank $(0,2)$ tensors, which also map from $V \times V$ to $\mathbb{F}$.
In fact, the inner product is one such tensor.
The $\tps{T}^0_2$ tensor $g_{\mu\nu} \vec{e}^\mu \otimes \vec{e}^\nu$ which acts as the inner product is known as the \emph{metric tensor}.
We can then define the inner product based on the component of this tensor, so $(\vec{e}_\mu,\vec{e}_\nu) = g_{\mu\nu}$.
What these values \emph{are} exactly is a more complicated part of the story, and doesn't necessarily have a single definite answer in the context of general relativity, but the point is that we can use this metric tensor as a representation of the inner product.
Then by the linearity of $g_{\mu,\nu}$ and the inner product,
\[ (A^\alpha \vec{e}_\alpha, B^\beta \vec{e}_\beta) = g_{\mu\nu} A^\alpha \langle \vec{e}^\mu, \vec{e}_\alpha \rangle B^\beta \langle \vec{e}^\nu, \vec{e}_\beta \rangle = A^\mu B^\nu g_{\mu\nu}, \]
and so we can extend the inner product to arbitrary vectors with the metric tensor.

\section{Linking $V$ and $V^*$}
Up until this point, we've established that by creating a vector space $V$, we automatically get the dual space $V^*$ without having to do anything else.
However, although these two spaces are clearly related to one another, there hasn't been a canonical way to relate vectors in $V$ to covectors in $V^*$.
The only way we've related them thus far is that we prefer to select a basis for $V^*$ such that $\langle \vec{e}^\nu, \vec{e}_\mu \rangle = \delta_\mu^\nu$, but this is just a choice we made for convenience; its not an implicit law of mathematics that this must be the basis we choose for the dual space.
What this means is that if we take an arbitrary vector $A^\mu \vec{e}_\mu$, we haven't been able to associate it with any particular covector in $V^*$.
All of this changes, however, with the promotion of $V$ to a metric space and the creation of the metric tensor.

We start with an arbitrary vector $A^\alpha \vec{e}_\alpha \in V$, and the metric tensor for $V$, $g_{\mu\nu} \vec{e}^\mu \otimes \vec{e}^\nu \in V^* \otimes V^*$.
The metric tensor normally takes two vectors as arguments, but in this case we will supply it with only the vector we've previously chosen.
If we apply the metric tensor to this vector, something interesting happens; we see that
\begin{align}
    g_{\mu\nu} \vec{e}^\mu \otimes \vec{e}^\nu (A^\alpha \vec{e}_\alpha, \cdot) &=
    g_{\mu\nu} A^\alpha \langle \vec{e}^\mu, \vec{e}_\alpha \rangle \langle \vec{e}^\nu, \cdot \rangle\label{eqn:metric-index-lowering} \\
    &= g_{\mu, \nu} A^\alpha \delta^\mu_\alpha \langle \vec{e}^\nu, \cdot \rangle \nonumber \\
    &= A^\mu g_{\mu\nu} \langle \vec{e}^\nu, \cdot \rangle \nonumber \\
    &= A^\mu g_{\mu\nu} \vec{e}^\nu \in V^* \nonumber.
\end{align}
The result is a covector (since it has a basis of $\vec{e}^\nu$ and a component of $A^\mu g_{\mu\nu}$) which will map a vector to $\mathbb{F}$.
Thus the metric tensor allows us to take a vector and find a corresponding covector from the dual space.
Since the component $A^\mu g_{\mu\nu}$ is already being summed over the $\mu$ index, we often simplify this for brevity's sake into $A_\nu$, making our final covector $A_\nu \vec{e}^\nu$.
Notice how we started with an upper index component $A^\mu$ and ended up with a lower index component, $A_\nu$; this is known as \emph{contracting} against the metric tensor.
Used in this way, the metric tensor kind of has two different functional definitions,
\begin{align*}
    g_{\mu\nu} &: V \times V \to \mathbb{F}, \\
    g_{\mu\nu} &: V \to V^*.
\end{align*}


\section*{RandQS}
\subsubsection*{For any given key $x$ in $S$, bound the expected depth of $x$ in the search tree generated by RandQS}

Let the key $x$ in $S$ have rank $i$ and denote it with $s_{(i)}$. Let $s_{(j)}$ be another key in $S$ with rank $j$ where $j>i$.
Then $s_{(j)}$ is only an ancestor to $s_{(i)}$ if $s_{(j)}$ is chosen as the pivot element before any of the elements in $S_{(ij)} = \{s_{(i)},\ldots,s_{(j)}\}$. For $j<i$ the $s_{(j)}$ element is an ancestor if it is the first element chosen in $S_{(ji)}$.

Define the random variable
$$
X_{ij}=
\begin{cases}
1, & s_{(j)} \text{ is ancestor to } s_{(i)}\\
0, &\text{else}
\end{cases}.
$$

For $j>i$
\\
The probability that $P(X_{ij}) = 1/(j-i+1)$.
\\
For $j=i$
\\
The probability that $P(X_{ij}) = 0$.
\\
For $j<i$
\\
The probability that $P(X_{ij}) = 1/(i-j+1)$.

The sum $\sum_{j=1}^n X_{ij}$ will count the number of ancestors of $s_{(i)}$, and the number of ancestors tells us at which depth the $s_{(i)}$ are in the BST.

So the expected depth $s_{(i)}$ is
\begin{align*}
E \sum_{j=1}^n X_{ij} &=  \sum_{j=1}^n EX_{ij} \\
                      &= \sum_{j<i} EX_{ij} + \sum_{j>i} EX_{ij} \\
											&= \sum_{j<i} P(X_{ij}) + \sum_{j>i} P(X_{ij})\\
											&= \sum_{j<i} \frac{1}{i-j+1} + \sum_{j>i} \frac{1}{j-i+1}\\
											&= \sum_{k=2}^i \frac{1}{k} + \sum_{k=2}^{n-i+1} \frac{1}{k}\\
											&\leq \sum_{k=1}^n \frac{1}{k} + \sum_{k=1}^{n} \frac{1}{k}\\
											&= 2\sum_{k=1}^n \frac{1}{k} = 2H_n = O(\ln(n))
\end{align*}
Where $H_n$ is defined as in the notes.

Hence the expected depth of any key $x$ in $S$ is bounded by $O(\ln(n))$


Let us define a data stream as a function $F(t): \mathbb{N} \rightarrow \R{1}{n} \times \R{1}{m}$ that, for $t \in \mathbb{N}$,
can be defined as
\begin{equation}
  F(t) := (\varphi_t, y_t)
\end{equation}
where $\varphi_t$ and $y_t$ are related according to
\begin{equation}
  y_t = \varphi_t \Theta + \epsilon_t, \quad \mathbb{E}\left[\epsilon_t\right] = 0
\end{equation}
Note that we assume that the output of the underlying linear model with true
parameters $\Theta$ is corrupted at each time step by some additive, zero-mean
white noise $\epsilon_t$. Since the purpose of this note is not to explore the
statistical properties of recursive least squares, we skip over further
clarification of the properties of this noise; results follow from the standard
statistical analysis of linear regression~\cite{bishop2006pattern}.

\subsection{Static Least Squares}
If we wait to observe $T \in \mathbb{N}$ time steps of input-output pairs, we can assemble the matrices $X_T$ and $Y_T$ by vertically stacking the $\varphi_t$ and $y_t$ observations for $t = 1,\ldots,T$. Given these matrices, the least squares estimation problem is formulated as
\begin{align}
  \min_{\hat\Theta} &=  \sum_{t=1}^\top ||y_t - \varphi_t\hat\Theta||_2^2\\
                    &=  ||Y_T - X_T\hat\Theta||_2^2
\end{align}
It is well known (see, e.g.,~\cite{bishop2006pattern}) that the solution to
this problem is given by the ``normal equation''
\begin{equation}
  \label{eq:theta_ls}
  \hat\Theta_{LS}(T) := \left(X_T^\top X_T\right)^{-1}X_T^\top Y_T
\end{equation}