\section{The symmetric group}
Given a set $X$, the class of all bijections $\pi \in Bij(X,X)$ is called the symmetric group
$$S(X),\ \tmr{or}\ S_X.$$
In particular, given the integer interval $N := \{1,\ldots,n\} \subset \zz$ we call
$$S_N := S_n$$
the $n$-th symmetric group.
\subsection{Representation theory of $S_n$}
The first to mention is the
\subsubsection{Trivial representation of $S_n$}
Given a unital commutative ring $R$, then the pair $(R,\rho_R)$
$$S_n \ni \pi \stackrel{\rho_R}{\longmapsto} 1_R$$
is called the trivial representation of $S_n$. We only mention it for completeness - as it will be used later on.
\subsubsection{Permutation representation of $S_n$}
Given $R$ and $S_n$ as above, we call
$$(M,\rho_{\tmr{perm}}),\ M = R^n,\ \pi \longmapsto \sum_{i=1}^n e_{\pi(i)i}$$
the permutation representation of $S_n$. Note, that although most often $R$ to be an algebraic field it need not be - $\zz$ as the initial object in the category of unital rings is sufficient.
\subsubsection{Standard representation of $S_n$}
Again, $R$, $S_n$ and now $M$ as above - we get
$$(M',\rho_{\tmr{std}}),\ M' = \left<e_i - e_n : 1 \leq i \leq n - 1\right> \subsetneq M,\ \pi \longmapsto \left[\sum_i^{n-1} \lambda_i \alpha_i \longmapsto \sum_i \lambda_i \right]$$
the standard representation. Here, $e_i$ denotes the canonical generators of $M$ and $\alpha_i = e_i - e_n$ are our generators. Let us expand:
$$\sum_i \lambda_i \alpha_i = \sum_i \lambda_i (e_i - e_n)\longmapsto \sum_i \lambda_i (e_{\pi(i)} - e_{\pi(n)}) ) = \sum_i \lambda_i(e_{\pi(i)} - e_n + e_n - e_{\pi(n)}) = \sum_i \lambda_i (\alpha_{\pi(i)} - \alpha_{\pi(n)})$$
In this context, $\alpha_{\pi(n)}$ is simply either zero if $\pi(n) = n$, otherwise $\pi(n) < n$ and thus there is an element $\alpha_{\pi(n)}$. I.e.:
$$\alpha_i \longmapsto \alpha_{\pi(i)} - (1 - \delta_{\pi(n),n}) \alpha_{\pi(n)}\ \forall 1 \leq i \leq n - 1.$$
In conclusion:
$$\pi \longmapsto \sum_{i=1}^{n-1} \left(f_{\pi(i)i} - (1 - \delta_{\pi(n),n}) f_{\pi(n)i}\right)$$
where $f_{ji} = [\alpha_i \longmapsto \alpha_j] \in \tmr{End}_R(M')\ \forall 1 \leq i,j \leq n - 1$.
The alert reader will have noticed that we already employed the following
\begin{prop} The permutation representation $(M,\rho_{\tmr{perm}})$ decomposes into the trivial and standard representation:
$$(M,\rho_{\tmr{perm}}) = (M',\rho_{\tmr{std}}) \oplus  (M'',\rho_{\tmr{trivial}}).$$
\end{prop}
We already saw that $M'$ is an $S_n$ submodule of $M$. The trivial representation is given by
$$M'' := \left<\sum_{i=1}^n e_i\right>$$
and the fact that any $\pi \in S_n$ operates trivially on $\sum_i e_i$ (i.e. just a reshuffle of the summands).


\section{Mean Value Theorem for Derivatives}
\begin{theorem}[Mean Value Theorem for Derivatives]
	If $f$ is continuous on the interval $[a,b]$ and differentiable on the interval $(a,b)$, then there exists at least one point in $(a,b)$ such that
	\begin{equation*}
		f^\prime(c) = \frac{f(b)-f(a)}{b-a}.
	\end{equation*}
\end{theorem}

That is, there's at least one point where the instantaneous rate of change and average rate of change are equal.
Another way of visualizing this is that there's al least one point where the tangent and secant lines are parallel.

\begin{figure}[H]
	\label{mvt}
	\centering
	\includegraphics[width = 0.5\textwidth]{./applications_derivative/mvt.png}
	\caption{\hyperref{https://en.wikipedia.org/wiki/Mean\_value\_theorem}{}{}{Wikipedia - Mean Value Theorem}}
\end{figure}

\begin{example}
	A trucker drives 150 miles of a route in 2 hours.
	The speed limit along the route is 65 miles per hour.
	Show that at at least one point, the trucker must have been speeding.
\end{example}
\begin{answer}
	We can model the trucker's position along the route $s$ as a function of time $t$ where $s(0)=0$ and $s(2)=150$.
	We know that velocity is the derivative of position, so $v(t) = s^\prime(t)$.
	It's reasonable to assume that $s$ is differentiable on the interval $[0,2]$.
	So, by the Mean Value Theorem, there must exist a point $c$ where
	\begin{equation*}
		v(t) = \frac{s(2)-s(0)}{2-0} = \frac{150-0}{2} = 75\text{mph}.
	\end{equation*}
	At this point, the trucker was exceeding the speed limit of 65 miles per hour.
\end{answer}

\begin{definition}
	Let $f$ be defined on an interval $I$.
	Let $a$ and $b$ be any two different points in $I$.
	\begin{align*}
		\text{$f$ increases on $I$ if } a < b &\implies f(a) < f(b). \\
		\text{$f$ decreases on $I$ if } a < b &\implies f(a) > f(b).
	\end{align*}
\end{definition}

\begin{corollary}
	Let $f$ be continuous of $[a,b]$ and differentiable on $(a,b)$.
	\begin{align*}
		\text{If $f^\prime > 0$ at every point on $(a,b)$ then $f$ increases on $[a,b]$}. \\
		\text{If $f^\prime < 0$ at every point on $(a,b)$ then $f$ decreases on $[a,b]$}.
	\end{align*}
\end{corollary}

This should make sense given our theorem about local extrema.
If $f$ could still increase/decrease while its derivative was negative/positive, then we couldn't be sure that $f$ is at a local maxima/minima when $f^\prime=0$.

\begin{corollary}
	If $f^\prime(x) = 0$ at all points in an interval $I$, then there is some constant $C$ such that $f(x) = C$ for all points in $I$.
\end{corollary}

This follows from the Mean Value Theorem.
Since $f^\prime = 0$, the numerator in the Mean Value Theorem, $f(b) - f(a)$, must also be 0, meaning $f(b) = f(a) = C$.

\begin{corollary}
	If $f^\prime(x) = g^\prime(x)$ at ever point in some interval $I$, then there is come constant $C$ such that $f(x) = g(x) + C$.
\end{corollary}

That is, functions with the same derivative differ by a constant.
This should make sense given our constant and sum and difference derivative rules.
If we let $h^\prime(x) = f^\prime(x) - g^\prime(x) = 0$ and apply the previous corollary, we get $C$.

\begin{definition}
	A function $F(x)$ is the antiderivative	of $f(x)$ if $F^\prime(x) = f(x)$ for all points in the domain of $f$.
\end{definition}

As we saw in the previous corollary, a function will have infinitely many antiderivatives that differ by a constant.


\subsection{Curl}

The curl of a vector field is defined as:

\(curl \mathbf F=\nabla \times \mathbf F\)

Where: \(\nabla =(\sum_{i=1}^n e_i\dfrac{\delta }{\delta x_i})\)

And: \(\mathbf x\times \mathbf y=\||\mathbf x|| ||\mathbf y|| \sin(\theta )\mathbf n\)

The curl of a vector field is another vector field.

The curl measures the rotation about a given point. For example if a vector field is the gradient of a height map, the curl is \(0\) at all points, however for a rotating body of water the curl reflects the rotation at a given point.

\subsection{Divergence of the curl}

If we have a vector field \(\mathbf F\), the divergence of its curl is \(0\):

\(\nabla . (\nabla \times \mathbf F)=0\)


\section{Polynomial functions}
A very useful family of real functions can be derived using only three fundamental operations: addition, multiplication and exponentiation: the (real) \emph{polynomial functions}. These are functions of the form
\begin{equation}
	P_{n}(x) = a_{0} + a_{1}x + a_{2}x^{2} + a_{3}x^{3} + \cdots + a_{n}x^{n},
	\label{eq:polynomial_function}
\end{equation}
where $a_{0},a_{1},\dots,a_{n}$ are real numbers called the \emph{coefficients} of the polynomial function. Note that $a_{n}\neq0$, i.e. the \emph{degree} of the polynomial function is the index of the highest non-zero coefficient (and thus the highest power in the expression). We also call this the \emph{order} of the polynomial function.

\begin{example}{Polynomial}{}
	The following is a polynomial function of degree $n=6$:
	\[
		P(x) = 4 + 2x - 3x^{2} + 7x^{4} - x^{5} + 3x^{6}.
	\]

	Breaking down this polynomial to its constituent terms:
	\begin{figure}[H]
		\centering
		\begin{tikzpicture}[node distance=15mm]
			\node (P) {$P(x)=$};
			\node[right of=P]  (x0) {$4$};
			\node[right of=x0] (x1) {$+2x$};
			\node[right of=x1] (x2) {$-3x^{2}$};
			\node[right of=x2] (x4) {$+7x^{4}$};
			\node[right of=x4] (x5) {$-x^{5}$};
			\node[right of=x5] (x6) {$+3x^{6}$};

			\node[below of=x0] (a0) {$a_{0}=4$};
			\node[below of=x1] (a1) {$a_{1}=2$};
			\node[below of=x2] (a2) {$a_{2}=-3$};
			\node[below of=x4] (a4) {$a_{4}=7$};
			\node[below of=x5] (a5) {$a_{5}=-1$};
			\node[below of=x6] (a6) {$a_{6}=3$};

			\draw[arrow, thin] (x0) -- (a0);
			\draw[arrow, thin] (x1) -- (a1);
			\draw[arrow, thin] (x2) -- (a2);
			\draw[arrow, thin] (x4) -- (a4);
			\draw[arrow, thin] (x5) -- (a5);
			\draw[arrow, thin] (x6) -- (a6);
		\end{tikzpicture}
	\end{figure}

	Note that $a_{3}$ is missing from the polynomial function (i.e. there is no $x^{3}$ term). This means that $a_{3}=0$.
\end{example}

A shorthand way to write the general form of a polynomial function is by using the \emph{summation notation}:
\begin{equation}
	P(x) = \sum\limits_{k=0}^{n}a_{k}x^{k}.
	\label{eq:summation_notation}
\end{equation}
This notation, called the \emph{Capital-sigma notation}, essentially represents addition of $n$ elements (in the case shown here), each with its own \emph{index of summation}, in this case $i$. The most general form of the summation notation is
\begin{equation}
	\sum\limits_{i=k}^{n}a_{i} = a_{k} + a_{k+1} + a_{k+2} + \cdots + a_{n-1} + a_{n},
	\label{eq:summation_notation_general}
\end{equation}
i.e. the notation tells us to add those elements $a_{i}$ for which $k\geq i\geq n$. Note that in the case of \autoref{eq:summation_notation}, when $k=0,\ x^{k}=x^{0}=1$ and the first term of the polynomial function has no $x$ power (i.e. it is simply $a_{0}$), and when $k=1,\ x^{k}=x^{1}=x$ and thus the second term is $a_{1}x$. We will encounter the summation notation in more details later in the book.

In the special case $n=0$, i.e. when $P(x)=a_{0}$, the function is constant. When $n=1$ the function $P(x)=a_{0}+a_{1}x$ is a line, and when $n=2,\ P(x)=a_{0}+a_{1}x+a_{2}x^{2}$ is a quadratic function.

\begin{example}{Polynomial functions for $n=0,1,2$}{special_polynomials}
	The following graphs represent the polynomial functions of degrees $n=0,1,2$ with coefficients $a_{0}=2,\ a_{1}=1,\ a_{2}=\frac{1}{2}$:

	\begin{figure}[H]
		\centering
		\begin{tikzpicture}
			\tikzset{flbl/.style={draw=#1, thick, fill=white, rounded corners}}
			\begin{axis}[
					graph2d,
					width=9cm, height=9cm,
					xmin=-9, xmax=9,
					ymin=-9, ymax=9,
					domain=-9:9,
					restrict y to domain=-9:9,
					declare function={p0(\x)=2;},
					declare function={p1(\x)=2+\x;},
					declare function={p2(\x)=2+\x+0.5*\x^2;},
				]
				\addplot[function, xred] {p0(x)} node[flbl=xred, below, pos=0.15, yshift=-1mm] {$P_{0}(x)=1$};
				\addplot[function, xblue] {p1(x)} node[flbl=xblue, below, pos=0.1, rotate=45, yshift=-1mm] {$P_{1}(x)=1+x$};
				\addplot[function, xgreen] {p2(x)} node[flbl=xgreen, right, pos=0.2, xshift=-3mm, yshift=5mm] {$P_{2}(x)=1+x+\frac{1}{2}x^{2}$};
			\end{axis}
		\end{tikzpicture}
	\end{figure}
\end{example}

The values $x\in\mathbb{R}$ for which $P(x)=0$ are called the \emph{roots} (also: \emph{zeros}) of the polynomial function.

\begin{example}{Roots of a polynomial function}{}
	The polynomial function $P(x) = 24x - 50x^{2} + 35x^{3} - 10x^{4} + x^{5}$ has the following $5$ roots: $x_{0}=0,\ x_{1}=1,\ x_{2}=2,\ x_{3}=3,\ x_{4}=4$. In the following graph of $P(x)$ the roots are shown as black dots.
	\begin{figure}[H]
		\centering
		\begin{tikzpicture}
			\begin{axis}[
					graph2d,
					width=9cm, height=6cm,
					xmin=-2, xmax=5,
					ymin=-5, ymax=5,
					domain=-2:5,
					restrict y to domain=-6:6,
					declare function={P(\x)=\x*(\x-1)*(\x-2)*(\x-3)*(\x-4);},
				]
				\addplot[function, xred] {P(x)};
				\addplot[black, only marks, mark=*, samples at={0,1,...,4}] {P(x)};
			\end{axis}
		\end{tikzpicture}
	\end{figure}
\end{example}

The maximum number of \textbf{real} roots of a polynomial function with degree $n\geq1$ is $n$, e.g. a polynomial of degree $n=4$ has at most $4$ real roots. This statement is a consequence of a very important theorem called \emph{the fundamental theorem of algebra}, which due to its importance we will mention here without proof:

\begin{theorem}{The fundamental theorem of algebra}{fundamental}
	For any $n\geq1$, the polynomial function $P(z)=a_{0}+a_{1}z+a_{2}z^{2}+\cdots+a_{n}z^{n}$, where $a_{0},a_{1},a_{2},\dots,a_{n}$ are all \textbf{complex numbers} and $a_{n}\neq0$, has $n$ complex roots.
\end{theorem}

Given a polynomial function $P(x)$ with $n$ roots $r_{1},\ r_{2},\ \cdots,\ r_{n}$, the function can be written as a product of terms of the form $x-r_{i}$ (up to a constant), e.g. the polynomial function of degree $n=3$ with roots $-1,1,2$ can be written as
\begin{equation}
	P(x) = (x+1)(x-1)(x-2) = x^{3}-2x^{2}-x+2.
	\label{eq:roots_form}
\end{equation}

\begin{example}{Higher order polynomial functions}{high_order_polynomials}
	The following are the graphs of high-order polynomial functions ($n=3,4,5,6$):

	\begin{figure}[H]
		\captionsetup[subfigure]{labelformat=empty}
		\centering
		\begin{subfigure}[b]{0.475\textwidth}
			\centering
			\begin{tikzpicture}
				\begin{axis}[
						graph2d,
						width=6cm, height=6cm,
						xmin=-3, xmax=3,
						ymin=-3, ymax=3,
						domain=-3:3,
						restrict y to domain=-3:3,
						declare function={P3(\x)=\x^3-\x^2-\x;},
					]
					\addplot[function, xred] {P3(x)};
				\end{axis}
			\end{tikzpicture}
			\caption{\textcolor{xred}{$\bm{x^{3}-x^{2}-x}$}}
		\end{subfigure}
		\hfill
		\begin{subfigure}[b]{0.475\textwidth}
			\centering
			\begin{tikzpicture}
				\begin{axis}[
						graph2d,
						width=6cm, height=6cm,
						xmin=-3, xmax=3,
						ymin=-3, ymax=3,
						domain=-3:3,
						restrict y to domain=-10:10,
						declare function={P4(\x)=\x^4-3*\x^2+\x+1;},
					]
					\addplot[function, xblue] {P4(x)};
				\end{axis}
			\end{tikzpicture}
			\caption{\textcolor{xblue}{$\bm{x^{4}-3x^{2}+x+1}$}}
		\end{subfigure}

		\begin{subfigure}[b]{0.475\textwidth}
			\centering
			\begin{tikzpicture}
				\begin{axis}[
						graph2d,
						width=6cm, height=6cm,
						xmin=-5, xmax=5,
						ymin=-5, ymax=5,
						domain=-5:5,
						restrict y to domain=-10:10,
						declare function={P5(\x)=0.03*(\x^5+3*\x^4-11*\x^3-27*\x^2+10*\x+24);},
					]
					\addplot[function, xgreen] {P5(x)};
				\end{axis}
			\end{tikzpicture}
			\caption{\textcolor{xgreen}{$\bm{\frac{3}{100}\left( x^{5}+3x^{4}-11x^{3}-27x^{2}+10x+24 \right)}$}}
		\end{subfigure}
		\hfill
		\begin{subfigure}[b]{0.475\textwidth}
			\centering
			\begin{tikzpicture}
				\begin{axis}[
						graph2d,
						width=6cm, height=6cm,
						xmin=-5, xmax=5,
						ymin=-5, ymax=5,
						domain=-5:5,
						restrict y to domain=-15:15,
						declare function={P6(\x)=0.01*(x^6-x^5-26*x^4+15*x^3+150*x^2-25*x-5);},
					]
					\addplot[function, xpurple] {P6(x)};
				\end{axis}
			\end{tikzpicture}
			\caption{\textcolor{xpurple}{$\bm{\frac{1}{100}\left( x^{6}-x^{5}-26x^{4}+15x^{3}+150x^{2}-25x-5\right)}$}}
		\end{subfigure}
	\end{figure}
\end{example}

As can be seen in \autoref{:high_order_polynomials}, the maximal number of `bends' in a polynomial function of order $n$ is $n-1$ (i.e. one less than the order of the function).

We will continue to explore polynomial functions in more details in future chapters.


\paragraph{Mean} \hspace{0pt}

\vspace{-20pt}
\begin{gather*}
\mean = \frac{1}{n}\sum_{i=1}^n X_i
\end{gather*}

\paragraph{Mean for grouped data} \hspace{0pt}

\vspace{-5pt}
\begin{gather*}
\mean = \frac{1}{n}\sum_{i=1}^n n_i x_i
\end{gather*}

where $n_i$ is count of $i$th result values, and $x_i$ is the $i$th result value.

%\paragraph{Weighted mean} \nodata

\paragraph{Median} \hspace{0pt}

\vspace{-20pt}
\begin{gather*}
\med = \frac{1}{2} \left( X_{\lfloor \frac{n+1}{2} \rfloor : n} + X_{\lceil \frac{n+1}{2} \rceil : n} \right) =\\
= \begin{cases}
 X_{\frac{n+1}{2} : n} \mbox{ if } n \mbox{ is odd} \\
 \frac{1}{2} \left( X_{\frac{n}{2} : n} + X_{\frac{n+2}{2} : n} \right) \mbox{ if } n \mbox{ is even}
\end{cases}
\end{gather*}

where $x_{k:n}$ is the $k$th sample element taken from sorted sample.

\paragraph{Weighted median}
\begin{gather*}
\med = X_{\med}^L + \frac{b}{n_{\med}} \left( \frac{n}{2} - \sum_{i=1}^{i_{\med}-1} n_i \right)
\end{gather*}

\paragraph{Mode}
is an observation that appears most often.

\paragraph{Trimmed mean} \hspace{0pt}

\vspace{-5pt}
\begin{gather*}
\mean_{t,k} = \frac{1}{n - 2k} \sum_{i=k+1}^{n-k} X_i
\end{gather*}

\paragraph{Winsorized mean}
is thought to decrease the effect of outliers.
\begin{gather*}
\mean_{w,k} = \frac{1}{n} \left[ (k+1)X_{k+1} + \sum_{i=k+2}^{n-k-1} + (k+1)X_{n-k} \right]
\end{gather*}

\paragraph{Geometrical mean}

\[ \mean_g = \sqrt[n]{x_1 x_2 \ldots x_n} \]

\paragraph{Harmonic mean} \hspace{0pt}

\vspace{-10pt}
\[ \mean_h = \frac{1}{ \frac{1}{n} \sum_{i=1}^n \frac{1}{X_i} } \]


\subsection{Information}

\subsubsection{Criteria}

Self information measures surprise of outcome. also called a surprisal.

When we observe an outcome we get information. We can develop a measure for how much information is associated with a specific measurement.

Rule 1: Information is always positive

Rule 2: If \(P(x)=1\), the the information for \(I(P(x))=0\).

Rule 3: If two events are independent, then their information is additive.

\begin{itemize}
\item \(P(C)=P(A)P(B)\)
\item \(I(P(C))=I(P(A)P(B))\)
\item \(I(P(A))+I(P(B))=I(P(A)P(B))\)
\end{itemize}

\subsubsection{Choice of function}

A function which satisifes this is \(I(P(A))=-\log(P(A))\)

Any base can be used. 2 is most common, information is in units of bit then.


\subsection{Product Rule}
\begin{lemma}
	Let $f$ and $g$ be differentiable functions. Then
	\begin{equation}
		(fg)^\prime = fg^\prime + gf^\prime.
	\end{equation}
\end{lemma}
\begin{proof}
	Applying the definition of the derivative and limit properties,
	\begin{align*}
		(fg)^\prime &= \lim_{h \to 0}{\frac{f(x+h)g(x+h) - f(x)g(x)}{h}} \\
		&= \lim_{h \to 0}{\frac{f(x+h)g(x+h)-f(x+h)g(x)+f(x+h)g(x)-f(x)g(x)}{h}} \\
		&= \lim_{h \to 0}{\frac{f(x+h)\left(g(x+h)-g(x)\right)}{h}} + \lim_{h \to 0}{\frac{g(x)\left(f(x+h)-f(x)\right)}{h}} \\
		&= \lim_{h \to 0}{f(x+h)} \lim_{h \to 0}{\frac{g(x+h)-g(x)}{h}} + g(x)\lim_{h \to 0}{\frac{f(x+h)-f(x)}{h}} \\
		&= fg^\prime + gf^\prime
	\end{align*}.
\end{proof}

\begin{example}
	Given the the derivative of $\sin{(x)}$ is $\cos{(x)}$, find the derivative of $x^2\sin{(x)}$.
\end{example}
\begin{answer}
	Using the product rule and power rule,
	\begin{equation*}
		f^\prime(x) = x^2\cos{(x)} + \sin{(x)}2x.
	\end{equation*}
\end{answer}


%\newbreak
\section{KL divergence and cross-entropy}
Cross-entropy minimization is frequently used in optimization and rare-event probability estimation. When comparing a distribution    against a fixed reference distribution, cross-entropy and KL divergence are identical up to an additive constant. See more details in \cite{murphy2012machine,kullback1951information,kullback1997information} and the reference therein.

The KL(Kullback--Leibler) divergence defines a special distance between two discrete probability distributions
$$
p=\left( \begin{array}{ccc}
p_1\\
\vdots \\
p_k
\end{array} \right),\quad  q=\left( \begin{array}{ccc}
q_1\\
\vdots \\
q_k
\end{array} \right)
$$
with $0\le p_i, q_i\le1$ and $\sum_{i=1}^{k}p_i=\sum_{i=1}^{k}q_i=1$ by
\begin{equation}
\label{KL-divergence}
D_{\rm KL}(q,p)= \sum_{i=1}^k q_i\log \frac{q_i}{p_i}.
\end{equation}

\begin{lemma}\mbox{}
$D_{\rm KL}(q,p)$ works like a ``distance" without the symmetry:
	\begin{enumerate}
		\item $D_{\rm KL}(q,p)\ge0$;
		\item $D_{\rm KL}(q,p)=0$ if and only if $p=q$;
	\end{enumerate}
\end{lemma}

\begin{proof}We first note that the elementary inequality
	\begin{equation}
	\log x \le x - 1, \quad\mathrm{for\ any\ }x\ge0,
	\end{equation}
	and the equality holds if and only if $x=1$.
	\begin{equation}
	-D_{\rm KL}(q,p) = - \sum_{i=1}^c q_i\log \frac{q_i}{p_i}   = \sum_{i=1}^k q_i\log \frac{p_i}{q_i} \le \sum_{i=1}^k q_i( \frac{p_i}{q_i}  - 1) = 0.
	\end{equation}
	And the equality holds if and only if
	\begin{equation}
	\frac{p_i}{q_i} = 1 \quad \forall i = 1:k.
	\end{equation}
\end{proof}

Define cross-entropy for distribution $p$ and $q$ by
\begin{equation}\label{Cross-Entropy}
H(q,p) = - \sum_{i=1}^k q_i \log p_i,
\end{equation}
and the entropy for distribution $q$ by
\begin{equation}\label{Entropy}
H(q) = - \sum_{i=1}^k q_i \log q_i.
\end{equation}
Note that
\begin{equation}
D_{\rm KL}(q,p)= \sum_{i=1}^k q_i\log \frac{q_i}{p_i} =  \sum_{i=1}^k q_i \log q_i - \sum_{i=1}^k q_i \log p_i
\end{equation}
Thus,
\begin{equation}\label{KLandEntropy}
H(q,p) = H(q) + D_{\rm KL}(q,p).
\end{equation}
It follows from the relation \eqref{KLandEntropy} that
\begin{equation}
\label{EntropyandKL}
\mathop{\arg\min}_p D_{\rm KL}(q,p)=\mathop{\arg\min}_p H(q,p).
\end{equation}

The concept of cross-entropy  can be used to define a loss function in machine learning and optimization.
Let us assume $y_i$ is the true label for $x_i$, for example $y_i = e_{k_i}$ if $x_i \in A_{k_i}$. Consider the predicted distribution
\begin{equation}\label{key}
\bm p(x; \bm \theta) = \frac{1}{\sum\limits_{i=1}^k e^{w_i x+b_i}}
\begin{pmatrix}
e^{w_1 x+b_1}\\
e^{w_2 x+b_2}\\
\vdots\\
e^{w_k x+b_k}
\end{pmatrix}
= \begin{pmatrix}
p_1(x; \bm\theta) \\
p_2(x; \bm\theta) \\
\vdots \\
p_k(x; \bm\theta)
\end{pmatrix}
\end{equation}
for any data $x \in A$.
By \eqref{EntropyandKL}, the minimization of KL divergence is equivalent to the minimization of the cross-entropy, namely
\begin{equation}
\mathop{\arg\min}_{\theta} \sum_{i=1}^N D_{\rm KL}(y_i,\bm p(x_i; \bm \theta)) = \mathop{\arg\min}_{\theta} \sum_{i=1}^N H(y_i, \bm p(x_i; \bm \theta)).
\end{equation}
Recall that we have all data $D = \{(x_1,y_1),(x_2,y_2),\cdots, (x_N, y_N)\}$.  Then, it is natural to consider the
loss function as following:
\begin{equation}
\sum_{j=1}^N H(y_i, \bm p(x_i; \bm \theta)),
\end{equation}
which measures the distance between the real label and predicted one for all data.
In the meantime, we can check that
\begin{equation}
\begin{aligned}
\sum_{j=1}^N H(y_j, \bm p(x_j; \bm \theta))&=-\sum_{j=1}^N y_j  \cdot \log  \bm p(x_j; \bm \theta )\\
&=-\sum_{j=1}^N  \log p_{i_j}(x_i; \bm \theta) \quad (\text{because}~y_j = e_{i_j}~\text{for}~x_j \in A_{i_j})\\
&=-\sum_{i=1}^k \sum_{x\in A_i}  \log p_{i}(x; \bm \theta) \\
&=-\log \prod_{i=1}^k \prod_{x\in A_i}   p_{i}(x; \bm \theta)\\
& = L(\theta)
\end{aligned}
\end{equation}
with $L(\theta)$ defined in \eqref{logistic} as
\begin{equation}
L(\bm \theta) = - \sum_{i=1}^k \sum_{x\in A_i} \log p_{i}(x;\bm \theta).
\end{equation}

That is to say, the logistic regression loss function defined by likelihood in \eqref{logistic} is exact
the loss function defined by measuring the distance between real label and predicted one via cross-entropy.
We can note
\begin{equation}\label{key}
\min_{\bm \theta} L_\lambda(\bm \theta) \Leftrightarrow \min_{\bm \theta} \sum_{j=1}^N H(y_i, \bm p(x_i; \bm \theta)) + \lambda R(\|\bm \theta\|)
\Leftrightarrow \min_{\bm \theta} \sum_{j=1}^N D_{\rm KL}(y_i, \bm p(x_i; \bm \theta)) + \lambda R(\|\bm \theta\|).
\end{equation}

\endinput


\section{ Gradient descent's algorithm }
\subsection{Introduction}

Gradient descent is a general technique for minimizing  differentiable functions through its slope. \cite{LFD}
It is used to find local minimums. The start point is crucial in the search.

The basic idea is to update the weights using the gradients until it is not possible to continuous minimizing the error.

\medskip

\subsection{Math}

In order to understand the algorithm we are going to define:

Let $w(0) \in \mathbb{R}^d$ be an arbitrary initial point,
$E : \mathbb{R}^d \times \mathbb{R}^d \longrightarrow \mathbb R$
a differentiable function. The learning rate or step size $\eta \in \mathbb{R}^+$
is a experimental coefficient about how much are we going to follow the slope to obtain the new weight.
Let  $w(t) \in \mathbb{R}^d  \quad t \in \mathbb N$ be the weight for $t$ iteration which is defined as

\begin{equation*}
  w(t+1) = w(t) - \eta \nabla E_{in}(w(t))
\end{equation*}

\subsubsection{ Properties}

\begin{itemize}
\item This algorithm gives local minimums.
\item Convergence is not assured in a finite time, so it would be necessary some stop criteria.
\item For a convex function it would be a unique global minimum.
\item The convergence success (in time) depends on the learning rate, $\eta$.
\end{itemize}

\subsection{Algorithm}

The following code snippet implements the algorithm, where $w(0)$ is the $\texttt{initial\_point}$, $E$ is the error, $\nabla E_{in}(w)$
is $\texttt{gradient\_function}$ and
finally $\eta$ is $eta.$ The value $\eta = 0.1$ is a heuristic based on purely practical observation \cite{LFD}.

In order to avoid an infinite search, our stop criteria are a limit in the number of iterations $\texttt{max\_iter}$ and an error tolerance.

\begin{minted}{python}

  def gradient_descent(initial_point, loss_function,
                gradient_function,  eta, max_iter, target_error):
    '''
    initicial point: w_0
    E: error function
    gradient_function
    eta:  step size

    ### stop conditions ###
    max_iter
    target_error

    #### return ####
    (w,iterations)
    w: the coordenates that minimize E
    it: the numbers of iterations needed to obtain w

    '''

    iterations = 0
    error = E( initial_point[0], initial_point[1])
    w = initial_point

    while ( (iterations < max_iter) and(error > target_error)):

        w = w - eta * gradient_function(w[0], w[1])

        iterations += 1
        error = loss_function(w[0], w[1])

    return w, iterations

\end{minted}

\subsection{Problem 1}

We want to solve the following problem: %\\

Use gradient descent's algorithm to find a minimum for the
function

\[E(u,v) = (u^3 e^{(v-2)} - 2* v^2 e^{-u})^2.\]

Set $(u,v)=(1,1)$ as initial point and use learning rate $\eta = 0.1$.

\subsubsection{Compute analytically the gradient of $E(u,v)$}

\begin{multline*}
  \nabla E(u,v) = \left( \frac{\partial}{\partial u}(u^3 e^{(v-2)} - 2* v^2 e^{-u})^2 , \frac{\partial}{\partial v} (u^3 e^{(v-2)} - 2 v^2 e^{-u})^2 \right) = \\
 =  \left(2(u^3 e^{(v-2)} - 2* v^2 e^{-u})(3u^2e^{(v-2)} + 2 v^2 e^{-u} ), 2(u^3 e^{(v-2)} - 2* v^2 e^{-u})(u^3 e^{(v-2)} - 4 v e^{-u}) \right)
\end{multline*}

\subsubsection{Number of iterations and final coordinates.}

Firstable we need to use 64-bits float, so we are going to use the data type $float64$ of numpy library \cite{float64}.

The functions' declaration are:

\begin{minted}{python}
  def dEu(u,v):
    '''
    Partial derivate of E with respect to the variable u
    '''
    return np.float64(
        2
        *( 3* u**2 * np.e**(v-2) + 2*v**2 * np.e**(-u) )
        *( u**3 * np.e**(v-2) - 2*v**2 * np.e**(-u))
    )

def dEv(u,v):
    '''
    Partial derivate of E with respect to the variable v
    '''
    return np.float64(
        2*
        ( u**3 * np.e**(v-2) - 2*v**2 * np.e**(-u) )
        *( u**3 * np.e**(v-2) - 4*v * np.e**(-u))
    )

def gradE(u,v):
    '''
        gradient of E
    '''
    return np.array([dEu(u,v), dEv(u,v)])

\end{minted}

To obtain the number of iterations and the final coordinates, the only thing we need to do is to call $\texttt{gradien\_descent}$ function with the initial conditions:

\begin{minted}{python}
eta = 0.01
max_iter = 10000000000
target_error = 1e-14
initial_point = np.array([1.0,1.0])
w, it = gradient_descent( initial_point,
                          E,
                          gradE,
                          eta,
                          max_iter,
                          target_error )
\end{minted}

The results are:

\begin{itemize}
\item Numbers of iterations: $178.$
\item Final coordinates: $( 1.162 ,  0.924 ).$
\end{itemize}

A 3d graph with the result is

\includegraphics[width=\linewidth]{1_graph.png}

\subsection{Problem 2}

Let's define the function $f(x,y) = (x+2)^2 + 2(y-2)^2 + 2 \sin (2 \pi x) \sin (2 \pi y)$.

\subsubsection{ Use gradient descent to minimize $f$}

The initial point is $(x_0 = -1, y_0 = 1)$,
learning rate is $\eta = 0.01$ and the maximum number of iterations must
be $50$.  Plot the result and repeat the experiment with  $\eta = 0.1$.

Firstly we are going to calculate partial derivatives and gradient of $f$.

\begin{equation*}
  \frac{\partial }{\partial x} f = 2 (x + 2) + 2 \sin (2 \pi y) \cos ( 2 \pi x) 2 \pi =  2 (x + 2) +  4 \pi \sin (2 \pi y) \cos ( 2 \pi x)
\end{equation*}

\begin{equation*}
  \frac{\partial }{\partial y} f = 4 (y - 2) +  4 \pi \sin (2 \pi x) \cos ( 2 \pi y)
\end{equation*}

It is important to realise that $f(x,y) <0$ for some values in $\mathbb R^3$ so the error target has been omitted in this algorithm.

Now the new algorithm is
\begin{minted}{python}
  def gradient_descent_trace(initial_point, loss_function,
    gradient_function,  eta, max_iter):
    '''
    initicial point: w_0
    loss_function: error function
    gradient_function
    eta:  step size

    ### stop conditions ###
    max_iter

    #### return ####
    (w,iterations)
    w: the coordenates that minimize loss_function
    it: the numbers of iterations needed to obtain w

    '''

    iterations = 0
    error = loss_function( initial_point[0], initial_point[1])
    w = [initial_point]

    while iterations < max_iter:

        new_w = w[-1] - eta * gradient_function(w[-1][0], w[-1][1])

        iterations += 1
        error = loss_function(new_w[0], new_w[1])
        w.append( new_w )

    return w, iterations

  \end{minted}

  After 50 iterations for  $\eta = 0.01$, the  final coordinates are $(-1.269, 1.287)$ and their value is $-0.381$. The graph which shows the relation between iterations and the function minimization is

  \includegraphics[width=\linewidth]{1_3_smaller_eta}

  As far as we have seen, before the $10^{th}$ iteration we are really close to the minimum and stay there without fluctuating.

  On the other hand, after 50 iterations for  $\eta = 0.1$ the final coordinate is $(-2.939, 1.608)$ and its value is $f(-2.939, 1.608) = 0.724$, so as we can see, this result is worse than the last one.

  In the following graph we can see how the images' value fluctuates iteration by iteration.

  \includegraphics[width=\linewidth]{1_3_bigger_eta}.

 The reason for this irregularity  is that the step size is too big, so it skips the minimum.

 We can also  compare the two experiment in the following graph.

\includegraphics[width=\linewidth]{1_3_comparation_eta}.

    Moreover, based on its mathematical proof, which use Taylor's series, we know that it should be small, but if it is too small the algorithm will never reach the minimum in time.

 Let's see a new example: now $\eta = 10^{-14}$, after 50 iterations  the final coordinates are $(-1, 1)$  and the value is $3$, so this new selection is even worse that the one with the bigger step's size, although it goes without oscillating.

 \includegraphics[width=\linewidth]{1_3_epsilon_eta}

 \includegraphics[width=\linewidth]{1_3_eta_and_smaller.png}

  As a conclusion, a priory, it is difficult to select a step's size value, each problem should have an appropriate one and the selection must be empirical. Even though some heuristic \cite{LFD} tell that $\eta = 0.01$ it is a good try.

\subsubsection{Minimum value }

Before running the algorithm is important to think about a good value for the learning rate $\eta$. Based on the last section, $\eta = 0.1$ is a good one.

The results are

\begin{center}
  \begin{tabular}{ |c|c|c| }
    \hline
    Initial point  & Final coordinates & Final value  \\
    \hline

    (-0.5 -0.5) &  (-0.793 -0.126) &   9.125 \\
(1 1) &  (0.677 1.29) &   6.437 \\
( 2.1 -2.1) &  ( 0.149 -0.096 ) &   12.491 \\
(-3  3) &  (-2.7315  2.713) &  -0.381 \\
(-2  2) &  (-2.  2.) &  0 \\

 \hline
\end{tabular}
\end{center}

This example gives the idea that the local minimums found depend on the start point $w_0$ and a priory, unless we know some properties
of the function such as convexity or monotony we are not able to assure that the minimum found is global.

Under a mathematical point of view, to study a function's monotony we need some more complex tools and conditions, such as solve equations; something that
the majority of times is impossible.

Fortunately, for the error functions such a the mean quadratic error, this is totally possible.

\subsection{ Final conclusion about finding global functions' minimum by gradient descent}

To sum up this first chapter,  gradient descent's algorithm is a technique to minimize differentiable functions. It dramatically depends  on the initial point and the learning rate. Moreover, it does not give global  minimums unless the  function is convex.

The computational cost of the function is $\mathcal O (N i)$ where $N$ is the size of the data set and $i$ the maximum number of iterations.

Some useful examples of functions that can be successfully mimimized with used this algorithm are the mean quadratic's error or the logistic's function.


\subsection{Arc length}

Let $g(t)$ be a function that draws a curve.
The arc length from $g(a)$ to $g(b)$ is given by
$$\int_a^b|g'(t)|\,dt$$
where $|g'(t)|$ is the length of the tangent vector at $g(t)$.
The integral sums over all of the tangent lengths to arrive at the total length
from $a$ to $b$.
For example, let us measure the length of the following curve.

{\color{blue}
\begin{verbatim}
xrange = (0,1)
yrange = (0,1)
draw(x^2)
\end{verbatim}
}

\begin{center}
\includegraphics[scale=0.2]{arc.png}
\end{center}

\noindent
A suitable $g(t)$ for the arc is
$$g(t)=(t,t^2),\quad0\le t\le1$$
Hence one Eigenmath solution for computing the arc length is

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
x = t
y = t^2
g = (x,y)
defint(abs(d(g,t)),t,0,1)
\end{Verbatim}

\noindent
$\displaystyle \tfrac{1}{2}\;5^{1/2}+\tfrac{1}{4}\log(5^{1/2}+2)$

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
float
\end{Verbatim}

\noindent
$\displaystyle 1.47894$

\bigskip
\noindent
As expected, the result is greater than $\sqrt2\approx1.414$,
the length of the
diagonal from $(0,0)$ to $(1,1)$.

\bigskip
\noindent
The result seems rather complicated given that we
started with a simple parabola.
Let us inspect $|g'(t)|$ to see why.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
g
\end{Verbatim}

\noindent
$\displaystyle g=\begin{bmatrix}t\\ t^2\end{bmatrix}$

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
d(g,t)
\end{Verbatim}

\noindent
$\displaystyle \begin{bmatrix}1\\ 2t\end{bmatrix}$

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
abs(d(g,t))
\end{Verbatim}

\noindent
$\displaystyle (4t^2+1)^{1/2}$

\bigskip
\noindent
The following script does a discrete computation of the arc length
by dividing the curve into 100 pieces.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
g(t) = (t,t^2)
h(k) = abs(g(k/100.0) - g((k-1)/100.0))
sum(k,1,100,h(k))
\end{Verbatim}

\noindent
$\displaystyle 1.47894$

\bigskip
\noindent
As expected, the discrete result matches the analytic result.

\bigskip
\noindent
Find the length of the curve $y=x^{3/2}$ from the origin to
$x=\tfrac{4}{3}$.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
x = t
y = x^(3/2)
g = (x,y)
defint(abs(d(g,x)),x,0,4/3)
\end{Verbatim}

\noindent
$\displaystyle \tfrac{56}{27}$

\bigskip
\noindent
Because of the way $t$ is substituted for $x$,
the following code yields the same result.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
g = (t,t^(3/2))
defint(abs(d(g,t)),t,0,4/3)
\end{Verbatim}

\noindent
$\displaystyle \tfrac{56}{27}$

\subsection{Line integrals}
There are two different kinds of line integrals,
one for scalar fields and one
for vector fields.
The following table shows how both are based on the calculation of
arc length.

\begin{center}
\begin{tabular}{|l|l|l|}
\hline
& Abstract form
& Computable form
\\
\hline
 & &\\
Arc length
& $\displaystyle{\int_C ds}$
& $\displaystyle{\int_a^b |g'(t)|\,dt}$\\
 & &\\
\hline
 & & \\
Line integral, scalar field
& $\displaystyle{\int_C f\,ds}$
& $\displaystyle{\int_a^b f(g(t))\,|g'(t)|\,dt}$\\
& &\\
\hline
 & & \\
Line integral, vector field
& $\displaystyle{\int_C(F\cdot u)\,ds}$
& $\displaystyle{\int_a^b F(g(t))\cdot g'(t)\,dt}$\\
 & & \\
\hline
\end{tabular}
\end{center}

\noindent
For the vector field form, the symbol $u$ is the unit tangent vector
$$u=\frac{g'(t)}{|g'(t)|}$$
The length of the tangent vector cancels with $ds$
as follows.
$$\int_C(F\cdot u)\,ds
=\int_a^b\bigg(F(g(t))\cdot\frac{g'(t)}{|g'(t)|}\bigg)\,\bigg(|g'(t)|\,dt\bigg)
=\int_a^b F(g(t))\cdot g'(t)\,dt
$$

\noindent
Evaluate
$$\int_Cx\,ds\quad\hbox{and}\quad\int_Cx\,dx$$
where $C$ is a straight line from $(0,0)$ to $(1,1)$.

\bigskip
\noindent
What a difference the measure makes.
The first integral is over a scalar field and the second is over a vector field.
This can be understood when we recall that
$$ds=|g'(t)|\,dt
$$
Hence for $\int_Cx\,ds$ we have

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
x = t
y = t
g = (x,y)
defint(x abs(d(g,t)),t,0,1)
\end{Verbatim}

\noindent
$\displaystyle \frac{1}{2^{1/2}}$

\bigskip
\noindent
For $\int_Cx\,dx$ we have

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
x = t
y = t
g = (x,y)
F = (x,0)
defint(dot(F,d(g,t)),t,0,1)
\end{Verbatim}

\noindent
$\displaystyle \tfrac{1}{2}$

\bigskip
\noindent
The following line integral problems are from
{\it Advanced Calculus, Fifth Edition} by Wilfred Kaplan.

\bigskip
\noindent
Evaluate $\int y^2\,dx$ along the straight
line from $(0,0)$ to $(2,2)$.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
x = 2t
y = 2t
g = (x,y)
F = (y^2,0)
defint(dot(F,d(g,t)),t,0,1)
\end{Verbatim}

\noindent
$\displaystyle \tfrac{8}{3}$

\bigskip
\noindent
Evaluate $\int z\,dx+x\,dy+y\,dz$
along the path
$x=2t+1$, $y=t^2$, $z=1+t^3$, $0\le t\le 1$.

\begin{Verbatim}[formatcom=\color{blue},samepage=true]
x = 2t+1
y = t^2
z = 1+t^3
g = (x,y,z)
F = (z,x,y)
defint(dot(F,d(g,t)),t,0,1)
\end{Verbatim}

\noindent
$\displaystyle \tfrac{163}{30}$


\section{Matrix Factorization}
\begin{theo}
If Gaussian elimination can be performed on the linear system $Ax=b$ without row interchanges, then the matrix $A$ can be factored into the product of a lower-triangular maxtrix $L$ and an upper-triangular matrix $U$, that is $A=LU$, where $m_{ji}=a_{ji}^{(i)}/a_{ii}^{(i)}$
\[
U=
\begin{pmatrix}
a_{11}^{(1)} & a_{12}^{(1)} & \cdots       & a_{1n}^{(1)} \\
0            & a_{22}^{(2)} & \ddots       & \vdots       \\
\vdots       & \ddots       & \ddots       & a_{n-1,n}^{(n-1)} \\
0            & \cdots       & 0            & a_{nn}^{(n)}
\end{pmatrix}
\quad
U=
\begin{pmatrix}
1            & 0            & \cdots       & 0            \\
m_{21}       & \ddots       & \ddots       & \vdots       \\
\vdots       & \ddots       & \ddots       & 0            \\
m_{n1}       & \cdots       & m_{n,n-1}    & 1
\end{pmatrix}
\]
\end{theo}


\section{Vector and Matrix Derivatives}
\label{sec:derivatives}

Derivatives involving matrices and vectors can look nonintuitive when the usual symbolic matrix notation is used, but can be derived handily when index notation is used. A very concise and helpful resource for this is \citeasnoun{barnesmatrixdiff}.

\subsection{Jacobian}
It is particularly helpful to remember the Jacobian, which is the derivative of a function with respect of a vector. The Jacobian of some function $f: \mathbb{R}^n \rightarrow \mathbb{R}^m$ is:

\begin{equation}
\frac{\mathrm{d}  \mathbf{f}(\mathbf{x})}{\mathrm{d} \mathbf{x}}=\left[\frac{\partial \mathbf{f}}{\partial x_1}, \hdots, \frac{\partial \mathbf{f}}{\partial x_n} \right]=\left[\begin{array}{ccc}
\frac{\partial  f_1}{\partial  x_1} & \hdots & \frac{\partial  f_1}{\partial  x_n} \\
\vdots & \vdots & \vdots \\
\frac{\partial  f_m}{\partial  x_1} & \hdots & \frac{\partial  f_m}{\partial  x_n} \\
\end{array}\right]
\end{equation}

I enjoy writing the gradient $\frac{\mathrm{d}}{\mathrm{d}\mathbf{x}}$ as $\nabla_\mathbf{x}$. The relationships below can all be derived as applications of the Jacobian.

\begin{equation}
\begin{array}{l}
\nabla_\mathbf{x} \left(\mathbf{u}^T\mathbf{x}\right) = \left[\frac{\partial }{\partial x_1}\left(\sum_i u_i x_i\right),...,\frac{\partial }{\partial x_n}\left(\sum_i u_i x_i\right)\right] = \mathbf{u}^T\\
\\
\nabla_\mathbf{x} \left(\mathbf{x}^T\mathbf{u}\right) = \left[\frac{\partial }{\partial x_1}\left(\sum_i u_i x_i\right),...,\frac{\partial }{\partial x_n}\left(\sum_i u_i x_i\right)\right] = \mathbf{u}^T\\
\\
\nabla_\mathbf{x} \left(\mathbf{x}^T\mathbf{x}\right) = \left[\frac{\partial }{\partial x_1}\left(\sum_i x_i^2\right),...,\frac{\partial }{\partial x_n}\left(\sum_i x_i^2\right)\right] = 2\mathbf{x}^T\\
\\
\nabla_\mathbf{x} \left(\mathbf{Ax}\right) = \left[
\begin{array}{ccc}
\underbrace{\frac{\partial }{\partial x_1}\left(\sum_i A_{1i} x_i\right)}_{A_{11}} &...& \underbrace{\frac{\partial }{\partial x_n}\left(\sum_i A_{1i} x_i\right)}_{A_1n}\\
\vdots&\vdots&\vdots\\
\underbrace{\frac{\partial }{\partial x_1}\left(\sum_i A_{ni} x_i\right)}_{A_{n1}} &...& \underbrace{\frac{\partial }{\partial x_n}\left(\sum_i A_{ni} x_i\right)}_{A_{nn}}\\
\end{array}\right] = \mathbf{A}
\end{array}
\end{equation}

\subsection{Inverse Function Theorem}
\label{sec:inverse_function_theorem}
The inverse function theorem gives a sufficient condition for the invertibility of a function near some point in its domain. If the derivative $f'$ of a function $f$ is continuous and non-zero near some point $a$ within its domain, then the function is invertible near that point. If $b = f(a)$, then:

\begin{equation}
\frac{d\left[f^{-1}(b)\right]}{dx} = \frac{1}{\frac{df(a)}{dx}}
\end{equation}

That is, the derivative of the inverse function at a point $b=f(a)$ of the range, is the reciprocal of the derivative of the function near the point $a$ in the domain. This extends to multivariable calculus. Given a function $\mathbf{f}: \mathbf{x} \rightarrow \mathbf{y}$:

\begin{equation}
\nabla_\mathbf{y}\left[\mathbf{f}^{-1}\right] = \left[\nabla_\mathbf{x} \mathbf{f} \right]^{-1}
\end{equation}

In words: the Jacobian of the inverse function at the point $\mathbf{b} = \mathbf{f}(\mathbf{a})$ is the matrix inverse of the Jacobian of the function at the point $\mathbf{a}$. The sufficient condition is that the Jacobian $\nabla_\mathbf{x}\mathbf{f}$ is continuous and \textit{nonsingular} near $\mathbf{a}$.

\subsection{Critical Points}
Critical points are points where the Jacobian does not have maximal rank. In case of a square Jacobian, this means that the Jacobian is singular.

\subsection{Differential Volume Element, Change of Variables}
The Jacobian is used when transforming between different coordinate systems. Consider a transformation $\mathbf{x} = \mathbf{H}(\mathbf{y})$, then:

\begin{equation}
\mathrm{d}^n x = \left|\nabla_\mathrm{y} \mathbf{H} \right| \mathrm{d}^n y
\end{equation}

And:

\begin{equation}
\int_\mathbf{x} \mathrm{d}^n\mathbf{x} f(\mathbf{x}) = \int_\mathbf{y} \mathrm{d}^n\mathbf{y} \left|\nabla_\mathrm{y} \mathbf{H} \right| f(\mathbf{H}(\mathbf{y}))
\end{equation}

Alternatively, if $\mathbf{y} = \mathbf{H}^{-1}(\mathbf{x})$:

 \begin{equation}
 \begin{array}{rl}
 \mathrm{d}^n y &= \left|\nabla_\mathrm{x} \mathbf{H}^{-1}(\mathbf{x})\right| \mathrm{d}^n x\\
 &= \left| \left[ \nabla_\mathrm{y} \mathbf{H}(\mathbf{y}) \right]^{-1} \right| \mathrm{d}^n x\\
 \mathrm{d}^n x &= \frac{1}{\left| \left[ \nabla_\mathrm{y} \mathbf{H}(\mathbf{y}) \right]^{-1} \right|}  \mathrm{d}^n y
 \end{array}
 \end{equation}

The Jacobian has to be nonsingular within the domain of integration. This implies that $\mathbf{x}$ and $\mathbf{y}$ have to have the same dimension. In the context of probability theory that sometimes requires artificially defining additional variables so that $\mathbf{H}$ is bijective because the quantity of interest has lower dimension (for example, if you calculate the mean of a random variable).

\subsection{Hessian}
\label{sec:hessian}

The Hessian is the second derivative of a scalar valued function $f:\mathbb{R}^{n} \rightarrow \mathbb{R}$ with respect to a vector, i.e. $\nabla\cdot\nabla f$. The elements are $\mathbf{H}(f)_{i,j} = \frac{\partial^2 f}{\partial x_i\partial x_j}$.

\begin{equation}
\mathbf{H}(f) = \left[\begin{array}{cccc}
\frac{\partial^2f}{\partial x_1^2}&\frac{\partial^2f}{\partial x_1\partial x_2} & \hdots & \frac{\partial^2f}{\partial x_1\partial x_n}\\
\frac{\partial^2f}{\partial x_2\partial x_1}&\frac{\partial^2f}{\partial x_2^2} & \hdots & \frac{\partial^2f}{\partial x_1\partial x_n}\\
\vdots & \vdots & \ddots & \vdots \\
\frac{\partial^2f}{\partial x_n\partial x_1}&\frac{\partial^2f}{\partial x_n\partial x_2} & \hdots & \frac{\partial^2f}{\partial x_n^2}
\end{array} \right]
\end{equation}

The Hessian of a vector valued function $f:\mathbb{R}^n \rightarrow \mathbb{R}^m$ is a third order tensor with elements  $\mathbf{H}(\mathbf{f})_{i,j,k} = \frac{\partial^2 f_k}{\partial x_i\partial x_j}$.

\subsubsection{Testing Convexity}
The definiteness (cf. section \ref{sec:definite}) of the Hessian is used to test convexity.

\begin{tabular}{ll}
$\mathbf{H}\succeq 0$ & convex\\
$\mathbf{H}\succ 0$ & strictly convex\\
$\mathbf{H}\preceq 0$ & concave\\
$\mathbf{H}\prec 0$ & strictly concave\\
\end{tabular}

If this holds at a point, the property is local (for example at a local maximum or minimum), and if it holds everywhere on the domain, then the property is global.


\section{Norm and distance}
\subsection{Norm}
\textit{Euclidean norm} (or just norm) is\\
$||x||= \sqrt{x_1^2+x_2^2+...+x_n^2}= \sqrt{x^Tx}$\\
\textbf{Properties}\\
\textbullet homogeneity: $||\beta x|| = |\beta ||x|||$\\
\textbullet triangle inequality: $||x + y|| \leq ||x|| + ||y||$\\
\textbullet non negativity: $||x|| \geq 0$\\
\textbullet definiteness: $||x|| = 0$ only if x = 0\\
\textit{positive definiteness} = non negativity + definiteness\\
\textbf{rms}(x) = $\sqrt{\frac{x_1^2+...+x_n^2}{n}} = \frac{||x||}{\sqrt{n}}$ \\
\textbf{Norm of block vectors}
$||(a,b,c)|| = \sqrt{||a||^2 + ||b||^2 + ||c||^2} = ||(||a||, ||b||, ||c||)||$\\
\textbf{Chebyshev inequality}
% Todo
Todo
\subsection{Distance}
$\textbf{dist}(a,b) = ||a - b||$\\
\textit{Triangle Inequality}: $||a - c|| = ||(a - b) + (b - c)|| \leq ||a-b|| + ||b-c||$\\
$z_j$ is the nearest neighbor of x if $||x-z_j|| \leq ||x-z_i||, i=1,..,m$\\

\subsection{Standard Deviation}
de-meaned vector: $\tilde{x} = x - \textbf{avg}(x)\textbf{1}$\\
standard deviation: \textbf{std}(x) = \textbf{rms}($\tilde{x}$) = $\frac{||x - (\textbf{1}^Tx/n)\textbf{1}||}{\sqrt{n}}$\\
${\textbf{rms}(x)}^2 = {\textbf{avg}(x)}^2 + {\textbf{std}(x)}^2$\\
By Chebyshev inequality, $|x_i - \textbf{avg}(x)| \geq \alpha \textbf{std}(x)$ is no more than $1/\alpha^2$ (for
$\alpha$ > 1)
\textit{Cauchy–Schwarz inequality}: $|a^Tb| \leq \Vert a\Vert \Vert b \Vert$
\subsection{Angle}
angle between two nonzero vectors a, b defined as\\
$\angle(a,b) = arccos(\frac{a^Tb}{\Vert a \Vert \Vert b \Vert})$\\
$a^Tb = \Vert a \Vert \Vert b \Vert cos (\angle(a,b))$\\
\textbf{Classification of angles}\\

$\theta = \pi/2$: $a \perp b$\\
$\theta = 0$: $a^Tb = \Vert a \Vert \Vert b \Vert$\\
$\theta = \pi = 180^{\circ}$ : $a^Tb = -\Vert a \Vert \Vert b\Vert$\\
$\theta \leq \pi/2 = 90^{\circ} = a^Tb \geq 0$\\
$\theta \geq \pi/2 = 90^{\circ} = a^Tb \leq 0$\\
\textbf{Correlation Coeficient}
$\rho = \frac{\tilde{a}^T\tilde{b}}{\Vert\tilde{a}\Vert\Vert\tilde{b}\Vert}$\\
With $u = \tilde{a}/\textbf{std}(a)$ \& $u = \tilde{b}/\textbf{std}(b)$, \\
$\rho = u^Tv/n$ where $\Vert u \Vert = \Vert v \Vert = n$
\subsection{Complexity}
% Todo
Todo


\subsection{Determinants}
\noindent
The determinant of a matrix is a signed number that tells by how much the transformation represented by a matrix scales volumes in a space.
The number is negative if the space was ``flipped'' during a transformation.
The number is zero if the dimension of the output space is less than that of the input space.\\

\noindent
The determinant is only defined for square matrices. It's easiest to understand the definition of a determinant recursively.
\begin{align*}
	\det{\left[ a \right]} &= \lvert a \rvert = a \\
	\det{\left[
		\begin{array}{cc}
			a & b \\
			c & d
		\end{array}
		\right]} &= \begin{array}{|cc|}
		a & b \\
		c & d
	\end{array} = ad - bc.
\end{align*}
We can define $a_{ij}$ as the entry in the ith row and jth column of matrix $A$ and $A_{ij}$ as the adjudicate matrix, which is the matrix $A$ if row $i$ and column $j$ were removed. This allows us to write a general formula for the determinant.
\begin{definition}
	\begin{equation*}
		\det{A} = \sum_{j=1}^{n}{\left(-1\right)^{i+j}a_{ij}A_{ij}} \text{ (for fixed i)} = \sum_{i=1}^{n}{\left(-1\right)^{i+j}a_{ij}A_{ij}} \text{ (for fixed j)}
	\end{equation*}
\end{definition}
\noindent
This formula allows us to use any row or column to calculate the determinant, which is especially useful if a certain row contains lots of 0's.\\

\noindent
Below are some properties of the determinant for some $n \times n$ matrix $A$ and scalar $\lambda$.
\begin{align*}
	\det{I_n} &= 1 \\
	\det{(A^T)} &= \det{A} \\
	\text{If $A$ is invertible, } \det{(A^{-1})} &= \frac{1}{\det{A}} \\
	\det{(\lambda A)} &= \lambda^n\det{A} \\
	\det{(AB)} &= \det{A}\det{B} \\
	\text{If $A$ is triangular, } \det{A} &= \prod_{i=1}^{n}{a_{ii}}
\end{align*}

\begin{example}
	Find the determinant of the following 3 x 3 matrix.
	\begin{equation*}
		A = \begin{bmatrix}
			1 & 3 & 7 \\
			0 & 2 & -1 \\
			2 & 7 & 9
		\end{bmatrix}
	\end{equation*}
\end{example}
\noindent
We'll use the first column since it has only two non-zero entries.
\begin{equation*}
	\begin{bmatrix}
		1 & 3 & 7 \\
		0 & 2 & -1 \\
		2 & 7 & 9
	\end{bmatrix} = 1 \text{ } \begin{array}{|cc|}
		2 & -1 \\
		7 & 9
	\end{array} + 2 \text{ } \begin{array}{|cc|}
		3 & 7 \\
		2 & -1
	\end{array} = (18+7) + 2(-3-14) = -9.
\end{equation*}


\section{Parametric Inference}

In parametric inference, the quantity of interest might be some function $T(\theta)$. The sought-after parameters are \textit{parameters of interest} and additional parameters that emerge as part of the model are \textit{nuisance parameters}. Parametric inference deals with creating parametric estimators.

\subsection{Method of Moments}
The method of moments relies on a system of linear equations to link estimators of moment to sample moments. The $j$th moment is given by:

\begin{equation}
\alpha_j = \int x^j dF_{\theta}(x)
\end{equation}

The $j$th sample moment is given by:

\begin{equation}
\hat{\alpha}_j = \frac{1}{n} \sum_{i=1}^ n X_i^j
\end{equation}

The method of moments estimator $\hat{\theta}_n$ is defined to be the value of $\theta$ so that:

\begin{equation}
\begin{array}{l}
\alpha_1(\hat{\theta}_n) = \hat{\alpha}_1\\
\alpha_2(\hat{\theta}_n) = \hat{\alpha}_2\\
\alpha_3(\hat{\theta}_n) = \hat{\alpha}_3\\
\alpha_4(\hat{\theta}_n) = \hat{\alpha}_4\\
\vdots
\end{array}
\end{equation}

The method of moments estimator satisfies:

\begin{enumerate}
\item The estimate $\hat{\theta}_n$ exists with probability tending to 1.
\item The estimate is consistent: $\hat{\theta}_n \xrightarrow{P}\theta$ (it converges in probability)
\item The estimate is asymptotically normal (cf. \cite{wasserman2003all}, pp.122)
\end{enumerate}

\subsection{Maximum Likelihood Estimation}

The maximum likelihood estimator is the value $\hat{\theta}$ that maximizes the joint probability density of the data, called the likelihood function.

For i.i.d. random variables with pdf $f(x;\theta)$, the likelihood function is:

\begin{equation}
\mathscr{L}_n(\theta) = \prod_{i=1}^n f(X_i;\theta)
\end{equation}

And the log likelihood functin is $l_n(\theta) = \log \mathscr{L}_n(\theta)$. Since $\log$ is a monotonic function, maximizing the log-likelihood yields the same estimator as maximizing the likelihood directly. Log-likelihood is often easier to deal with, and alleviates numerical issues associated with the often sharply spiked likelihood function.

MLE estimators have a flurry of desirable properties under certain smoothness conditions on the density function.

\begin{itemize}
\item Consistency: convergence in probability upon the true value
\item Equivariance: if $\hat{\theta}_n$ is the MLE of $\theta$ then $g(\hat{theta}_n)$ is the MLE of $g(\theta)$
\item Asymptotically Normal
\item Asymptotically Optimal / Efficient: smallest variance, at least for large samples.
\item Approximately the Bayes estimator.
\end{itemize}

\subsection{Parametric Confidence Intervals}

Confidence intervals for infered parameters in the parametric setting can be derived, for example, using the delta method (assuming that the estimators are asymptotically normal) or using parametric bootstrap. In the nonparametric case, bootstrap sampled from the empirical CDF. In the parametric case, bootstrap sample from the density $f(X;\hat{\theta})$ where $\hat{\theta}$ is the estimator.

\section{Score Function, Fisher Information}
Given some pdf $f(X;\theta)$, the score function is given by:

\begin{equation}
s(X;\theta) \frac{\partial \log f(X;\theta)}{\partial \theta}
\end{equation}

The Fisher information is the variance of the score function at each datapoint:

\begin{equation}
I_n(\theta) = \mathbb{V}_{\theta} \left(\sum_{i=1}^n s(X_i ; \theta) \right)
\end{equation}


\section{Vector Spaces and Subspaces}
Let $F$ be an arbitrary field.
\begin{definition}
    An $F$-vector space (or a vector space over $F$) is an abelian group $(V,+)$ equipped with a function $F\times V\to V,(\lambda,v)\mapsto \lambda v$ such that for any $v,v_1,v_2\in V,\lambda,\mu\in F$:\\
    1. $\lambda(v_1+v_2)=\lambda v_1+\lambda v_2$.\\
    2. $(\lambda_1+\lambda_2)v=\lambda_1v+\lambda_2v$.\\
    3. $\lambda(\mu v)=(\lambda\mu v)$.\\
    4. $1v=v$.\\
    This function is often called scalar multiplication of the vector space.
\end{definition}
\begin{example}
    1. Take $n\in\mathbb N$.
    Then the set of $n$-tuples in $F$, denoted $F^n$, is a vector space under the operations
    \begin{align*}
        (x_1,\ldots,x_n)+(y_1,\ldots,y_n)&=(x_1+y_1,\ldots,x_n+y_n)\\
        \lambda(x_1,\ldots,x_n)&=(\lambda x_1,\ldots,\lambda x_n)
    \end{align*}
    2. For any set $X$, write $\mathbb R^X=\{f:X\to\mathbb R\}$.
    It is a vector space over $\mathbb R$ via
    $$(f_1+f_2)(x)=f_1(x)+f_2(x),(\lambda f)(x)=\lambda f(x)$$
    3. The set $M_{n,m}(F)$ consisting of $F$-valued $n\times m$ matrices is a vector space by interpreting it as $F^{n\times m}$.
\end{example}
\begin{remark}
    The axioms of scalar multiplication imply that $0v=0$ for any $v\in V$, as one can check.
\end{remark}
\begin{definition}[Subspace]
    Let $V$ be a vector space over $F$.
    A subset $U\subset V$ is a subspace of $V$ (or $U\le V$ as vector spaces) iff $U\le V$ as subgroups and $\forall\lambda\in F,u\in U,\lambda u\in U$.
\end{definition}
So basically, a subgroup $U$ is a subspace if we can properly restrict the original scalar product to make it a vector space over $F$ as well.
One can also check oneself that a subspace of a subspace is also a subspace of the original space.
\begin{example}
    Take $V=\mathbb R^{\mathbb R}$.
    $C(\mathbb R)\le V$ (the set of continuous functions $\mathbb R\to\mathbb R$) is a subspace of $V$, and the set of polynomials is a subspace of $C(\mathbb R)$.\\
    Take $V=\mathbb R^3$, then a line is a subspace iff it passes through the origin.
    A plane is a subspace iff it passes through the origin as well.
\end{example}
\begin{proposition}
    Let $V$ be an $F$-vector space and $U,W\le V$ as vector spaces, then $U\cap W\le V$ as vector spaces.
\end{proposition}
\begin{proof}
    Just check.
\end{proof}
However, the union of two subspaces is generally not a subspace unless one is contained in the other already.
\begin{example}
    Take $V=\mathbb R^2$ and $U,W$ two axes, then $(1,0)+(0,1)=(1,1)$ is already not in the union.
\end{example}
\begin{definition}
    Let $V$ be an $F$-vector space.
    Let $U,W\le V$.
    The sum of $U$ and $W$ is the set $U+W=\{u+w:u\in U, w\in W\}$.
\end{definition}
\begin{example}
    Take $V=\mathbb R^2$ and $U,W$ two axes again, then $U+W=V$.
\end{example}
\begin{proposition}
    The sum of two subspaces is a subspace.
\end{proposition}
\begin{proof}
    Obvious.
\end{proof}
Just to mention, one can easily check that $U+W$ is the smallest subspace of $V$ containing $U$ and $W$.
\begin{definition}
    Let $V$ be an $F$-vector space and $U\le V$.
    The quotient space $V/U$ is the quotient group equipped with the scalar multiplication $\lambda(v+U)=\lambda v+U$.
\end{definition}
\begin{proposition}
    This scalar multiplication is well-defined and makes $V/U$ a vector space.
\end{proposition}
\begin{proof}
    Just check.
\end{proof}


%% Magic command to compile root document
% !TEX root = ../../thesis.tex

%% Reset glossary to show long gls names
\glsresetall

%% Set path to look for the images
\graphicspath{{./Sections/Basics/Resources/}}

% what is a ANN

Roughly speaking, an \gls{ann} is a non-linear function $f:\mathbb{R}^D \rightarrow \mathbb{R}^L$, that maps an input $\bs{x}\in\mathbb{R}^D$ with an output $\bs{y}\in\mathbb{R}^L$. Of course, to consider $f$ as a \gls{ann}, $f$ must have a specific form that will be address later. However, for the sake of this explanation, let us start by defining a simple function as follow

\begin{equation}
  \begin{split}
    f(\bs{x},\bs{w}) &:= h \left(w_0 + \sum_{j=1}^{M-1}w_j\phi_j(\bs{x}) \right) \\
    &= h(\bs{w}^T\bs{\phi}(\bs{x})) \\
    &:= h(z)
  \end{split}
  \label{eq:basics:slp}
\end{equation}

\noindent where $\bs{\phi}:\mathbb{R}^{D+1} \rightarrow \mathbb{R}^M$ is an element-wise function, with $\phi_0:=1$, know as \hl{basis function}, $h:\mathbb{R} \rightarrow \mathbb{R}$ is a function know as \hl{activation function} and $\bs{w}\in\mathbb{R}^M$ is the parameter vector. The parameters $w_j$, with $j\in\{1,\dots M-1\}$ are known as \hl{weights}, while the parameter $w_0$ is know as \hl{bias}.

Then, an \gls{ann} is composition of functions of the same form as \ref{eq:basics:slp}, with non-linear \hl{activation functions}, and where the basis functions are also of the same form as \ref{eq:basics:slp} \cite{bishop2006pattern}

\begin{equation}
  F(\bs{x}, \bs{W}) :=
  h_K(\bs{w}^T_K h_{K-1}(\bs{w}^T_{K-1} \dots h_{0}(\bs{w}^T_0 \bs{x}) \dots ))
  \label{eq:basics:ann}
\end{equation}

The subscript in the parameter vectors $\bs{w}_k$ and the activation functions $h_k$, with $k\in\{0, \dots, K\}$, of \ref{eq:basics:ann} represents the depth of the layers. Note that unlike the other layers, the base function of the \hl{input layer} (k = 0) is the identity function. Furthermore, the activation function of the \hl{output layer} $h_K$ does not necessarily have to be non-linear. Instead, it is chosen based on the type of function we want to approximate. In our case, since we have a regression problem (predicting \gls{tr}), $h_K$ is chosen as the identity function.

There are different non-linear activation functions that can be chosen for the hidden units. However, all the models showed in this work use the \gls{relu}

\begin{equation}
  ReLU := max\{0, x\}
\end{equation}

Figure \ref{fig:basics:ann:relu} shows the \gls{relu} activation function.

% Figure made in notebook Preprocessing_resources.ipynb
\begin{figure}[!ht]
  \centering
  \includegraphics[width=0.6\linewidth]{ReLU.jpg}
  \caption{\gls{relu} activation function.}
  \label{fig:basics:ann:relu}
\end{figure}

Figure \ref{fig:basics:ann:ann} shows a graphical representation of a \gls{ann}. The circles represent the activation function applied to what is inside it. Black colored circles represent the identity function, red colored circles the non-linear activation function for the hidden layers, while green any function for the output layer that suits the problem we want to solve. Note that values inside the circles of the hidden and output layers $z^k_i$, for $k\in\{0, \dots, K\}$ and $i$ representing one of the units of the $k$ layer, are the output of a function of the same form as \ref{eq:basics:slp}. The lines connecting the circles represent the weights and biases corresponding to each layer $\bs{W}_k$, for $k\in\{0, \dots, K\}$. The circles in the \hl{hidden layers} are known as \hl{hidden units}.

\begin{figure}[!ht]
  \centering
  \includegraphics[width=0.8\linewidth]{Diagrams/ANN.jpg}
  \caption{Graphical representation of an \gls{ann}. The color of the circles represents the type of activation function. Black means the identity, red a non-linear function for the hidden layers and green any function for the output layer.}
  \label{fig:basics:ann:ann}
\end{figure}

Strictly speaking, equation \ref{eq:basics:ann} and figure \ref{fig:basics:ann:ann} represent a \hl{fully connected feedforward neural network}. However, in this work we will refer to it just as \gls{ann}, which in some literature is also known as \gls{mlp}. Also, hidden layers are also known as \hl{Dense layers}.

\subsubsection{Update rule}

Sow far we have introduced the general form an \gls{ann} must have. Moreover, equation \ref{eq:basics:ann} shows that an \gls{ann} is simply a non-linear function controlled by a set of adjustable parameters $\bs{W}$. Therefor the question is, how can we approximate this parameters?

Recall that we are dealing with a supervised learning problem, which means that we can use both the input data (images of cell nucleus, $\bs{X}$) and the output data (the \glspl{tr}, $\bs{Y}$) to approximate $\bs{W}$. Therefore, we can fed the \gls{ann} with $\bs{X}$, and then measure its performance by comparing its output $\hat{\bs{Y}}$ against the true values $\bs{Y}$.

This comparison is made by means of a \hl{loss function} $\mathcal{L}$ that must be chosen beforehand. The choice of $\mathcal{L}$ depends mainly on the type of problem you are solving (regression, classification, etc.). However, even for each type, there are many different options. For now, let us just say that $\mathcal{L}$ should return high values when $\hat{\bs{Y}}$ is far from the true values $\bs{Y}$, and low when they are close.

Then, we can fit the values of $\bs{W}$, by minimizing the loss function $\mathcal{L}$ each time the model is fed with an input value $\bs{x}$. Since the gradient of $\mathcal{L}$ with respect to $\bs{W}$ (i.e., $ \nabla_{\bs{W}} \mathcal{L}$) returns the direction in which the loss function grows the fastest, then we choose $- \nabla_{\bs{W}} \mathcal{L}$ as the direction of our update rule

\begin{equation}
  \bs{W}_{new} = \bs{W}_{old} - \alpha \nabla_{\bs{W}} \mathcal{L}(\bs{W}_{old})
  \label{eq:basics:ann:learn_rule}
\end{equation}

\noindent where $\alpha \in \mathbb{R}^+$ (known as \hl{learning rate}) controls how much we move in the direction of $-\nabla_{\bs{W}} \mathcal{L}(\bs{W}_{old})$ on every step.

The iterative method in which \ref{eq:basics:ann:learn_rule} is applied over elements of $\bs{X}$ to optimize $\bs{W}$ is known as \gls{gd} \cite{bishop2006pattern}.
However, in practice \ref{eq:basics:ann:learn_rule} is not applied for a single element of $\bs{X}$ every time, but to a random subset of $\bs{X}$ (known as a \hl{Batch}) instead.
The number of elements in batch is fixed over all the iteration (training), and is an hyperparameter known as \hl{Batch Size} $bs$\footnote{Normally the training data is separated in disjoint batches, which means that it could happen that last batch to be smaller than the selected $bs$.}.
As a rule of thumb, $bs$ should be no less than 30 (for the selected sample to be representative of $\bs{X}$). In practice $bs$ is usually chosen as a power of 2.
This process is known as \gls{sgd} and computationally is less  expensive than \gls{gd}.

However, \gls{gd} (\gls{sgd}) has a downside, the choice of its hyperparameter $\alpha$ (learning rate). In practice, it has been shown that the correct choice of $\alpha$ is essential to train an \gls{ann} successfully. Therefore, other algorithms (\hl{optimizers}) have been proposed to mitigate this problem. The revision of these optimizers is out of the scope to this work. However, all of them follow the same idea proposed by \gls{gd}. For example, instead of having a fixed learning rate $\alpha$ as in \gls{gd}, the \gls{adam} optimizer adapts its learning rate dynamically during training depending on the mean and variance of the loss function \cite{kingma2014adam}.

\subsubsection{Back propagation}

Nevertheless, there is still one question that needs to be answered, which is how to efficiently calculate the derivative of the loss function ($\nabla_{\bs{W}} \mathcal{L}$) with respect to all the parameters of the \gls{ann}. The answer to this is through an algorithm called \hl{backpropagation}, which is performed during the \hl{training process}. Again, there is a lot of literature that explains this in depth(for instance \cite{Goodfellow-et-al-2016} or \cite{bishop2006pattern}). Therefor, here we will just provide the intuition behind it.

Recall that $\mathcal{L}$ is a function of the true values $y$ and $\hat{y}$ i.e., $\mathcal{L}(y, \hat{y})$. Also from equation \ref{eq:basics:ann} and figure \ref{fig:basics:ann:ann} note that

\begin{equation}
  \begin{split}
    y &:= F(\bs{x}, \bs{W}) \\
    &= h_K(\bs{z}^K) \\
    &= h_K(\bs{W}_K^T h_{K-1}(\bs{z}^{K-1}))
  \end{split}
  \label{eq:basics:ann:backprop_1}
\end{equation}

and therefore

\begin{equation}
  \begin{split}
    \nabla_{\bs{W}_K} \mathcal{L} &= \frac{\partial \mathcal{L}}{\partial \bs{W}_K} \\
    &= \frac{\partial \mathcal{L}}{\partial \hat{y}}
    \frac{\partial \hat{y}}{\partial \bs{z^K}}
    \frac{\partial \bs{z^K}}{\partial \bs{W}_K} \\
  \end{split}
  \label{eq:basics:ann:backprop_2}
\end{equation}

\noindent which is just the product of the derivative of the loss function w.r.t. $\hat{y}$ (i.e., $\frac{\partial \mathcal{L}}{\partial \hat{y}}$), the derivative of the activation function of the output layer w.r.t the argument of the last layer (i.e., $\frac{\partial \hat{y}}{\partial \bs{z^K}}$) and the output of the layer $K-1$ (i.e., $\frac{\partial \bs{z^K}}{\partial \bs{W}_K}=h_{K-1}(\bs{z^{K-1}})$).

Note that we can easily compute the gradient of $\mathcal{L}$ w.r.t deeper parameters $\bs{W}_k$ (for $k\in\{0, \dots, K-1\}$), just by extending \ref{eq:basics:ann:backprop_1} and \ref{eq:basics:ann:backprop_2}.

This shows how by means of the \hl{chain rule}\footnote{$(f \circ g)'=(f \circ g) \cdot g'$, or equivalently $h'(x)=f'(g(x))g'(x)$, for $h(x):=f(g(x))$.}, the backpropagation algorithm can compute the gradient of the loss function w.r.t. a specific parameter, just by multiplying the derivative of the loss function, the derivative of the activation functions and some values computed during the evaluation of the \gls{ann}.

\subsubsection{Model development}

The properties of \glspl{ann} have been studied extensively before (\cite{cybenko1989approximation}, \cite{hornik1989multilayer}, \cite{funahashi1989approximate}) and established in the \hl{Universal approximation theorem}

\begin{theorem}[Universal approximation theorem]
  An \gls{mlp} with a linear output layer and one hidden layer can approximate any continuous function defined over a closed and bounded subset of $\mathbb{R}^D$, under mild assumptions on the activation function (\hl{squashing} activation function) and given the number of hidden units is large enough.
\end{theorem}

For this reason \gls{ann} are known as \hl{universal approximators}, since they are able to approximate any continuous function on a compact\footnote{A set $A$ in a metric space is said to be \hl{compact} if it is close (i.e., it contain all its limit points) and bounded (i.e., all its points lie within some fixed distance of each other) \cite{bartle2000introduction}.} input domain with an arbitrary accuracy \cite{bishop2006pattern}.

These means that, as long as a \gls{ann} has a sufficiently large number of hidden units, the loss function can be reduced as much as desired. However, this nice property can also lead to an unwanted one known as \hl{overfitting}.
Intuitively this means that the \gls{ann} \hl{memorize} the data used to train it (low error/bias), and therefore it is not able to perform (or \hl{generalize}) well when it is fed with new data (high error/bias and variance). This happens mainly when the \gls{ann} is optimized/fed too many times with the same data.

On the other hand, \hl{underfitting} means that the \gls{ann} performs poorly on both new data and data used to train the network (high bias and low variance). This usually happens when the training time is insufficient or the \gls{ann} is not complex enough (too few hidden units and/or layers).

Figure \ref{fig:basics:ann:fitting} shows synthetic data (blue circles), generated from a sine function (green line) and random noise sampled from a normal distribution.
The red line in figure \ref{fig:basics:ann:fitting:under} represents a fitted model with high bias and low variance (underfitting), while in figure \ref{fig:basics:ann:fitting:over} a model with low bias and high variance (overfitting). The red line in figure \ref{fig:basics:ann:fitting:good}, represents a model with low bias and variance (good fit and good generalization).

% this plots were extracted from page 7 (Bishop) and adapted in the file overfitting.odg
\begin{figure}[htb]
  \centering
  \begin{subfigure}[t]{.3\linewidth}
    \includegraphics[width=\linewidth]{Diagrams/underfitting.jpg}
    \caption{Underfitted model.}
    \label{fig:basics:ann:fitting:under}
  \end{subfigure}
  \vspace{3mm}
  \begin{subfigure}[t]{.3\linewidth}
    \includegraphics[width=\linewidth]{Diagrams/goodfit.jpg}
    \caption{Model with good fit and generalization.}
    \label{fig:basics:ann:fitting:good}
  \end{subfigure}
  \vspace{3mm}
  \begin{subfigure}[t]{.3\linewidth}
    \includegraphics[width=\linewidth]{Diagrams/overfitting.jpg}
    \caption{Overfitted model.}
    \label{fig:basics:ann:fitting:over}
  \end{subfigure}
  \caption{Representation of a model (red line) with underfitting \subref{fig:basics:ann:fitting:under}), good fit \subref{fig:basics:ann:fitting:good}) and overffiting \subref{fig:basics:ann:fitting:over}), trained over synthetic data (blue small circles). The synthetic data was generating by adding random noise to a sine function (green line) on the interval $[0,1]$. Image source \cite{bishop2006pattern}.}
  \label{fig:basics:ann:fitting}
\end{figure}

In practice, we seek to fit models that has low bias and low variance (i.e., good accuracy and good generalization). Therefore, to prevent overfitting we split the data into 3 different sets; \hl{training}, \hl{validation} and \hl{test}, and train the model using only the first set. Then, during model training, we measure how well the model is generalizing by comparing the value of the loss function when it is evaluated in the training and validation set \footnote{This is usually known as the \hl{bias–variance tradeoff}.}.
During the model development, the \hl{test} set is never evaluated and is only used at the end, to report the model performance. This methodology is shown in figure \ref{fig:basics:model_train_process}

Figure \ref{fig:basics:bias_variance} shows this \hl{bias–variance tradeoff} between training and validation set. In practice, multiple versions of the model are saved during training and then the one with the lowest validation error is chosen (red dot on figure \ref{fig:basics:bias_variance}).

% figure taken from BL_070121_0942.ipynb
\begin{figure}[!ht]
  \centering
  \includegraphics[width=0.8\linewidth]{bias_variance.png}
  \caption{Bias–variance tradeoff. In orange (respectively blue) the loss function curve when it is evaluated in the validation (respectively training) set. The red dot shows the lowest loss for the validation set.}
  \label{fig:basics:bias_variance}
\end{figure}

\begin{figure}[!ht]
  \centering
  \includegraphics[width=0.8\linewidth]{Diagrams/Model_methodologt.jpg}
  \caption{Model development methodology.}
  \label{fig:basics:model_train_process}
\end{figure}

The methodology shown in figure \ref{fig:basics:model_train_process} is also used to optimize the hyperparameters of the model, like the number hidden units/layers or the activation function of the hidden layers.

\subsubsection{Batch Normalization}

However, overfitting is not the only problem we may encounter when training an \gls{ann}. Training \gls{ann} with several layers can be complicates, since the distribution of the data can change from layer to layer. This means that the input and output distribution of a layer will not necessarily be the same. It has been empirically proven that this can affect the training performance, since it require the use of lower learning rates \cite{ioffe2015batch}. This can also lead to \hl{saturation}\footnote{\hl{Saturation} is a commonly used term to refer to the situation when the evaluation of a "squashing" function returns values close to some of its horizontal asymptotes most of the time. Remember that these "squash" functions (like \hl{Sigmoid} or \hl{tanh}) compress the real line $(-\inf, \inf)$ into an interval of finite length $(a, b)$.} of the activation functions, so a more careful initialization of the \gls{ann} parameters is required. To address this problem Ioffe et al. \cite{ioffe2015batch} proposed to normalize the layer inputs.

Roughly speaking, batch normalization consist of two main steps; 1) the standardization of the layer input and 2) the normalization of the standardized data. For the first step the layer input is standardized using parameters extracted from the \hl{batch}

\begin{equation}
    \bs{z}'_k := \frac{\bs{z}_k-\bs{\mu}_k}{\sqrt{\bs{\sigma}_k^2-\epsilon}}
\end{equation}
\noindent with
\begin{equation}
  \begin{split}
    \bs{\mu}_k &= \frac{1}{M}\sum_{m=1}^M \bs{z}_k \\
    \bs{\sigma}_k^2 &= \frac{1}{M}\sum_{m=1}^M (\bs{z}_k - \bs{\mu}_k)^2 \\
  \end{split}
\end{equation}

\noindent where $M$ is the \hl{Batch} size and $k$, with $k \in \{0 \dots K\}$, denotes the layer.

Note that for each layer $k$ we have different normalization parameters $\bs{\mu}_k$ and $\bs{\sigma}_k$. Moreover, this normalization parameters are vectors of the same shape as the layer size (i.e., one pair of normalization parameters per unit/neuron).

The second step in batch normalization consist on normalizing the standardized data $\bs{z}'_k$ using parameters $\bs{\gamma}_k$ and $\bs{\beta}_k$ learned during training

\begin{equation}
    \overset{\sim}{\bs{z}}_k := \bs{\gamma}_k \odot \bs{z}'_k + \bs{\beta}_k
\end{equation}

\noindent where $\odot$ denotes \hl{element-wise} multiplication. At the beginning of the training $\bs{\gamma}_k=1$ and $\bs{\beta}_k=0$ are used for all the layers and units.

During training, the normalization parameters of each epoch are stored, so the average ($\bar{\bs{\gamma}}_k$ and $\bar{\bs{\beta}}_k$) can be used during evaluation (when the model is not training).

\subsubsection{Residual Block V2}

As already mentioned, the \hl{Universal approximation theorem} guarantees that the training error can be reduced by adding more layer to an \gls{ann}. However, in practice it is not that simple. As we add layers to an \gls{ann}, the training becomes more unstable and difficult as we can face vanishing or exploding gradients (when the value of the gradients become very close to 0 or $\inf$ respectively during back propagation). To overcome this problem, He et al. (\cite{he2015deep} and \cite{he2016identity}) proposed the \hl{residual blocks}, which have been empirically shown to make deep \gls{ann} training more stable.
The core idea of residual blocks is to reformulate the layers as \hl{learning residual functions} with reference to the layer inputs, by adding an \hl{identity connection}. Then, if a layer is not longer beneficial to the \gls{ann} (e.g. in case of gradient vanishing), the \gls{ann} can just "skip" it. Figure \ref{fig:basics:residual_block} shows a diagram of the second version of a residual block \cite{he2016identity}.

\begin{figure}[!ht]
  \centering
  \includegraphics[width=\linewidth]{Diagrams/Residual_block_v2.jpg}
  \caption{Residual block V2.}
  \label{fig:basics:residual_block}
\end{figure}


\section{K-Means Clustering}
CLustering seeks to group similiar points of data together in a cluster. We
denote $c^{(i)}$ as the cluster for data point $i$ and $\mu_j$ as the center
for cluster $j$. We denote $k$ as the number of clusters and $n$ as the
dimension of our data.
\subsection{Algorithm}
After randomly initializing the cluster centroids
$\mu_1, \mu_2, \hdots, \mu_k \in \mathbb{R}^n$, repeat until convergence:
\begin{enumerate}
  \item For every data point $i$:
    \begin{equation}
      c^{(i)}=\underset{j}{\textrm{arg min}}||x^{(i)}-\mu_j||^2
    \end{equation}
  \item For each cluster $j$:
    \begin{equation}
      \mu_j=\frac{\displaystyle\sum_{i=1}^m1_{\{c^{(i)}=j\}}x^{(i)}}{\displaystyle\sum_{i=1}^m1_{\{c^{(i)}=j\}}}
    \end{equation}
\end{enumerate}
The first step is known as cluster assignment, and the second updates the
cluster center (i.e. the average of all points in the cluster). In order to
see if it converges, use the distortion function:
\begin{equation}
  J(c,\mu)=\sum_{i=1}^m||x^{(i)}-\mu_{c^{(i)}}||^2
\end{equation}
The distortion function $J$ is non-convex, and coordinate descent of $J$ is not
guaranteed to converge to the global minimum (i.e. susceptible to local optima).
\subsection{Hierarchical Clustering}
Hierarchical clustering is a clustering algorithm with an agglomerative
hierarchical approach that builds nested clusters in a successive manner. The
types are:
\begin{enumerate}
  \item Ward Linkage: minimize within cluster distance
  \item Average Linkage: minimize average distance between cluster pairs
  \item Complete Linkage: minimize maximum distance between cluster pairs
\end{enumerate}
\subsection{Clustering Metrics}
In an unsupervised learning setting, it is often hard to assess the
performance of a model since we don't have the ground truth labels as was
the case in the supervised learning setting.
\paragraph{Silhouette coefficient} By noting $a$ and $b$ the
mean distance between a sample and all other points in the same class,
and between a sample and all other points in the next nearest cluster,
the silhouette coefficient $s$ for a single sample is defined as follows:
\begin{equation}
  s = \frac{b - a}{\max(a, b)}
\end{equation}
\paragraph{Calinskli-Harabaz Index} By noting $k$ the number of clusters, $B_k$
and $W_k$ the between and within-clustering dispersion matricies defined as:
\begin{equation}
  B_k=\sum_{j=1}^kn_{c^{(i)}}(\mu_{c^{(i)}}-\mu)(\mu_{c^{(i)}}-\mu)^T
\end{equation}
\begin{equation}
  W_k=\sum_{i=1}^m(x^{(i)}-\mu_{c^{(i)}})(x^{(i)}-\mu_{c^{(i)}})^T
\end{equation}
the Calinksli-Harabaz index $s(k)$ indicated how well a clustering model
defines its clusters, such that higher scores indicate more dense and well
separated cluster assignments. It is defined as:
\begin{equation}
  s(k)=\frac{\textrm{Tr}(B_k)}{\textrm{Tr}(W_k)}\times\frac{N-k}{k-1}
\end{equation}


The Weibull distribution may be useful for fitting fat-tailed
empirical distributions.

In the literature, the Weibull is sometimes called a ``stretched
exponential'' distribution when its shape parameter $\tau$ is less
than 1. ``Stretched exponential'' distributions in the literature are
either Weibull (PDF $ = \lambda \tau (\lambda x)^\tau exp\left[-
(\lambda x)^tau \right]$ or a more simple PDF $\propto exp\left[-
{\lambda(x-\mu)}^tau \right]$. Easel treats the latter form in the
\eslmod{stretchexp} module.

\subsection{Weibull densities}

The probability density function (PDF) is:

\begin{equation}
P(X=x) = \lambda \tau [\lambda(x - \mu)]^{\tau-1} e^{- [\lambda(x-\mu)]^{\tau}}
\label{eqn:weibull_pdf}
\end{equation}

The cumulative distribution function (CDF) is:

\begin{equation}
P(X \leq x) = 1 - e^{- [\lambda(x-\mu)]^{\tau}}
\label{eqn:weibull_cdf}
\end{equation}

Variate $x$ ranges $\mu \leq x < \infty$. (However, for $\tau < 1$,
the PDF goes to infinity at $x=\mu$, so evaluating at $x=\mu$ may not
be desired.)

Location parameter $\mu$ is unconstrained, $-\infty < \mu <
\infty$. (Weibull distributions are usually represented without an
explicit location parameter, implicitly assuming $\mu = 0$.)

Scale parameter $\lambda$ is nonnegative, $\lambda >
0$. (Alteratively, Weibull distributions are also sometimes
represented with a scale parameter $b = \frac{1}{\lambda}$.)

Shape parameter $\tau$ is nonnegative, $\tau > 0$.


\section{Derivatives of VVFs}
\noindent
Just like functions from Calc I and II, we can differentiate VVFs.
In fact, the limit definitions of the derivative are nearly identical.
Let $\vec{r}(t) = \langle x(t), y(t), z(t) \rangle$.
\begin{align*}
	\vec{r^\prime}(t) &= \lim_{h\to 0}{\frac{\vec{r}(t+h)-\vec{r}(t)}{h}} \\
	&= \lim_{h\to 0}{\bigg\langle \frac{x(t+h)-x(t)}{h}, \frac{y(t+h)-y(t)}{h}, \frac{z(t+h)-z(t)}{h} \bigg\rangle}.
\end{align*}
The limit distributes inside the vector, so
\begin{equation*}
	\vec{r^\prime}(t) = \langle x^{\prime}(t), y^{\prime}(t), z^{\prime}(t) \rangle.
\end{equation*}

\noindent
Like a position function from Calc I and II, the derivative of a VVF representing position gives a VVF representing velocity, and the 2nd derivative gives a VVF representing acceleration.
The magnitude of the velocity VVF, the speed, is commonly notated $v(t)$.\\

\noindent
There are 5 important properties of the derivatives of VVFs.
These properties are similar to single-variable derivatives.
Let $\vec{r}(t)$ and $\vec{s}(t)$ be VVFs, $a(t)$ be a scalar function, and $c$ be a scalar.
\begin{enumerate}[label=]
	\item \textbf{Linearity}
	\begin{equation*}
		\frac{\mathrm{d}}{\mathrm{d}t}c\vec{r}(t) = c\vec{r^\prime}(t)
	\end{equation*}
	\item \textbf{Product Rule for Scalar Functions}
	\begin{equation*}
		\frac{\mathrm{d}}{\mathrm{d}t}a(t)\vec{r}(t) = a(t)\vec{r^\prime}(t) + \vec{r}(t)a^{\prime}(t)
	\end{equation*}
	\item \textbf{Dot Product Rul}e
	\begin{equation*}
		\frac{\mathrm{d}}{\mathrm{d}t}\vec{s}(t)\cdot\vec{r}(t) = \vec{s}(t)\cdot\vec{r^\prime}(t) + \vec{r}(t)\vec{s^\prime}(t)
	\end{equation*}
	\item \textbf{Cross Product Rule}
	\begin{equation*}
		\frac{\mathrm{d}}{\mathrm{d}t}\vec{s}(t)\times\vec{r}(t) = \vec{s}(t)\times\vec{r^\prime}(t) + \vec{s^\prime}(t)\times\vec{r}(t)
	\end{equation*}
	\item \textbf{Chain Rule}
	\begin{equation*}
		\frac{\mathrm{d}}{\mathrm{d}t}\vec{r}(a(t)) = \vec{r^\prime}(a(t))a^{\prime}(t)
	\end{equation*}
\end{enumerate}
A quotient rule doesn't make sense because we don't have an operation for dividing two vectors by each other.\\

\noindent
Just like in single variable calculus, we can use the derivative of VVFs to find tangent lines to the curve.
Similar to how $f^{\prime}(a)$ represents the slope of $f$ at $a$, $\vec{r^\prime}(a)$ represents the direction of the tangent line at $a$.
Remembering the VVF form of a line, the tangent line to $\vec{r}$ at $t$ is
\begin{equation*}
	\vec{l}(t)=\vec{r}(t)+t\vec{r^\prime}(t).
\end{equation*}
In fact, tangent lines appear so often, that we have a special unit vector representing the direction of the tangent line.
\begin{equation*}
	\hat{T}(t) = \frac{\vec{r^\prime}(t)}{\norm{\vec{r^\prime}(t)}}.
\end{equation*}
You can remember $\hat{T}$ as the ``tangent'' vector.

\begin{figure}[H]
	\centering
	\includegraphics[scale=0.33]{Images/vectorValuedFunctions/TangentVector}
	\caption{$\hat{T}$ is $\vec{r^\prime}$ normalized.}
\end{figure}


\lecture{Sept. 27}

\begin{defn}
\textbf{Interpretation} is a choice of a non-empty set u, and constant, functions and relations for each constant, function and relation symbol.

An \textbf{Assignment} in u, $$\alpha \colon \{\text{variale symbols}\}\to u$$

We write $\alpha (F) = 1$ when F is true in u under $\alpha$.

We write $\alpha (F) = 0$ when F is false in u under $\alpha$.

We say F is true in u when $\alpha (F) = 1$ for \textbf{every} assignment $\alpha \in u$
\end{defn}

\begin{exmp}
the formula $x\times y = y \times x$ is true in $\mathbb{Z}$ but not true in $n\times n$ matrices.
\end{exmp}

\begin{defn}
For formulas F and G and a set of formulas S, we say that F is a \textbf{tautology} and we write $\vDash F$, when for every interpretation u and every assignment $\alpha \in u$, $\alpha (F) = 1$
\end{defn}

\begin{defn}
We say that F and G are \textbf{equivalent}, and we write $F\equiv G$, when for every interpretation u, for every assignment $\alpha \in u$, $\alpha (F) = \alpha (G)$, we say that the argument ``F there fore G" is valid, or that ``S induces G", or that ``G is a consequences of S", when for every interpretation u and for every assignment $\alpha \in u$, if $\alpha (F) = 1$ for every $F\in S$ then $\alpha (G) = 1$
\end{defn}

\begin{defn}
Given a formula G and a set of formulas S, such that $S\vDash G$, a \textbf{derivation} for the valid argument $S\vDash G$ is a list of valid arguments
$$S_1\vDash G_1,S_2\vDash G_2,S_3\vDash G_3,\dots$$
where for some index k we have $S_k=S$ and $G_k = G$, such that each valid argument in the list is obtained from previous valid arguments in the list by applying one of the basic validity rules.
\end{defn}

\section{Basic Validity Rules}
Each basic validity rule is a formal and precise way of describing standard method of mathematical proof.

Rules V1, V2 and V3 are used in derivations because we make a careful distinction between \textbf{premises} and \textbf{conclusions}. In standard mathematical proofs we do not make a careful distinction.

\begin{enumerate}
\item[Premise V1.] If $F\in S$ then $S\vDash F$. In words, if we assume $F$, we can conclude $F$.
\item[V2.] If $S\vDash F$ and $S\subseteq \mathcal{T}$ then $\mathcal{T}\vDash F$. In words, if we can prove $F$ without assuming $G$, then we can still prove $F$ if we assume $G$.
\item[Chain Rule V3.] If $S\vDash F$ and $S\cup \{F\}\vDash G$ then $S\vDash G$. In words, if we can prove $F$, and by assuming $F$ we can prove $G$, then we can prove $G$ directly without assuming $F$.
\item[Proof by Cases V4.] If $S\cup \{F\}\vDash G$ and $S\cup \{\neg F\}\vDash G$ then $S\vDash G$. In words, in either case $G$ is true.
\item[Contradiction V5.] If $S\cup \{\neg F\} \vDash G$ and $S\cup \{\neg F\} \vDash \neg G$ then $S\vDash F$. In words, to prove F by contradiction, we suppose, for a contradiction, that F is false, we choose a formula G, then we prove that G is true and we prove that G is false.
\item[V6.] If $S\cup \{ F\} \vDash G$ and $S\cup \{ F\} \vDash \neg G$ then $S\vDash \neg F$
\item[V7.] If $S\vDash F$ and $S\vDash \neg F$ then $S\vDash G$
\item[Conjunction V8.] $S\vDash F\wedge G \Longleftrightarrow (S\vDash F \text{ and } S\vDash G)$
\item[V9.] If $S\cup \{F,G\} \vDash H$ then $S\cup \{F\wedge G\}\vDash H$
\item[V10.] ...
\item[V11.] ...
\item[V12.] ...
\item[Disjunction V13.] $S\vDash F\vee G \Longleftrightarrow S\cup \{\neg F\}\vDash \Longleftrightarrow S\cup \{\neg G\}\vDash F$
\item[V14.] ...
\end{enumerate}


	\chapter{Vector Spaces}

	A vector space is the general mathemtical structure we need to be able to talk about linear mappings. To introduce a vector space we need a lot of mathematical concepts. So we start with a group.

	\begin{definition}
		A set $G$ of elements with an operation $+$: $G \times G \rightarrow G$ is called a group if the following properties hold:
	\end{definition}

	\begin{itemize}
		\item[G1] Associativity: $ \forall a, b, c \in G$ : $(a + b) + c = a + (b + c) $
		\item[G2] Identity element: $\exists e \in G: \forall g \in G: e + g = g + e = g $
		\item[G3] Inverse element: $ \forall a \in G$, $ \exists b \in G : a + b = b + a = e$
	\end{itemize}

	A group is called a commutative group (Abelian group) if we have additionally that $ \forall a, b \in G: a + b = b + a$ \\

	\textbf{Examples}

	\begin{itemize}
		\item $(\Re^n, +)$: This can be thought of as an $n$-dimensional vector with addition as the associated operation. The addition of three vectors can be done in any order and is thus associative. The identity element in this case is the zero vector. The inverse element is the negative of each element. Thus this combination forms a group.

		\item $(\Re^+, .)$: This is the set of positive real numbers with multiplication as the associated operation. This also forms a group.

		\item $(\Re^-, .)$: This is set of negative real numbers with associated operation as multiplication. It does not form a group, since multiplication of two negative real numbers gives us a positive real number and that is out of the set considered. We can also say that the set of negative real numbers is not closed with respect to multiplication.

	\end{itemize}

	\begin{definition}
		A set $F$ with two operation $ (+ , \cdot) : F \times F \rightarrow F$ is called a field if the following properties hold:
	\end{definition}

	\begin{itemize}
		\item[F1] $ (F, +) $ is a commutative group with identity element 0.
		\item[F2] $(F \setminus\{0\}, \cdot)$ is a commutative group with identity element 1.
		\item[F3] Distributivity:  $\forall a, b, c \in F: a \cdot (b + c) = a \cdot b + a \cdot c$
	\end{itemize}

	The two most common fields are the ral numbers $(\Re, +, \cdot)$ and complex numbers $(\mathbb{C}, +, \cdot)$ with defined addition and multiplication.

	\begin{definition}
		Let $F$ be a field with identity elements 0 and 1. A vector space over the field $F$ is a set $V$ with a mapping: $ (+) : V \times V \rightarrow V$ (vector addition)  and a mapping $(\cdot) : F \times V \rightarrow V$ (scalar multiplication) such that:
	\end{definition}

	\begin{itemize}
		\item[V1] $(V, +)$ is a commutative group.
		\item[V2] Multiplicative identity: $\forall v \in V: 1 \cdot v = v$
		\item[V3] Distributive property: $\forall a, b  \in F$ and $ \forall u, v \in V$
			\subitem $a \cdot (u + v) = a \cdot u + a \cdot v$
			\subitem $(a + b)u = a \cdot u + b \cdot u $
	\end{itemize}

	Elements of $V$ are called vectors and elements of $F$ are called scalars. Depending on whether the field is real or complex we call the space as real vector space or complex vector space. \\

	\textbf{Examples}

	\begin{itemize}

		\item $\Re^n$ with standard operations of adding vectors and multiplying vectors with a scalar.
		\item Function spaces: These are spaces which consists of functions. And essentially the whole field of functional analysis exploits the fact if you group them in a vector space, there are many properties that you can find about functions without even explicitly looking at what type of functions you are talking about.
		\subitem $\Re^\chi$ : $\{f: \chi \rightarrow \Re\}$ the space of all real valued functions on a set $\chi$. No we define the two operations:
		\subitem $ + : \Re^\chi \times \Re^\chi \rightarrow \Re^\chi$ , $(f + g)(x) := f(x) + g(x)$
		\subitem $\cdot : \Re \times \Re^\chi \rightarrow \Re^\chi, (\lambda \cdot f)(x) := \lambda \cdot (f(x))$ \\
		Then $(\Re^\chi, +, \cdot)$ is a vector space.
	\end{itemize}


 By \textbf{Taylors Formula}, we have for $t,t_0 \in I$:

 \begin{equation}
 \label{eq:taylor:n}
  \Vector{r}(t)
  =
  \Vector{r}(t_0)
  +
  \Vector{r}'(t_0)(t-t_0)
  +
  \ldots+
  \frac{1}{n!} \Vector{r}^{(n)}(t_0)(t-t_0)^n
  +
  \Vector{\epsilon}_n(t)
 \end{equation}

 In fact, Taylors Formula ensures, that for $t>t_0$, $\exists \xi \in [t_0,t]$, so that:

 \[
   |\Vector{\epsilon}_n(t)|
   \leq
   \frac{1}{(n+1)!}
   |\Vector{r}^{(n+1)}(\xi)|(t-t_0)^{n+1}
 \]

Hence, for the \emph{Error Vector}, $\Vector{\epsilon}(t)$, we have:

\begin{equation}
  \lim_{t\rightarrow t_0} \frac{\Vector{\epsilon}_n(t)}{ (t-t_0)^n }
 \label{eq:taylor:err:n}
  =
  \Vector{0}
\end{equation}

\subsection{First Order Aproximation}
input{Introduction/First}
Deriving:

\[
 v(t)=\sqrt{ x'(t)^2+y'(t)^2},
\]

we observe:

\[
  v'(t)
  =
  \frac{x'(t)+y'(t)}{ \sqrt{ x'(t)^2+y'(t)^2} }
  =
  \frac{x'(t)+y'(t)}{ v }
\]

\subsection{Second Order Aproximation}
\input{Introduction/Second}


\subsection{Transposition and conjugation}

\subsubsection{Transposition}

A matrix of dimensions \(m*n\) can be transformed into a matrix \(n*m\) by transposition.

\(B=A^T\)

\(b_{ij}=a{ji}\)

\subsubsection{Transpose rules}

\((M^T)^T=M\)

\((AB)^T=B^TA^T\)

\((A+B)^T=A^T+B^T\)

\((zM)^T=zM^T\)

\subsubsection{Conjugation}

With conjugation we take the complex conjugate of each element.

\(B=\overline A\)

\(b_{ij}=\overline a_{ij}\)

\subsubsection{Conjugation rules}

\(\overline {(\overline A)}=A\)

\(\overline {(AB)}=(\overline A)( \overline B)\)

\(\overline {(A+B)}=\overline A+\overline B\)

\(\overline {(zM)}=\overline z \overline M\)

\subsubsection{Conjugate transposition}

Like transposition, but with conjucate.

\(B=A^*\)

\(b_{ij}=\bar{a_{ji}}\)

Alternatively, and particularly in physics, the following symbol is often used instead.

\((A^*)^T=A^\dagger\)


\subsection{Infinitum and supremum}

\subsubsection{Infinitum}

Consider a subset \(S\) of a partially ordered set \(T\).

The infinitum of \(S\) is the greatest element in \(T\) that is less than or equal to all elements in \(S\).

For example:

\(\inf [0,1]=0\)

\(\inf (0,1)=0\)

\subsubsection{Supremum}

The supremum is the opposite: the smallest element in \(T\) which is greater than or equal to all elements in \(S\).

\(\sup [0,1]=1\)

\(\sup (0,1)=1\)

\subsubsection{Max and min}

If the infinitum of a set \(S\) is in \(S\), then the infinimum is the minimum of set \(S\). Otherwise, the minimum is not defined.

\(\min [0,1]=0\)

\(\min (0,1)\) isn't defined.

Similarly:

\(\max [0,1]=1\)

\(\max (0,1)\) isn't defined.


\subsection{Complex Numbers}
\begin{definition}
	$i$ is called the imaginary unit. It's defined by $i^2 = -1$.
\end{definition}

\noindent
The set of complex numbers ($\mathbb{C}$) is an extension of the real numbers. Complex numbers have the form $z = \alpha + \beta i$, where $\alpha$ and $\beta$ are real numbers. The $\alpha$ part of $z$ is called the real part, so $\Re(z) = \alpha$. The $\beta$ part of $z$ is called the imaginary part, so $\Im(z) = \beta i$.\\

\noindent
Often, complex numbers are visualized as points or vectors in a 2D plane, called the complex plane, where $\alpha$ is the x-component, and $\beta$ is the y-component. Thinking of complex numbers like points helps us define the magnitude of complex numbers and compare them. Since a point $(x,y)$ has a distance $\sqrt{x^2+y^2}$ from the origin, we can say the magnitude of $z$, $\lvert z \rvert$ is $\sqrt{\alpha^2 + \beta^2}$. Thinking of complex numbers like vectors helps us understand adding two complex numbers, since you just add the components like vectors.\\

\noindent
A common operation on complex numbers is the complex conjugate. The complex conjugate of $z = \alpha + \beta i$ is $\overline{z} = \alpha - \beta i$. $z$ and $\overline{z}$ are called a conjugate pair.\\

\noindent
Conjugate pairs have the following properties.
Let $z, w \in \mathbb{C}$.
\begin{align*}
	\overline{z \pm w} &= \overline{z} \pm \overline{w} \\
	\overline{zw} &= \overline{z}\overline{w} \\
	\overline{z} &= z \Leftrightarrow z \in \mathbb{R} \\
	z\overline{z} &= \lvert z \rvert^2 = \lvert \overline{z} \rvert^2 \\
	\overline{\overline{z}} &= z \\
	\overline{z}^n &= \overline{z^n} \\
	z^{-1} &= \frac{\overline{z}}{\lvert z \rvert^2}
\end{align*}