\chapter{Introduction, Wiener-Hopf Equations, and Normal Equations}
\label{ch:intro_WHE_NE}
\section{Review of the Basics}
\subsection{Linear Algebra}
\subsubsection{Notation}
An $N$-dimensional vector is written as
\bmath
  \vect{x} = \bbmtx x_1 \\ x_2 \\ \vdots \\ x_N \ebmtx = \bbmtx x_1 & x_2 & \cdots & x_N \ebmtx^T
\emath
where $(\cdot)^T$ denotes the transpose. An $N\times M$-dimensional matrix is written as
\bmath
  \vect{A} =
  \bbmtx
    a_{11} & \cdots & a_{1M}\\
    \vdots & \ddots & \vdots\\
    a_{N1} & \cdots & a_{NM}\\
  \ebmtx\ .
\emath
\subsubsection{Matrix Properties}
Let $\vect{A}$ be a square matrix. We then have that
\begin{alignat}{3}
  &\vect{A}\text{ is orthogonal}    &\ &\iff &\ \vect{A}^T&=\vect{A}^{-1}\ \iff\ \vect{I} = \vect{AA}^T =  \vect{A}^T\vect{A}\\
  &\vect{A}\text{ is symmetric}     &\ &\iff &\ \vect{A}^T&=\vect{A}\\
  &\vect{A}\text{ is skew-symmetric}&\ &\iff &\ \vect{A}^T&=-\vect{A}\ .
\end{alignat}
Let $Q(\vect{x}) = \vect{x}^T\vect{Ax}$ where $\vect{A}$ is a square and symmetric matrix. We then have that
\begin{alignat*}{4}
  &\vect{A}\text{ is positive definite (p.d.)}      &\ &\iff &\ Q(\vect{x})&>    0   &\quad &\forall\ \vect{x}\neq 0\\
  &\vect{A}\text{ is positive semidefinite (p.s.d.)}&\ &\iff &\ Q(\vect{x})&\geq 0   &\quad &\forall\ \vect{x}\\
  &\vect{A}\text{ is negative definite (n.d.)}      &\ &\iff &\ Q(\vect{x})&<    0   &\quad &\forall\ \vect{x}\neq 0\\
  &\vect{A}\text{ is negative semidefinite (n.s.d.)}&\ &\iff &\ Q(\vect{x})&\leq 0   &\quad &\forall\ \vect{x}\\
  &\vect{A}\text{ is indefinite otherwise.}         & &      &             &         &\quad &
\end{alignat*}
The interested reader can find more on linear algebra in for example \cite{Lay2005} and \cite{Strang2009}.

\subsection{Optimisation}
\label{ssec:optimisation}
\subsubsection{Gradiant and Hessian}
Let $f(\vect{x})$ be a scalar function with continuous second-order partial derivatives. The gradient vector of $f(\vect{x})$ is defined as
\bmath
  \vect{g}(\vect{x}) = \vect{\nabla}f(\vect{x}) = \frac{\partial f}{\partial\vect{x}} = \bbmtx\frac{\partial f}{\partial x_1} & \frac{\partial f}{\partial x_2} & \cdots & \frac{\partial f}{\partial x_N}\ebmtx^T\ .
\emath
The Hessian matrix of $f(\vect{x})$ is symmetric and defined as
\begin{align}
  \vect{H}(\vect{x}) &= \vect{\nabla}[\vect{g}^T(\vect{x})] = \vect{\nabla}[\vect{\nabla}^Tf(\vect{x})] = \frac{\partial^2 f}{\partial\vect{x}\partial\vect{x}^T}\notag\\
  &=
  \bbmtx
    \frac{\partial^2 f}{\partial x_1^2} & \cdots & \frac{\partial^2 f}{\partial x_1\partial x_N}\\
    \vdots & \ddots & \vdots\\
    \frac{\partial^2 f}{\partial x_N\partial x_1} & \cdots & \frac{\partial^2 f}{\partial x_N^2}
   \ebmtx\ .
\end{align}
When $f(\vect{x})$ consists of a constant term, we have that
\bmathnt
  f(\vect{x}) = \vect{c}\ \implies
  \begin{cases}
    \vect{g}(\vect{x})=\vect{0}\\
    \vect{H}(\vect{x})=\vect{0}\\
  \end{cases}\ .
\emathnt
When $f(\vect{x})$ consists of a linear term, we have that
\bmathnt
  f(\vect{x}) = \vect{v}^T\vect{x} = \vect{x}^T\vect{v}\ \implies
  \begin{cases}
    \vect{g}(\vect{x})=\vect{v}\\
    \vect{H}(\vect{x})=\vect{0}\\
  \end{cases}\ .
\emathnt
When $f(\vect{x})$ consists of a quadratic term, we have that
\bmathnt
  f(\vect{x}) = \vect{x}^T\vect{Ax}\ \implies
  \begin{cases}
    \vect{g}(\vect{x})=(\vect{A}+\vect{A}^T)\vect{x} = 2\vect{Ax}\\
    \vect{H}(\vect{x})=\vect{A}+\vect{A}^T = 2\vect{A}\\
  \end{cases}
\emathnt
where the last equality for the gradient and the Hessian holds if and only if $\vect{A}$ is symmetric.
\subsubsection{Unconstrained Optimisation}
We can solve an unconstrained optimisation problem by following a five step recipe.
\begin{enumerate}
  \item Construct the cost function $f(\vect{x})$.
  \item Find the gradient $\vect{g}(\vect{x})=\frac{\partial f}{\partial\vect{x}}$.
  \item Solve $\vect{g}(\vect{x})=\vect{0}$ for $\vect{x}$. The solutions $\{\vect{x}_i\}$ are called critical points.
  \item Find the Hessian $\vect{H}(\vect{x})=\frac{\partial^2 f}{\partial\vect{x}\partial\vect{x}^T}$ and compute it for all the critical points. If
\begin{alignat*}{3}
  &\vect{H}(\vect{x}_i)\text{ is p.d}           &\ &\implies &\ &f(\vect{x}_i)\text{ is a local minimum}\\
  &\vect{H}(\vect{x}_i)\text{ is n.d}           &\ &\implies &\ &f(\vect{x}_i)\text{ is a local maximum}\\
  &\vect{H}(\vect{x}_i)\text{ is indefinite}    &\ &\iff     &\ &f(\vect{x}_i)\text{ is a saddle point}\\
  &\vect{H}(\vect{x}_i)\text{ is p.s.d or n.s.d}&\ &\iff     &\ &\text{further analysis is necessary \cite[p.~41]{Antoniou2007}}
\end{alignat*}
  \item Find the global minimiser/maximiser by evaluating $f(\vect{x}_i)$ for every critical point.
\end{enumerate}
If $f(\vect{x})$ is a convex\footnote{If we should be completely accurate, we should distinguish between weakly and strictly convex functions. However, in this note, we only use convexity in the strict sense.} function, we have the following useful fact \cite[p.~57]{Antoniou2007}
\bmathnt
  f(\vect{x})\text{ is a convex function}\ \iff\ \vect{H}(\vect{x})\text{ is p.d. }\forall\ \vect{x}\ .
\emathnt
Thus, there is only one critical point corresponding to a minimum if and only if $f(\vect{x})$ is a convex function.

The interested reader can find more on optimisation in for example \cite{Antoniou2007} and \cite{Boyd2004}.

\subsection{Stochastic Processes}
Let $X(n)$ for $n=n_0,n_0+1,\ldots,n_0+N-1$ be a stochastic process with joint probability density function (pdf) $p_X(x(n_0),x(n_0+1),\ldots,x(n_0+N-1))$.
\subsubsection{Mean, Covariance and Correlation}
The mean, covariance and correlations sequences are defined as
\begin{alignat}{2}
  &\text{Mean sequence:}       &\quad \mu_X(n)  ={}&E[X(n)] = \int x(n)p_X(x(n))dx(n)\\
  &\text{Covariance sequence:} &\quad c_X(n,n+k)={}&\text{cov}(X(n),X(n+k))\notag\\
  &                            &\quad           ={}& E[(X(n)-\mu_X(n)) (X(n+k)-\mu_X(n+k))]\\
  &\text{Correlation sequence:}&\quad r_X(n,n+k)={}& E[X(n)X(n+k)]\ ,
\end{alignat}
and they are related by
\bmath
  c_X(n,n+k) = r_X(n,n+k) - \mu_X(n)\mu_X(n+k)\ .
\emath
For a finite number of observation, we may define the mean vector
\bmath
  \vect{\mu}_X = \bbmtx\mu_X(n_0) &\mu_X(n_0+1)&\cdots & \mu_X(n_0+N-1)\ebmtx^T\ ,
\emath
the covariance matrix
\bmath
  \vect{C}_X =
  \bbmtx
    c_X(n_0,n_0) & \cdots & c_X(n_0,n_0+N-1)\\
    \vdots & \ddots & \vdots\\
    c_X(n_0+N-1,n_0) & \cdots & c_X(n_0+N-1,n_0+N-1)\\
  \ebmtx\ ,
\emath
and the correlation matrix
\bmath
  \vect{R}_X =
  \bbmtx
    r_X(n_0,n_0) & \cdots & r_X(n_0,n_0+N-1)\\
    \vdots & \ddots & \vdots\\
    r_X(n_0+N-1,n_0) & \cdots & r_X(n_0+N-1,n_0+N-1)\\
  \ebmtx\ .
\emath
They are related by
\bmath
  \vect{C}_X = \vect{R}_X - \vect{\mu}_X\vect{\mu}_X^T\ .
\emath
$\vect{C}_X$ and $\vect{R}_X$ are symmetric and p.s.d. They are p.d. if $X(n)$ is \textit{not} perfectly predictable.
\subsubsection{Stationarity and Wide Sense Stationarity (WSS)}
\begin{alignat*}{3}
  &X(n)\text{ is stationary} &\ &\iff &\ & p_X(x(n_0+m),x(n_0+1+m),\ldots,x(n_0+N-1+m))\\
  &{}\qquad\Downarrow        &  &     &  &{}\ =p_X(x(n_0),x(n_0+1),\ldots,x(n_0+N-1))\quad \forall\  n_0,m,N\\
  &X(n)\text{ is WSS}        &\ &\iff &\ &
  \begin{cases}
    \mu_X(n) = \mu_X & \text{(a constant)}\\
    r_X(n,n+k) = r_X(0,k) & \text{(a function of $k$ not $n$)}
  \end{cases}
\end{alignat*}
Since $r_X(0,k)$ does not depend on $n$, it is often written as $r_X(k)$. A Venn diagram for all stochastic processes is shown in Fig.~\ref{fig:rand_proc_relation}.

\begin{figure}[htbp]
  \centering
  \inputTikZ{figures/rand_proc_relation}
  \caption{A Venn diagram for all stochastic processes.}
  \label{fig:rand_proc_relation}
\end{figure}

Let $Y(n)$ be another stochastic process.
\bmathnt
  X(n) \text{ and } Y(n) \text{ are jointly WSS }\iff\
  \begin{cases}
    X(n) \text{ is WSS}\\
    Y(n) \text{ is WSS}\\
    r_{X,Y}(n,n+k) = r_{X,Y}(0,k)
  \end{cases}
\emathnt
where $r_{X,Y}(n,n+k)$ is the cross-correlation sequence.
\subsubsection{Estimation of Statistics}
Often, we wish to estimate the statistics of $X(n)$ from a single realisation $x(n)$. If $X(n)$ is (wide sense) stationary and \textit{ergodic}\footnote{Ergodicity basically means that we can infer something about the statistics of a stochastic process from a single realisation of it.} in some sense (for example, in mean, in correlation, in power, or in distribution), we may estimate some or all of its statistics from a single realisation $x(n)$. Examples are the unbiased estimators of the mean
\bmath
  \hat{\mu}_X = \frac{1}{N}\sum_{n=n_0}^{n_0+N-1} x(n)
\emath
and the correlation sequence
\bmath
  \hat{r}_X(k) = \frac{1}{N-k}\sum_{n=n_0}^{n_0+N-1-k} x(n)x(n+k)\ .
\emath

The interested reader can find more on stochastic processes in for example \cite{Kay2005} and \cite{Leon-Garcia2008}.

\section{Block Diagram of Adaptive Filtering}
The adaptive filtering problem is shown in Fig.~\ref{fig:wss_block_diagram}.
\begin{figure}[htbp]
  \centering
  \inputTikZ{figures/wss_block_diagram}
  \caption{Block Diagram of Adaptive Filtering in a WSS environment.}
  \label{fig:wss_block_diagram}
\end{figure}
From the figure, we have that
\begin{itemize}
  \item[$u(n)$:] zero-mean, WSS input signal
  \item[$w_m$:] $M$-tap FIR-filter with impulse response $w_0,w_1,\ldots,w_{M-1}$
  \item[$y(n)$:] output signal given by
  \bmath
    y(n) = \sum_{m=0}^{M-1}w_m u(n-m)
  \emath
  \item[$d(n)$:] zero-mean, WSS desired signal
  \item[$e(n)$:] error signal
\end{itemize}
Moreover, $u(n)$ and $d(n)$ are assumed to be jointly WSS.
\subsection{Mean-Squared Error and Squared Error cost functions}
Define
\begin{align}
  \vect{w} &= \bbmtx w_0 & w_{1} & \cdots & w_{M-1}\ebmtx^T\\
  \vect{u}(n)&= \bbmtx u(n) & u(n-1) & \cdots & u(n-M+1)\ebmtx^T\\
  \vect{R}_u &= E[\vect{u}(n)\vect{u}^T(n)]\\
  \vect{r}_{ud} &= E[\vect{u}(n)d(n)]\ .
\end{align}
Here, $\vect{R}_u$ is the correlation matrix of the input signal vector $\vect{u}(n)$, and $\vect{r}_{ud}$ is the cross-correlation vector between the input signal vector $\vect{u}(n)$ and the desired signal $d(n)$.

For $n=n_0,n_0+1,\ldots,n_0+K-1$ with $K\geq M$, define
\begin{align}
  \vect{A} &= \bbmtx \vect{u}(n_0) & \vect{u}(n_0+1) & \cdots & \vect{u}(n_0+K-1)\ebmtx^T\label{eq:ls_phi}\\
  \vect{d} &= \bbmtx d(n_0) & d(n_0+1) & \cdots & d(n_0+K-1)\ebmtx^T\label{eq:ls_d}\\
  \vect{e} &= \bbmtx e(n_0) & e(n_0+1) & \cdots & e(n_0+K-1)\ebmtx^T\label{eq:ls_e}\ .
\end{align}
Then
\bmath
  e(n) = d(n) - \vect{u}^T(n)\vect{w}\quad\iff\quad d(n) = \vect{u}^T(n)\vect{w} + e(n)
\emath
and
\bmath
  \vect{e} = \vect{d}-\vect{A}\vect{w}\quad\iff\quad\vect{d}=\vect{A}\vect{w}+\vect{e}\ .
  \label{eq:ls_model}
\emath
We observe the input signal $u(n)$ and the desired signal $d(n)$, and we wish to find the filter vector $\vect{w}$ which minimises the error in some sense. Two popular choices for the cost function of this problem are
\begin{alignat}{3}
 &\text{Mean-squared error:} &\qquad J_1(\vect{w}) &= E[e(n)^2] &\qquad &\text{(statistics is known)}\\
 &\text{Squared error:} &\qquad J_2(\vect{w}) &= \sum_{n=n_0}^{n_0+K-1}e^2(n)=\vect{e}^T\vect{e} &\qquad &\text{(statistics is unknown)}
  \label{eq:cf_squared_error}
\end{alignat}
If $\rank{\vect{A}}>M$, \eq{eq:ls_model} constitutes an overdetermined system of equations which do not have an exact solution that makes $J_2(\vect{w})=0$. The problem of minimising the squared error is often referred to as \textit{the method of least-squares}, and the argument minimising the squared error is referred to as \textit{the least-squares solution}. Moreover, the squared error divided by the number of elements in the error vector $\vect{e}$ can be seen as an unbiased estimate of the mean-squared error when $u(n)$ and $d(n)$ are assumed to be jointly WSS. That is,
\bmath
  E\left[\frac{1}{K}J_2(\vect{w})\right] = J_1(\vect{w})\ .
\emath

\section{The Wiener-Hopf Equations}
We want to solve the following unconstrained optimisation problem
\bmath
  \vect{w}_o = \argmin_{\vect{w}} J_1(\vect{w})
\emath
where $\vect{w}_o$ is the vector containing the optimal filter coefficients. We use the five step recipe in Sec.~\ref{ssec:optimisation} to solve the optimisation problem.
\begin{enumerate}
  \item Construct the cost function
  \begin{align}
    J_1(\vect{w}) &= E[e(n)^2] = E[(d(n)-\vect{u}^T(n)\vect{w})^2] = E[(d(n)-\vect{u}^T(n)\vect{w})^T(d(n)-\vect{u}^T(n)\vect{w})]\notag\\
    &= E[d(n)^2]+E[\vect{w}^T\vect{u}(n)\vect{u}^T(n)\vect{w}]-E[d(n)\vect{u}^T(n)\vect{w}]-E[\vect{w}^T\vect{u}(n)d(n)]\notag\\
    &= E[d(n)^2]+\vect{w}^TE[\vect{u}(n)\vect{u}^T(n)]\vect{w}-2\vect{w}^TE[\vect{u}(n)d(n)]\notag\\
    &= \sigma_d^2 + \vect{w}^T\vect{R}_u\vect{w}-2\vect{w}^T\vect{r}_{ud}\quad\text{(quadratic cost function)}
  \end{align}
  \item Find the gradient
  \bmath
    \vect{g}(\vect{w}) = (\vect{R}_u+\vect{R}_u^T)\vect{w}-2\vect{r}_{ud}=2\vect{R}_u\vect{w}-2\vect{r}_{ud}
  \emath
  \item Solve $\vect{g}(\vect{w})=\vect{0}$ for $\vect{w}$
  \begin{alignat}{3}
    &&\qquad \vect{g}(\vect{w}) &= \mathrlap{2\vect{R}_u\vect{w}-2\vect{r}_{ud} = \vect{0}}\notag\\
    \ArrowBetweenLines
    &&\qquad \alignedbox{\vect{R}_u\vect{w}}{=\vect{r}_{ud}}  &\qquad &\text{(Wiener-Hopf Equations)}\\
    \ArrowBetweenLines
    &&\qquad           \vect{w} &= \vect{R}_u^{-1}\vect{r}_{ud} &\qquad &\text{(If }\vect{R}_u\text{ is invertible)}
  \end{alignat}
  \item Find the Hessian
  \bmath
    \vect{H}(\vect{w}) = 2\vect{R}_u
  \emath
  which is p.d. for all $\vect{w}$ if $u(n)$ is not perfectly predictable.
  \item This implies that
  \begin{itemize}
    \item $J_1(\vect{w})$ is a convex function,
    \item $\vect{R}_u$ is invertible,
    \item $\vect{w}_o = \vect{R}_u^{-1}\vect{r}_{ud}$ is the global minimiser, and
    \item $J_1(\vect{w}_o) = \sigma_d^2-\vect{r}_{ud}^T\vect{R}_u^{-1}\vect{r}_{ud}$\ .
  \end{itemize}
\end{enumerate}
The solution $\vect{w}_o$ is often referred to as the least-mean-squares solution.

\subsection{Principle of Orthogonality}
The Wiener-Hopf equations can also be derived from the principle of orthogonality. For $m=0,1,\ldots,M-1$, the principle of orthogonality states
\bmath
  0=E[u(n-m)e_o(n)]\ \iff\ 0=E[y_o(n)e_o(n)]\ \iff\ \vect{w}_o = \vect{R}_u^{-1}\vect{r}_{ud}\ .
\emath
where $e_o(n)$ and $y_o(n)$ are the error and the filter output, respectively, when the filter is optimised in the mean-squared sense.

\subsection{The Modified Yule-Walker Equations}
The Wiener-Hopf equations are closely related to the Modified Yule-Walker equations. To see this, let $d(n)=u(n+1)$. Then
\bmath
  \vect{r}_{ud} = E[\vect{u}(n) d(n)] = E[\vect{u}(n) u(n+1)] = \vect{p}_u
\emath
which implies that
\bmath
  \vect{R}_u\vect{w} = \vect{p}_u \qquad \text{(Modified Yule-Walker Equations)}\ .
\emath

\section{The Normal Equations}
We want to solve the following unconstrained optimisation problem
\bmath
  \vect{w}_o = \argmin_{\vect{w}} J_2(\vect{w})
\emath
where $\vect{w}_o$ is the vector containing the optimal filter coefficients. We use the five step recipe in Sec.~\ref{ssec:optimisation} to solve the optimisation problem.
\begin{enumerate}
  \item Construct the cost function
  \begin{align}
    J_2(\vect{w}) &= \vect{e}^T\vect{e} = (\vect{d}-\vect{A w})^T(\vect{d}-\vect{A w})\notag\\
    &= \vect{d}^T\vect{d}+\vect{w}^T\vect{A}^T\vect{A w}-2\vect{w}^T\vect{A}^T\vect{d}\quad\text{(quadratic cost function)}
  \end{align}
  \item Find the gradient
  \bmath
    \vect{g}(\vect{w}) = (\vect{A}^T\vect{A}+\vect{A}^T\vect{A})\vect{w}-2\vect{A}^T\vect{d}=2\vect{A}^T\vect{A}\vect{w}-2\vect{A}^T\vect{d}
  \emath
  \item Solve $\vect{g}(\vect{w})=\vect{0}$ for $\vect{w}$
  \begin{alignat}{3}
    &&\qquad \vect{g}(\vect{w}) &= \mathrlap{2\vect{A}^T\vect{A}\vect{w}-2\vect{A}^T\vect{d} = \vect{0}}\notag\\
    \ArrowBetweenLines
    &&\qquad \alignedbox{\vect{A}^T\vect{A}\vect{w}}{=\vect{A}^T\vect{d}}  &\qquad &\text{(Normal Equations)}\\
    \ArrowBetweenLines
    &&\qquad           \vect{w} &= (\vect{A}^T\vect{A})^{-1}\vect{A}^T\vect{d} &\qquad &\text{(If }\vect{A}\text{ has full rank)}
  \end{alignat}
  \item Find the Hessian
  \bmath
    \vect{H}(\vect{w}) = 2\vect{A}^T\vect{A}
  \emath
  which is p.d. for all $\vect{w}$  if $\vect{A}$ has full rank.
  \item This implies that
  \begin{itemize}
    \item $J_2(\vect{w})$ is a convex function,
    \item $\vect{w}_o = (\vect{A}^T\vect{A})^{-1}\vect{A}^T\vect{d}$ is the global minimiser, and
    \item $J_2(\vect{w}_o) = \vect{d}^T(\vect{I}-\vect{P}_u)\vect{d} = \vect{d}^T\vect{P}_u^{\perp}\vect{d}$ where $\vect{P}_u=\vect{A}(\vect{A}^T\vect{A})^{-1}\vect{A}^T$ is the orthogonal projection matrix and $\vect{P}_u^{\perp}$ is the complementary projection matrix to $\vect{P}_u$.
  \end{itemize}
\end{enumerate}
The solution $\vect{w}_o$ is often referred to as the least-squares solution.

\subsection{Principle of Orthogonality}
\begin{figure}
  \centering
  \inputTikZ{figures/orthogonality_principle}
  \caption{The principle of orthogonality for the squared error cost function.}
  \label{fig:orthogonality_principle}
\end{figure}
The normal equations can also be derived from the principle of orthogonality
\bmath
  \vect{0}=\vect{A}^T\vect{e}_o\ \iff\ \vect{0}=\vect{y}_o^T\vect{e}_o\ \iff\ \vect{w}_o = (\vect{A}^T\vect{A})^{-1}\vect{A}^T\vect{d}\ .
\emath
where $\vect{e}_o$ and $\vect{y}_o$ are the error vector and the filter output vector, respectively, when the filter is optimised in the squared sense. Fig.~\ref{fig:orthogonality_principle} illustrates the principle of orthogonality.

\subsection{Estimation of the (Cross-)Correlation}
Comparing the normal equations with the Wiener-Hopf equations, we see that
\begin{align}
  \hat{\vect{R}}_u &= \frac{1}{K}\vect{A}^T\vect{A}\label{eq:ls_Ru_est}\\
  \hat{\vect{r}}_{ud} &= \frac{1}{K}\vect{A}^T\vect{d}\label{eq:ls_rud_est}
\end{align}
are the build in estimates of the correlation matrix and the cross-correlation vector, respectively, in the method of least-squares.

\subsection{Data Windowing}
Recall the definition of $\vect{A}^T$ in \eq{eq:ls_phi} with $n_0=1$ and $K=N+M-1$.
\bmathnt
\left[
  \begin{array}{@{}c@{}|c@{}|c@{}}
    \begin{matrix}
      u(1)    & \cdots & u(M-1)\\
      u(0)    & \cdots & u(M-2)\\
      \vdots  & \ddots & \vdots\\
      u(2-M)  & \cdots & u(0)\\
    \end{matrix} &
    \begin{matrix}
      u(M)    & \cdots & u(N)\\
      u(M-1)  & \cdots & u(N-1)\\
      \vdots  & \ddots & \vdots\\
      u(1)    & \cdots & u(N-M+1)\\
    \end{matrix} &
    \begin{matrix}
      u(N+1)  & \cdots & u(N+M-1)\\
      u(N)    & \cdots & u(N+M-2)\\
      \vdots  & \ddots & \vdots\\
      u(N-M+2)& \cdots & u(N)\\
    \end{matrix}
  \end{array}
  \right]
\emathnt
In real-world applications, we only observe a finite amount of data. Assume that we observe $u(n)$ for $n=1,\ldots,N$ while the remaining data for $n\leq 0$ and $n>N$ are unobserved. If we set the unobserved data equal to zero, we obtain the structure for $\vect{A}$ shown in Fig~\ref{fig:ls_windowing}.
\begin{figure}[htbp]
  \centering
  \inputTikZ{figures/ls_windowing}
  \caption{Four different data windowing methods.}
  \label{fig:ls_windowing}
\end{figure}
The various choices for $\vect{A}$ are
\begin{alignat*}{3}
  &\vect{A}_1\text{ - covariance method:}   &\qquad n_0&=M &\qquad K&=N-M+1\\
  &\vect{A}_2\text{ - autocorrelation method:}  &\qquad n_0&=1 &\quad K&=N+M-1\\
  &\vect{A}_3\text{ - prewindowing method:} &\qquad n_0&=1 &\quad K&=N    \\
  &\vect{A}_4\text{ - postwindowing method:}&\qquad n_0&=M &\quad K&=N
\end{alignat*}
The cost function and the length of $\vect{e}$ in \eq{eq:ls_e} and $\vect{d}$ in \eq{eq:ls_d} must be adjusted according to the windowing method. The covariance method leads to unbiased estimates of $\vect{R}_u$ and $\vect{r}_{ud}$ in \eq{eq:ls_Ru_est} and \eq{eq:ls_rud_est}, respectively.


\chapter{How to Make a Map}\label{chap:maps}
\section{Notation}
Let's start with a four dimensional real vector space $V$.
Associated with it, we have some set of basis vectors, so we can say that any vector $\vec{m} = a\vec{w} + b\vec{v} + c\vec{p} + d\vec{q}$.
Out of the infinite possibilities for a basis for $V$, let's pick an arbitrary basis, and call each element $e$, using a \emph{subsecript} to identify which of the four basis vectors it is; so our basis is $\{\vec{e}_0, \vec{e}_1, \vec{e}_2, \vec{e}_3\}$.
If we want to talk about a general basis vector, we'll call it $\{\vec{e}_\mu\}$. Notice that the identifier for basis vectors in on the \emph{bottom}. This will be a very important idea.

Let's consider a vector $\vec{A} \in V$; We can write $\vec{A}$ in the following way.
\begin{align*}
    \vec{A} &= a\vec{e}_0 + b\vec{e}_1 + c\vec{e}_2 + d\vec{e}_3 \\
            &= A^0\vec{e}_0 + A^1\vec{e}_1 + A^2\vec{e}_2 + A^3\vec{e}_3 \\
            &= \sum A^\mu \vec{e}_\mu \\
            &= A^\mu \vec{e}_\mu.
\end{align*}
This is an example of \emph{Einstein Summation Convention}, where adjacent upper and lower indices are automatically summed over.

\section{Maps}
Let's create two vector spaces, a four dimensional real vector space $V$ with basis $\{\vec{e}_\mu\}$ and a 10 dimensional real vector space $W$ with basis $\{\vec{f}_\mu\}$.
Now let's create a \emph{linear}\footnote{Recall that a linear map $\Lambda : D \to R$ satisfies the following: $\Lambda(0_D) = 0_R$, $\Lambda(\vec{u} + \vec{v}) = \Lambda(\vec{u}) + \Lambda(\vec{v})$, and $\Lambda(\alpha\vec{v}) = \alpha\Lambda(\vec{v})$.} map $\Lambda$ which takes any vector in $V$ and assigns it to a vector in $W$.
We write this as $\Lambda : V \to W$.
If we want to show how this map acts on a particular domain vector, we can write it in several different ways:
\begin{align*}
    \Lambda : V &\to W \tag{Definition} \\
    \Lambda\vec{v} &= \vec{w} \tag{Operator notation} \\
    \Lambda(\vec{v}) &= \vec{w} \tag{Function notation} \\
    \langle \Lambda,\vec{v} \rangle &= \vec{w}. \tag{Bracket notation}
\end{align*}
This last notation is the most useful one for discussing tensors.
It has several benefits for us: if we write something like $\langle \Lambda, \cdot \rangle$, it looks like a function waiting to accept a certain vector in order to get an output $\vec{w}$; conversely, if we write $\langle \cdot, \vec{v} \rangle$, it looks like we're looking for a certain map to act upon $\vec{v}$ in order to get $\vec{w}$.

Now let's consider what happens when we apply this map to each of the basis vectors $\vec{e}_\mu$:
\begin{align*}
    \langle \Lambda, \vec{e}_0 \rangle &= 3\vec{f}_1 + 2\vec{f}_4 + 5\vec{f}_9 \\
    \langle \Lambda, \vec{e}_1 \rangle &= \pi \vec{f}_3 + \vec{f}_0 \\
    \langle \Lambda, \vec{e}_2 \rangle &= \vec{f}_2 \\
    \langle \Lambda, \vec{e}_3 \rangle &= \vec{f}_3 + \vec{f}_5 + \vec{f}_7 + \vec{f}_9
\end{align*}
Obviously, this is a rather random mapping. The outputs here are completely arbitrary. The important thing to note is that this map is linear, so $\langle \Lambda, \alpha\vec{v} + \beta\vec{p} \rangle = \alpha\langle \Lambda, \vec{v} \rangle + \beta\langle \Lambda, \vec{p} \rangle$. Notice that, in the left hand side of the equation, the addition operation is the one from $V$ ($+_V$), while in the right hand side, it is the addition operation from $W$ ($+_W$), since $\langle \Lambda, \alpha\vec{v} \rangle,\langle \Lambda, \beta\vec{p} \rangle \in W$.

If we apply $\Lambda$ to the arbitrary vector $\vec{A}$ we defined previously, we see that
\begin{align*}
    \langle \Lambda, \vec{A} \rangle &= \langle \Lambda, A^\mu \vec{e}_\mu \rangle \\
    &= A^\mu \langle \Lambda, \vec{e}_\mu \rangle
\end{align*}
Since we already know how $\langle \Lambda, \vec{e}_\mu \rangle$ transforms, we can express what happens to any vector under $\Lambda$ this way.


\section{Inequalities}
\begin{theorem}[Markov's Inequality]
    Suppose we have a nonnegative random variable $X$, then for any $a>0$,
    $$\mathbb P(X\ge a)\le\frac{\mathbb E[X]}{a}$$
\end{theorem}
\begin{proof}
    Observe that $X\ge a1_{X\ge a}$ by checking each $\omega\in\Omega$, then we take expectation to get the inequality.
\end{proof}
\begin{theorem}[Chebyshev's Inequality]
    Let $X$ be a random variable with finite expectation, then for all $a>0$,
    $$\mathbb P(|X-\mathbb E[X]|\ge a)\le\frac{\operatorname{Var}(X)}{a^2}$$
\end{theorem}
\begin{proof}
    $$\mathbb P(|X-\mathbb E[X]|\ge a)=\mathbb P((X-\mathbb E[X])^2\ge a^2)\le\frac{\operatorname{Var}(X)}{a^2}$$
    by Markov's Inequality.
\end{proof}
\begin{theorem}[Cauchy-Schwarz Inequality]
    Let $X,Y$ be random variables, then $\mathbb E[|XY|]\le\sqrt{\mathbb E[X^2]\mathbb E[Y^2]}$
\end{theorem}
\begin{proof}
    It suffices to prove the case where $X,Y$ are nonnegative, in which case we can discard the absolute value.
    Also it is trivial when one of $\mathbb E[X^2],\mathbb E[Y^2]$ is infinite, so we assume that they are finite afterwards.
    So $\mathbb E[XY]\le (\mathbb E[X^2]+\mathbb E[Y^2])/2$ is also finite.
    Also if $\mathbb E[X^2]\mathbb E[Y^2]=0$ then the (in)equality is trivial, so we assume henceforth that they are both positive.
    There we have $0\le (X-tY)^2=X^2-2tXY+t^2Y^2$, from where taking expectation on both sides then yields $\mathbb E[X^2]-2t\mathbb E[XY]+t^2\mathbb E[Y^2]\ge 0$ for any $t$.
    The inequality follows by taking determinant, also equality happens iff $X=tY$ for some $t$, i.e. $X,Y$ are linearly dependent.
\end{proof}
\begin{theorem}[Jensen's Inequality]
    Let $f$ be a convex function defined on $\mathbb R$, that is for any $x,y\in\mathbb R, t\in[0,1]$,
    $$f(tx+(1-t)y)\le tf(x)+(1-t)f(y)$$
    Then if $X$ is a random variable,
    $$\mathbb E[f(X)]\ge f(\mathbb E[X])$$
\end{theorem}
\begin{proof}
    A convex function is equal to the supremum of all lines lying below it.
    That is, $\forall m\in\mathbb R,\exists a,b\in\mathbb R$ such that $f(m)=am+b$ and $\forall x\in\mathbb R,ax+b\le f(x)$.
    This is obvious by choosing $x,y$ with $x<m<y$ and apply the definition of convex functions, whence $\exists a\in\mathbb R$,
    $$\frac{f(m)-f(x)}{m-x}\le a\le \frac{f(y)-f(m)}{y-m}$$
    For any $x$ it gives $f(x)\ge a(x-m)+f(m)$, so we just take $b=-am+f(m)$.\\
    Now we go back to our proof.
    Let $m=\mathbb E[X]$, then we can choose $a,b$ such that $f(X)\ge aX+b$ and $f(m)=am+b$, taking expectation gives $f(\mathbb E[X])=a\mathbb E[X]+b\le \mathbb E[f(X)]$.
\end{proof}
Obviously we want to know when does the equality case hold in Jensen's Inequality.
Let $f$ be a convex function, assume that $\exists m\in\mathbb R$ such that $f(m)=am+b$ and $f(x)>ax+b$ for some $a,b$ real.
Suppose $m=\mathbb E[X]$, then consider $Y=f(X)-(aX+b)$ which is a nonnegative random variable.
Assume that $\mathbb E[f(X)]=f(\mathbb E[X])$, then $\mathbb E[Y]=0$, so $\mathbb P(Y=0)=1$, hence $\mathbb P(f(X)=aX+b)=1$, therefore $\mathbb P(X=m)=1$.
\begin{corollary}
    Let $f$ be convex and let $x_1,\ldots,x_n$ be real, then
    $$\frac{1}{n}\sum_{k=1}^nf(x_k)\ge f\left(\frac{1}{n}\sum_{k=1}^nx_k \right)$$
\end{corollary}
\begin{proof}
    Take $X$ be a random variable taking values uniformly in $x_1,\ldots,x_n$ and apply Jensen's Inequality.
\end{proof}
\begin{corollary}
    For positive $x_1,\ldots,x_n$,
    $$\frac{1}{n}\sum_{k=1}^nx_k\ge \left( \prod_{k=1}^nx_k \right)^{1/n}$$
\end{corollary}
\begin{proof}
    Take $f=-\log$.
\end{proof}


\textbf{Modify the code \texttt{fluidflowFD.m} to solve the equations}
\begin{align*}
 w_t + \psi_y w_x - \psi_xw_y &= Pr \Delta w + RaPrT_x, \\
 T_t + \psi_y T_x - \psi_xT_y &= \Delta T, \\
\Delta \psi &= -w,
\end{align*}
\textbf{where $(x,y) \in (0,1) \times (0,1)$ and $t>0$. For this exercise, set $Ra = 2 \times 10^5$ and $Pr = 0.71$ (air).}

\textbf{The fluid is at rest at $t=0$, with $T = \psi = w = 0$. The boundary conditions for the stream function is $\psi_{\Gamma} = 0$, which implies that there is no mass transfer through the boundary $\Gamma$. The value of the vorticity at the walls is expressed as $w_{\Gamma} = -\Delta \psi_{\Gamma}$ and the temperature at $\Gamma$ is defined by}
\begin{align*}
T(t,x,y) = \begin{cases}
2^9\tanh^4(100t)x^5(x-1)^4,& y =0,~0\leq x \leq 1,~t>0, \\
0, & (x,y) \in \Gamma,~y\neq 0,~t>0.
\end{cases}
\end{align*}

By replicating the code given to solve the fluid flow, adding the extra step of the temperature, we obtain the following result.

\begin{figure}[H]
\center{\includegraphics[scale=.75]{solutionProblem4.eps}}
\caption{Solution at $t=1$ second}
\end{figure}

\subsection*{Matlab code for shit part}
\begin{verbatim}
%% Problem 4
Ra=2e5;
Pr=0.71;
Problem4(Ra,Pr)

% Fluid flow in a cavity (Navier-Stokes equations)
% Stream-function and vorticity formulation
%
% Rodrigo Platte, Arizona State University, April 2013.

function Problem4(Ra,Pr)
% Parameters
path='Latex/FIGURES/';
N = 150;
dt = 1e-5;

h = 1/N; % N+1 points in each direction
[xx,yy] = meshgrid(0:h:1); % 2D gridpoints

% Laplacian with zero BCs
o = ones(N-1,1);
D2 = (diag(-2*o) + diag(o(1:N-2),-1)+ diag(o(1:N-2),1))/h^2;
% We need evals and evects to solve Poisson equation
[eV,eval] = eig(D2); lam = diag(eval);

% Initial condition & pre-allocate memory (start with zeros at t = 0)
u = 0*xx; v = u; w = u; str = u; T=u;

% boundary condition
ii = 2:N; jj = 2:N;  % index for interior nodes
wi = zeros(N-1,N-1); % interior values of vorticity
Ti = zeros(N-1,N-1); % interior values of temperature

count = 0;
time = 0;
outputTime=[1];
endtime=outputTime(end);
n=1;
% main loop
while time<endtime
	if ( time < outputTime(n) && time+dt >= outputTime(n) ) %#ok<ALIGN>
        dt=outputTime(n)-time;
        n=n+1;
    else
        dt = 1e-5;
    end
   % update vorticity at boundary
   w = BCw(u,v,w);
   % update Temperature at boundary
   T = BCT(xx,time,T);

   % advance vorticity and temperature with forward Euler
    [rhsT,Tx]=Trhs(u,v,T);
    Ti = Ti + dt*rhsT;
    T(ii,jj)=Ti;
    wi = wi + dt*wrhs(u,v,w,Pr,Ra,Tx);
    w(ii,jj) = wi;

   % compute stream function with conjugate gradient
   str(2:end-1,2:end-1) = SolvePoisson(wi);

   % update velocity (interior nodes only)
   u(ii,jj) =  (str(ii+1,jj)-str(ii-1,jj))/(2*h);
   v(ii,jj) = -(str(ii,jj+1)-str(ii,jj-1))/(2*h);

   time = time+dt;
   count = count + 1;

   % Plot results every now and then
   if ismember(time,outputTime);
       plotresults(u,v,str,w,T);
   end

end

    % Solve Poisson equation (Sylvester equation)
    function S = SolvePoisson(wi)
        ff = -eV'*wi*eV;
        S = ff;
        for j = 1:N-1
            S(:,j) = ff(:,j)./(lam(j)+lam);
        end
        S = eV*S*eV';
    end

    % Right-hand side of vorticity equation
    function rhs = wrhs(u,v,w,Pr,Ra,Tx)
        wx = (w(ii,jj+1)-w(ii,jj-1))/(2*h);
        wy = (w(ii+1,jj)-w(ii-1,jj))/(2*h);
        Lw = (w(ii,jj+1)+w(ii,jj-1)+w(ii+1,jj)+w(ii-1,jj)-4*w(ii,jj))/h^2;
        rhs = -u(ii,jj).*wx - v(ii,jj).*wy + Pr*Lw+Ra*Pr*Tx;
    end
    % Right-hand side of temperature equation
    function [rhs, Tx] = Trhs(u,v,T)
        Tx = (T(ii,jj+1)-T(ii,jj-1))/(2*h);
        Ty = (T(ii+1,jj)-T(ii-1,jj))/(2*h);
        LT = (T(ii,jj+1)+T(ii,jj-1)+T(ii+1,jj)+T(ii-1,jj)-4*T(ii,jj))/h^2;
        rhs = -u(ii,jj).*Tx - v(ii,jj).*Ty + LT;
    end

    % Enforce boundary condition (vorticity)
    function w = BCw(u,v,w)
        % three point sided FD formula (second order accurate)
        w(1,:) = -(-3*u(1,:)+4*u(2,:)-u(3,:))/(2*h);      % = -u_y @ y = 0
        w(N+1,:) = (-3*u(N+1,:)+4*u(N,:)-u(N-1,:))/(2*h); % = -u_y @ y = 1
        w(:,1) = (-3*v(:,1)+4*v(:,2)-v(:,3))/(2*h);       % =  v_x @ x = 0
        w(:,N+1) = -(-3*v(:,N+1)+4*v(:,N)-v(:,N-1))/(2*h);% =  v_x @ x = 1
    end
    % Enforce boundary condition (temperature)
    function T = BCT(xx,t,T)
        T(1,:)=2^9*tanh(100*t)^4*xx(1,:).^5.*(xx(1,:)-1).^4;
    end

    % Plot results
    function plotresults(u,v,str,w,T)
        % Use a subset of gridpoints
       ip = 1:3:N+1;
       up = u(ip,ip); vp = v(ip,ip);
       xp = xx(ip,ip); yp = yy(ip,ip);

       % Velocity
       subplot(2,2,1)
       quiver(xp,yp,up,vp,10)
       axis([0 1 0 1]), axis square
       title('velocity field','fontsize',16)

       % Normalized velocity
       subplot(2,2,2)
       contourf(xx,yy,T)
       axis([0 1 0 1]), axis square
       title('normalized velocity','fontsize',16)

       % Stream function
       subplot(2,2,3)
       mp = max(max(str));
       mm = min(min(str));
       contour(xx,yy,str,-logspace(-10,log10(-mm),30),'r'),  hold on
       contour(xx,yy,str, logspace(-10,log10(mp),20),'k'),   hold off
       axis square, title('streamlines (logscale)','fontsize',16)

       % Vorticity
       subplot(2,2,4)
       logw = log10(abs(w));    logw(abs(w)==0) = nan;
       contourf(xx,yy,logw,-3:0.5:2),
       H = colorbar; set(H,'fontsize',16)
       set(gca,'position',[0.5703 0.1100 0.3347 0.3412])
       axis square, title('vorticity (log(abs(w))','fontsize',16)
       drawnow
        txt=[path,'solutionProblem4'];
        saveas(gcf,txt,'epsc')
    end

end

\end{verbatim}


\section{Implementation of Feed-Forward Neural Network (30 points) }
Here, you will implement a simple one-layer feed-forward neural network (perceptron) from scratch to solve a classification task. To recap, a fully connected layer can be represented as a function parameterized by weight $\mathbf{W}$ and bias $b$, such that for row vector $\mathbf{x}$:
\begin{align}
f_{\mathbf{W},b}(\mathbf{x}) = \mathbf{x}\mathbf{W} + b
\end{align}
For classification tasks, one of the most common practice is to use a softmax function followed by a cross entropy loss. As shown in section 13.2.5 of the textbook, for a vector $\mathbf{x}=[x_1, x_2, \dotsc, x_n]$, the softmax function is defined as:
\begin{align}
\mu (\mathbf{x})_i = \frac{e^{x_i}}{\sum_j e^{x_j}}
\end{align}
\begin{align}
\mu (\mathbf{x}) = \left[\frac{e^{x_1}}{\sum_j e^{x_j}}, \dotsc, \frac{e^{x_n}}{\sum_j e^{x_j}}\right]
\end{align}

As shown in section 13.2.9 of the textbook, for target probability distribution $\mathbf{p}=[p_1, p_2, \dotsc, p_n]$ and a different probability distribution $\mathbf{q}=[q_1, q_2, \dotsc, q_n]$, the cross entropy $L(\mathbf{p}, \mathbf{q})$ is defined as:
\begin{align}
L(\mathbf{p}, \mathbf{q}) = - \sum_{i=1}^{n} p_i \log q_i
\end{align}

Note that in supervised learning, the target probability distribution $\mathbf{p}$ is usually represented as a one-hot vector, where $p_i=1$ shows that the corresponding input belongs to the $i$-th class with probability 1.

We can now represent our one-layer feed-forward neural network as a function $f_N$, such that for input vector $\mathbf{x}$ and its one-hot label vector $\mathbf{y}$:
\begin{align}
f_N(\mathbf{x}) = \mu(f_{\mathbf{W},b}(\mathbf{x}))
\end{align}

The loss for this input is thus $L(\mathbf{y}, f_N(\mathbf{x}))$. For a dataset $\mathcal{D}$ of size $N$, the loss is:
\begin{align}
L(\mathcal{D}) = \frac{1}{N} \sum_{i=1}^N L(\mathbf{y_i}, f_N(\mathbf{x_i}))
\end{align}

In order to minimize the loss function, we need to obtain the gradient with respect to weight $\mathbf{W}$ and bias $b$. In the next questions you will derive these gradient expressions using the chain rule.

\subquestion{(a) [5 Points]}

\task{What is $\frac{\partial L(\mathbf{y}, \mathbf{q})}{\partial q_i}$, the gradient of cross entropy loss $L(\mathbf{y}, \mathbf{q})$ with respect to $q_i$?}

\subquestion{(b) [5 Points]}
\task{What is $\frac{\partial \mu(\mathbf{x})_j}{\partial x_i}$, the gradient of softmax function $\mu(\mathbf{x})_j$ with respect to $x_i$? You should consider both $i=j$ and $i\neq j$.}

\subquestion{(c) [5 Points]}
\task{What is $\frac{\partial f_{\mathbf{W},b}(\mathbf{x})}{\partial W_{ij}}$ and $\frac{\partial f_{\mathbf{W},b}(\mathbf{x})}{\partial b}$, the gradient of fully connected layer $f_{\mathbf{W},b}(\mathbf{x})$ with respect to the $ij$-th entry of weight $\mathbf{W}$ and bias $b$?}

\bigskip

Now, you should be able to assemble the full gradient of weight and bias $\frac{\partial L(\mathcal{D})}{\partial W_{ij}}$, $\frac{\partial L(\mathcal{D})}{\partial b}$ using chain rule and the gradients you derived above.

\subquestion{(d) [15 Points]}

Next, let's go ahead and implement the one-layer neural network. Implement both the forward pass $L(\mathcal{D})$ and the backward gradient $\frac{\partial L(\mathcal{D})}{\partial W_{ij}}$ and $\frac{\partial L(\mathcal{D})}{\partial b}$. For optimization, we will implement Minibatch gradient descent and compare different batch sizes. \textbf{For this problem, you are allowed to keep the dataset in memory, and you do not need to use Spark. You are also allowed to use any external library, but we encourage you to implement gradients from scratch to deepen your understanding.}

\bigskip

\textbf{Mini batch gradient descent}: Go through the dataset in batches of predetermined size and update the parameters as follows:
\begin{algorithmic}
\WHILE {convergence criteria not reached}
\STATE Randomly pick $n=batch\_size$ random samples $\mathbf{x_k}$ from the training data

\FOR {$k = 1,...,n$}
\STATE Update $W_{ij} \leftarrow W_{ij} - \eta \frac{\partial L(\mathcal{D})}{\partial W_{ij}}$ for all i, j in $\mathbf{W}$
\STATE Update $b \leftarrow b - \eta \frac{\partial L(\mathcal{D})}{\partial b}$
\ENDFOR
\ENDWHILE
\end{algorithmic}
where $\eta$ is the learning rate and $batch\_size$ is the number of training samples considered in each batch. \\

For \textbf{convergence criteria}, you should stop learning when the cross entropy loss on \textbf{all data samples} is smaller than $0.4$. You are encouraged to vectorize your loss function and gradient calculation, but it's not required.

You will compare the performance of 3 different batch sizes.
\begin{enumerate}
    \item $batch\_size=1$. This is equivalent to stochastic gradient descent. Please use $\eta=0.1$.
    \item $batch\_size=20$. Please use $\eta=0.1$.
    \item $batch\_size=N$, where N is the total number of data samples. This is equivalent to full batch gradient descent. Please use $\eta=0.25$.
\end{enumerate}

Run your implementation on the data set in \textbf{q1/data}. The data set contains the following files :
\begin{enumerate}
\item \texttt{features.txt} : Each line contains features (comma-separated values) for a single datapoint. It has 6414 datapoints (rows) and 122 features (columns).

\item \texttt{targets.txt} : Each line contains the target variable (y = 0 or 1) for the corresponding row in \texttt{features.txt}.
\end{enumerate}

\task{Plot the value of cross entropy loss on \textbf{all data samples} $L(\mathcal{D})$ vs. the number of iterations ($k$) for all three batch sizes  until \textbf{convergence}. Report the total wall-clock runtime (second) taken for convergence by each of the batch size. Comment on the plots and the time for convergence. What can you infer from them?}

The diagram should have graphs from all the three batch sizes on the same plot.

\bigskip
\textbf{Important Note}
\begin{itemize}
    \item You should initialize your weight $\mathbf{W}$ following a normal distribution with mean 0 and standard deviation 1. You should initialize your bias $b$ to zeros. To make sure we have a fair comparison, please use the same initialization of weight and bias for all three runs.
    \item When computing the loss, remember to divide the total summed loss and gradient by the number of data points (i.e., batch size). This has been shown in Equation 6.
    \item You should calculate your cross entropy loss on \textbf{all data samples} for plotting and identifying convergence, not on the mini-batch of data samples.

\end{itemize}
As a sanity check, using batch size $N$ should converge in 10-400 iterations, batch size 20 between 400-1000 iterations and batch size 1 between 1000-2500 iterations. However, the number of iterations may vary greatly due to the high randomness of this problem. If your implementation consistently takes longer iterations though, you may have a bug.

\subsection*{What to submit}
\begin{enumerate}[(i)]
	\item Equation for part (a)
	\item Equation for part (b)
	\item Two equations for part (c)
	\item Plots for the cross entropy loss $L(\mathcal{D})$ vs. the number of iterations ($k$) for all three batch sizes 1, 20 and $N$. Total time taken for convergence by each of the batch size.  Interpretation of plot and convergence times. [part (d)]
	\item Submit the code on Gradescope submission website. [part (d)]
\end{enumerate}


\lab{Applications}{Markov Chains I}{Markov Chains I}
\label{lab:MarkovGraph}

\objective{This section teaches about two simple applications of Linear Algebra.
First, it teaches about Markov Chains, which, in this context, represent discrete random transitions.
Second, it teaches about Graph Theory, which can be used to represent many physical problems.}

\section*{Markov Chains}

%Lab \ref{Markov}

A Markov Chain describes a particular type of random variable.
This sort of random variable is characterized by the fact that all relevant information is related to its current state.
We can easily model this type of random variable using matrices.
We will start with a canonical example of a frog jumping from one lilypad to another.

Fredo the Frog hops around between the three lily pads 1, 2, and 3.
If he's on lily pad 1 and jumps, there is a 25\% chance that he will land back on lily pad 1, a 25\% chance that he will land on lily pad 2, and a 50\% chance that he will land on lily pad 3.
We can find similar probabilities if he starts on lily pad 2 or 3.
In figure \ref{markov1} we have a transition diagram that reflects the various probabilities that Fredo will go from one lily pad to another.

\begin{figure}[h!]
\begin{center}
\includegraphics[scale = 1.4]{markov1}
\end{center}
\caption{Transition diagram for Fredo the Frog}
\label{markov1}
\end{figure}

We can convert our transition diagram into a transition matrix, where the $(i,j)$-entry of the matrix corresponds to the probability that Fredo jumps from the $j^{th}$ lily pad to the $i^{th}$ lily pad (where $1$ is the first lily pad, 2 is the second, and so on).
In Fredo's case, the transition matrix is
\[
A = \begin{pmatrix}
1/4 & 1/2 & 1/2\\
1/4 & 1/6 & 1/2\\
1/2 & 1/3 & 0
\end{pmatrix}
\]
Note that all of the columns add up to one.
This is important.

If Fredo is on lily pad 1, where will he be after two jumps?
By multiplying the matrix $A$ by itself, we have (approximately)

\[
A^2 = \begin{pmatrix}
0.4375 & 0.3750 & 0.3750\\
0.3542 & 0.3194 & 0.2083\\
0.2083 & 0.3056 & 0.4167
\end{pmatrix}
\]
From this, we infer that there is a 43.75\% chance he will still be on lily pad 1 after two jumps.
Note that he might have jumped from 1 to 1 to 1, denoted $1 \rightarrow 1 \rightarrow 1$, or he could have jumped to one of the other lily pads and then back again, that is, either $1 \rightarrow 2 \rightarrow 1$ or $1 \rightarrow 3 \rightarrow 1$.
In addition, there is a 35.42\% chance he will be on lily pad 2 and a 20.83\% chance that he will be on lily pad 3.
Using Python, we can type in our transition matrix and see where Fredo will be after 5, 10, 20 or 100 jumps.

\begin{lstlisting}[style=python]
#Remember, the 1.'s in the numerator force floating point division
: A = sp.array([[1./4,1./2,1./2],[1./4,1./6,1./2],[1./2,1./3,0]])
: np.linalg.matrix_power(A,5)
: np.linalg.matrix_power(A,10)
: np.linalg.matrix_power(A,20)
: np.linalg.matrix_power(A,100)
\end{lstlisting}

Note that in the limit that the number of jumps goes to infinity, we get
\[
A^\infty = \begin{pmatrix}
0.4 & 0.4 & 0.4\\
0.3 & 0.3 & 0.3\\
0.3 & 0.3 & 0.3
\end{pmatrix}
\]
This means that after several jumps, the probability that we will find Fredo on a given lily pad will have nothing to do with where he started initially.

\section*{Markov Chains}

We can generalize this notion beyond that of frogs and lily pads.
Let the state of our system be represented by a probability vector
\[
\x = \begin{bmatrix}
x_1\\
x_2\\
\vdots\\
x_n
\end{bmatrix}
\]
where each entry represents the probability of being in that state.
Note that each entry is nonnegative and the sum of all the entries adds up to one.
For example, in the case of Fredo, if we know initially that he is on lily pad 1, then we have the state vector
\[
\x_0 = \begin{bmatrix}
1\\
0\\
0
\end{bmatrix}
\]
because we know for certainty (100\%) that Fredo is in the first state.
After one jump, we have
\[
\x_1 = A \x_0 = \begin{bmatrix}
0.25\\
0.25\\
0.50
\end{bmatrix}
\]
After two jumps, we have
\[
\x_2 = A \x_1 = A^2 \x_0 = \begin{bmatrix}
0.4375\\
0.3542\\
0.2083
\end{bmatrix}
\]
After a large number of jumps $(n>>1)$, we have
\[
\x_n = A \x_{n-1} = \dots = A^n \x_0 \approx \begin{bmatrix}
0.4\\
0.3\\
0.3
\end{bmatrix}
\]
Since all of the columns are the same for $A^\infty$, then for any initial probability vector $\x_0$, we get the same limiting output, or in other words, all initial vectors converge to the same point, call it $\x_\infty$.
Moreover, we have that
\[
\x_\infty = A \x_\infty
\]
This is called a stable fixed point.
How can we check that a stable fixed point exists?
Hint: Think eigenvalues and eigenvectors.

\section*{Example}

Consider the Markov chain given by
\[
A = \begin{pmatrix}
0.5 & 0.3 & 0.4\\
0.2 & 0.2 & 0.3\\
0.3 & 0.5 & 0.3
\end{pmatrix}.
\]
We show that it has a stable fixed point by checking that it has a single eigenvalue $\lambda=1$.
We do this via Python:
\begin{lstlisting}[style=python]
: A = sp.array([[.5,.3,.4],[.2,.2,.3],[.3,.5,.3]])
: V = la.eig(A)[1]
\end{lstlisting}
Note that the entries in the $\lambda=1$ eigenvector do not generally add up to one.
Indeed, any multiple of an eigenvector is an eigenvector.
So we need to multiply it by the appropriate constant so that all of the entries add up to one.
\begin{lstlisting}[style=python]
: x = V[:,0]
: x = x/sp.sum(x);x
array([ 0.41836735,  0.23469388,  0.34693878])
\end{lstlisting}
We can check this answer by taking $A$ to a high exponent, say $A^{100}$.

\begin{problem}
Suppose a basketball player's success at shooting free throws can be described with the following Markov chain
\[
A = \begin{pmatrix}.75&.50\\.25&.50\end{pmatrix}
\]
where the first state corresponds to success and the second state to failure.
\begin{enumerate}
\item If the player makes his first free throw, what is the probability that he also makes his third one?
\item What is the player's average free throw percentage?
\end{enumerate}
\end{problem}

\begin{problem}
Consider the Markov process given by the transition diagram in Figure \ref{markov2}.
\begin{enumerate}
\item Find the transition matrix.
\item If the Markov process is in state 1 initially, find the probability that it is in state 2 after 2 periods.
\item Find the stable fixed point if it exists.
\end{enumerate}
\end{problem}

\begin{figure}[h!]
\begin{center}
\includegraphics[scale = 1.4]{markov2}
\end{center}
\caption{Transition diagram}
\label{markov2}
\end{figure}

\newpage

\section*{Graph Theory}
\begin{figure}[h!]
\includegraphics[scale = .4]{graphExample}
\caption{A simple graph}
\label{markov:example_graph}
\end{figure}

Graph theory is an important branch of mathematics and computer science.
It describes how objects are connected to one another.
In a rigorous sense, a graph is composed of two sets: a set of nodes and a set of edges that connect these nodes.

A graph is directed if connections are uni-directional, and undirected if they are bi-directional.
Figure \ref{markov:example_graph} shows a simple undirected graph.
We can write a matrix that describes this type of graph.
We let each row of our matrix represent our starting point and each column represent our destination.
We put a 1 if there is a path and a 0 if there is not.
For the above graph we generate the following matrix:

\[
A = \begin{pmatrix}
0 & 1 & 0 & 0 & 1 & 0\\
1 & 0 & 1 & 0 & 1 & 0\\
0 & 1 & 0 & 1 & 0 & 0\\
0 & 0 & 1 & 0 & 1 & 1\\
1 & 1 & 0 & 1 & 0 & 0\\
0 & 0 & 0 & 1 & 0 & 0
\end{pmatrix}
\]

This matrix is called an adjacency matrix.
This matrix is symmetric since the graph is undirected.

What happens if we square an adjacency matrix?
It turns out that raising an adjacency matrix to the $n$ power yields the number of paths of length $n$ between two vertices.
For example, by squaring the above matrix Python gives:
\begin{lstlisting}[style=python]
: np.linalg.matrix_power(A,2)
array([[2, 1, 1, 1, 1, 0],
       [1, 3, 0, 2, 1, 0],
       [1, 0, 2, 0, 2, 1],
       [1, 2, 0, 3, 0, 0],
       [1, 1, 2, 0, 3, 1],
       [0, 0, 1, 0, 1, 1]])
\end{lstlisting}

Now try to find the number of connections of length 6 from node 3 to itself.
This is simple to do in Python:
\begin{lstlisting}[style=python]
: np.linalg.matrix_power(A,6)
array([[45, 54, 38, 45, 54, 16],
       [54, 86, 29, 77, 51, 11],
       [38, 29, 55, 15, 70, 27],
       [45, 77, 15, 75, 31,  4],
       [54, 51, 70, 31, 93, 34],
       [16, 11, 27,  4, 34, 14]])
\end{lstlisting}
It turns out that there are 55 unique paths of length 6 from node 3 to itself.
Imagine trying to count all of those paths by hand!
It would be very easy to count incorrectly.
However, this method makes it very simple to count paths without any mistakes.

\begin{problem}
Let the following matrix represent a directed graph
\[
A = \begin{pmatrix}
0 & 0 & 1 & 0 & 1 & 0 & 1\\
1 & 0 & 0 & 0 & 0 & 1 & 0\\
0 & 0 & 0 & 0 & 0 & 1 & 0\\
1 & 0 & 0 & 0 & 1 & 0 & 0\\
0 & 0 & 0 & 1 & 0 & 0 & 0\\
0 & 0 & 1  & 0 & 0& 0 & 1\\
0 & 1 & 0 & 0 & 0 & 0 & 0
\end{pmatrix}
\]
The greatest number of paths of length five are from which node to each node?
From which node to which node is their no path of length seven?
\end{problem}

It turns out that the study of graphs and connectivity has a variety of applications.
For example, connections between web pages can be described as graphs.
So can flights between airports or friends on social networking sites.
The same ideas are applied frequently in computer chip design and in the preservation of endangered species.

We can use an adjacency matrix to map out paths from one part of our graph to another.
Consider the following maze:

\begin{figure}
\includegraphics[width=.8\textwidth]{maze.pdf}
\caption{A maze for problem \ref{maze_prob}.}
\label{maze_fig}
\end{figure}

With this lab we have included the file \li{maze.npy} containing the adjacency matrix of this graph.

\begin{problem} \label{maze_prob}
Write a function that, given the numbers of two different nodes and an adjacency matrix for a graph, will find a shortest path (there may be more than one, just return one) between any two nodes in the graph.
Use the function you just wrote to solve the maze shown in figure \ref{maze_fig}.

Here is how this can be done:
\begin{itemize}
\item make a copy of the adjacency matrix
\item make a list containing the new aray
\item while the second node is not connected to the first node, update the copy to be itself times the adjacency matrix (using matrix multiplication) and append the new matrix to the end of the list.
Do not update your arrays in place because you need to be appending different arrays to the end of your list.
\item Once the second node is connected to the first node, work through your list of arrays.
Make a path starting at your first node and then consider all points that are one step away from the first node.
At least one of them should be connected to the final node in the second to the last matrix in the list.
Add one such node to your path and then continue in the same manner until you have a path between the nodes.
\end{itemize}

Have your function raise an error if there is no path between the nodes it is given.
One way to do this is to have it raise an error if it has found no connection after raising the adjacency matrix to the power of the number of nodes involved.
Essentially, if your paths are long enough to hit every node in the system and the two nodes you want to connect are still not connected, no connection can be found.

This is still not a fully optimized algorithm for solving this problem, but it works pretty well for small systems.
\end{problem}


\subsection{Mean value theorem}

\subsubsection{Mean value theorem}

Take a real function \(f(x)\) on closed interval \([a,b]\), differentiable on \((a,b,)\).

The mean value theorem states that:

\(\exists c\in(a,b) (f’(c)=\dfrac{f(b)-f(a)}{b-a})\)


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Discrete Optimal Transport}

Monge's original formulation of the OT problem corresponds to minimizing the cost for transporting a distribution $\mu_X$ onto another distribution $\mu_Y$ using a map $T$
\eql{
	\min_{T} \int_X c(x,T(x)) \d \mu_X(x),
	\qwhereq
	 T\#\mu_X=\mu_Y.
}
Here, $\mu_X,\mu_Y$ are measures in $\RR^d$, $T: \RR^d \rightarrow \RR^d$ is a $\mu_X$-measurable function, $c : \RR^d \times \RR^d \rightarrow \RR^+$ is a $\mu_X \otimes \mu_Y$-measurable function, and $\#$ is the push forward operator.

We focus here on the case where the measures are discrete, have the same number of points, and all points have the same mass, thus
\eq{
	\mu_X = \frac{1}{N} \sum_{i=1}^{N} \delta_{X_i}
	\qandq
	\mu_Y = \frac{1}{N} \sum_{j=1}^{N} \delta_{Y_j},
}
where $\delta_x$ is the Dirac measure at location $x \in \RR^d$, and where the position of the supporting points are $X = (X_i)_{i=1}^N$, and $Y = (Y_j)_{j=1}^N$, where  $X_i,Y_j \in \RR^d$. In this context, the transport between $X$ and $Y$ is a one-to-one assignment,  i.e.
% \todo{It was written $T(X_i) = X_{\sigma(i)}$ ! }
$T(X_i) = Y_{\sigma(i)}$ where $\sigma$ is a permutation of $\{1,\ldots,N\}$, which can be encoded using a permutation matrix $\Sigma$ such that
\eq{ \Sigma_{i,j} =
 	\choice{
		1 \qifq j=\sigma(i), \\
		0 \quad \text{otherwise}.
	}
}
A more compact way to denote the transport is $T(X_i)=\left(\Sig Y\right)_i, \forall i=\{1,\ldots,N\}$. Introducing the cost matrix
\eq{
	C_{X,Y} \in \RR^{N \times N}
	\qwhereq
	\foralls (i,j) \in \{1,\ldots,N\}^2, \quad (C_{X,Y})_{i,j} = c(X_i,Y_j),
}
this permutation matrix $\Sig$ is thus the solution to the following optimization problem
\eql{
	\umin{\Sig \in \Perm}
		\dotp{C_{X,Y}}{\Sigma} = \sum_{i,j=1}^N c(X_i,Y_j)  \Sig_{i,j},
\label{eqW}}
where $\Perm$ is the set of permutation matrices
\eq{
	\Perm = \enscond{\Sig \in \RR^{N \times N}}{ \Sig^* \U = \U, \Sig \U = \U, \Sig_{i,j} \in \{0,1\}},
}
see~\cite{Villani03} for more details. We have denoted $\U=(1,\ldots,1)^* \in  \RR^N$, and $A^*$ as the adjoint of the matrix $A$, that for real matrices amounts to the transpose operation.

In the special case where
\eq{
	\left(C_{X,Y}\right)_{i,j} = c(X_i,Y_j)=\norm{X_i-Y_j}^{\al}
} where $\norm{\cdot}$ is the Euclidean norm in $\RR^d$ and $\al \ge 1$,  the value of the optimization problem~\eqref{eqW} is called the $L^\alpha$-Wasserstein distance (to the power $\alpha$), and is denoted $W_\alpha(\mu_X,\mu_Y)^\alpha$. It can be shown that $W_\alpha$ defines a distance on the set of distributions that have moments of order $\al$.

%  Note that computing the Wasserstein distance requires the computation of the transport map $T(X)=\Sig X$.

%%%
\paragraph{ Kantorovich OT formulation}

The set of permutation matrices $\Perm$ is not convex.  Its convex hull is the set of bi-stochastic matrices
\eq{
	\Matr_1 = \enscond{\Sig \in \RR^{N \times N}}{ \Sig \U = \U, \Sig^* \U = \U, \Sig_{i,j} \in [0,1]}.
}
One can show that the relaxation
\eql{\label{eqMK}
	\min_{\Sig \in \Matr_1}  \dotp{C_{X,Y}}{\Sigma}
}
of~\eqref{eqW} is tight, meaning that there exists a solution of~\eqref{eqMK} which is a binary matrix, hence being also a solution of the original non-convex problem~\eqref{eqW}, see~\cite{Villani03}.


\subsection{Surface Integrals of Scalar Functions}
\begin{definition}
	The surface integral of a scalar function $f(x,y,z)$ is
	\begin{equation*}
		\iint\limits_{S}{f(x,y,z)\mathrm{d}s} = \iint\limits_{D}{(f\circ\vec{r})\norm{\vec{r_u} \times \vec{r_v}}\mathrm{d}A}
	\end{equation*}
\end{definition}

\noindent
Let’s apply a surface integral to a real problem. Let the cap of the sphere or radius $8\text{m}$ centered at the origin between $z = 7$ and $z = 8$ have a charge density $\sigma(x,y,z) = z \text{ } \mu \text{C}/ \text{m}^2$. Find the total charge $Q$ on the cap.

[INSERT IMAGE]

\noindent
We will parameterize the cap using spherical coordinates.
\begin{equation*}
	D = \left\{(\rho, \theta, \phi) \mid \rho=8, 0 \leq \theta \leq 2\pi, 0 \leq \phi \leq \cos^{-1}{\left(\frac{7}{8}\right)}\right\}
\end{equation*}
\begin{equation*}
	\vec{r}(\theta,\phi) = \langle 8\sin{\phi}\cos{\theta}, 8\sin{\phi}\sin{\theta}, 8\cos{\phi}\rangle
\end{equation*}
\begin{equation*}
	\vec{r_\theta} = \langle -8\sin{\phi}\sin{\theta}, 8\sin{\phi}\cos{\theta}, 0\rangle \text{, }
	\vec{r_\phi} = \langle 8\cos{\phi}\cos{\theta}, 8\cos{\phi}\sin{\theta}, -8\sin{\phi}\rangle
\end{equation*}
\begin{equation*}
	\sigma\circ\vec{r} = 8\cos{\phi} \text{, }
	\norm{\vec{r_\theta} \times \vec{r_\phi}} = 64\sin{\phi}
\end{equation*}
\begin{align*}
	Q &= \int_{0}^{2\pi}{\int_{0}^{\cos^{-1}{7/8}}{64\sin{\phi} \cdot8 \mathrm{d}\phi\mathrm{d}\theta}} \\
	& = 8^3 2\pi \int_{0}^{\cos^{-1}{7/8}}{\sin{\phi}\mathrm{d}\phi} \\
	&= 8^3 2\phi(-\cos{\phi})\rvert_{0}^{cos^{-1}{7/8}} = 8^3 2\pi\left(-7/8 + 1\right) = 128\pi\text{ }\mu\text{C}
\end{align*}


\subsection{Morphism}

Morphisms are functions which preserve the relationships between members of a set, and specified functions.

That is, if:

\(a\odot b=c\)

Then \(f(x)\) is morphism if:

\(f(a)\odot f(b)=f(a\odot b)\)

Here we discuss morphisms in the context of groups, but we can define morphisms for sets with more than one function, for example with addition and multiplication.

Morphisms are also known as homomorphisms.

The following are morphisms of the additive group of integers.

Where we refer to \(c\), \(c\ne 0\in \mathbb{I}\).

\begin{itemize}
\item \(f(x)=0\)
\item \(f(x)=x\)
\item \(f(x)=cx\)
\item Converting natural numbers to integers
\end{itemize}

The following are not morphisms

\begin{itemize}
\item \(f(x)=x+1\)
\end{itemize}

\subsubsection{Isomorphism}

An isomorphism is a morphism which has an inverse.

This means the function is bijective.

The following are isomorphisms:

\begin{itemize}
\item \(f(x)=x\)
\item \(f(x)=cx\)
\item Converting natural numbers to integers
\end{itemize}

The following are not isomorphisms

\begin{itemize}
\item \(f(x)=0\)
\item \(f(x)=x+1\)
\end{itemize}

\subsubsection{Endomorphism}

An endomorphism is one where the domain and codomain are the same.

The following are endomorphisms:

\begin{itemize}
\item \(f(x)=0\)
\item \(f(x)=x\)
\item \(f(x)=cx\)
\end{itemize}

The following are not endomorphisms

\begin{itemize}
\item Converting natural numbers to integers
\item \(f(x)=x+1\)
\end{itemize}

\subsubsection{Automorphism}

An endomorphism which is also an isomorphism

The following are automorphisms:
\begin{itemize}
\item \(f(x)=x\)
\item \(f(x)=cx\)
\end{itemize}

The following are not automorphisms

\begin{itemize}
\item \(f(x)=0\)
\item \(f(x)=x+1\)
\item Converting natural numbers to integers
\end{itemize}

\subsubsection{Monomorphism}

A morphism which is injective. That is:

\(f(a)=f(b)\rightarrow a=b\)

The following are monomorphisms:

\begin{itemize}
\item \(f(x)=x\)
\item \(f(x)=cx\)
\item Converting natural numbers to integers
\end{itemize}

The following are not monomorphisms:

\begin{itemize}
\item \(f(x)=0\)
\item \(f(x)=x+1\)
\end{itemize}


%---------------------------Scaled Jacobian-----------------------------
\section{Scaled Jacobian\label{s:tri-scaled-jacobian}}

First, let $L_{\max}$ be the product of the lengths of the 2 longest edges:
\[
  L_{\max} = \max\left\{
    \normvec{ L_0} \normvec{ L_1},
    \normvec{ L_0} \normvec{ L_2},
    \normvec{ L_1} \normvec{ L_2}
  \right\}
\]
Let $J^{\prime}$ be the Jacobian of the triangle.
If the triangle surface normal $\hat n$ is evaluated at the center of the triangle
and $\hat n\cdot\left(\vec L_2\times\vec L_1\right) < 0$, then take $J = -J^{\prime}$.
Otherwise take $J = J^{\prime}$.
The scaled Jacobian is then
\[
  q = \frac{2\sqrt{3}}{3} \frac{J}{L_{\max}}
\]
which is normalized so that a unit equilateral triangle has value $1$.

Note that if $L_{\max} \leq DBL\_MIN$, we set $q = 0$.

\trimetrictable{scaled Jacobian}%
{$1$}%                                                Dimension
{$[0.5,\frac{2\sqrt{3}}{3}]$}%                        Acceptable range
{$[-\frac{2\sqrt{3}}{3},\frac{2\sqrt{3}}{3}]$}%       Normal range
{$[-DBL\_MAX,DBL\_MAX]$}%                             Full range
{$1$}%                                                Unit equilateral triangle value
{\cite{knu:00}}%                                      Reference(s)
{v\_tri\_scaled\_jacobian}%                            Verdict function name


\subsection{Softmax}

The softmax function is often used in the last layer of a classification network.

It takes a vector of dimension \(k\) and returns another vector of the same size. Only, this time all numbers are between \(0\) and \(1\) and the values sum to \(1\).

The softmax function is based on the sigmoid function.

\(a_j(z)=\dfrac{e^{z_j}}{\sum_{i}e^{z_i}}\)

\subsection{Temperature for Softmax}


\section{Discrete Fourier Transform Basics}

\begin{enumerate}[label=\alph*), leftmargin=*]
%% a)
\item
%
The Fourier Transform (FT) of a sine wave at $f_{0}=20Hz$ is discrete, since the time series is periodic, comprising of Dirac $\delta$ functions at $\pm f_{0}$,
provided at figure \ref{fig:1_1_a}. Since the windowing (multiplication) operation in time domain is equivalent to convolution in the frequency domain,
the Discrete Time Fourier Transform (DTFT) of a windowed sine wave at $20Hz$ is derived by convolution of the ideal FT of the sine wave,
Dirac $\delta$ functions, with the FT of the rectangular window, $sinc$ function, illustrated at figure \ref{fig:1_1_a} for different window lengths $\tau$.
The spectrum leakage is demonstrated, as well as the trade-off between side-lobes peak and window length $\tau$ (inverse relationship).
%
\begin{figure}[h]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/discete-fourier-transform-basics/assets/a/sine-ideal-spectrum}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/discete-fourier-transform-basics/assets/a/dtft-theoretical-window}
    \end{subfigure}
    \caption{Ideal magnitude spectrum (left) and theoretical DTFT (right) of $20Hz$ sine wave.}
    \label{fig:1_1_a}
\end{figure}

%% b)
\item
%
The Discrete Fourier Transform (DFT) spectrum of a finite length sine wave at $f_{0}=20Hz$ is considered. The signal can be treated as
a windowed and sampled version of the continuous, infinite length sine wave. Consequently, its DFT is expected to be a sampled version of the DTFT shown
in figure \ref{fig:1_1_a}, at frequency intervals defined by the frequency resolution $\Delta f = \frac{F_{s}}{K}$, where $F_{s}$ and $K$ the sampling frequency
and the sequence length, respectively. In case of $K=100$, we obtain $F_{s}=1000Hz$ and $\Delta f = 10Hz$, leading to elimination of spectrum leakage or
\textbf{coherent sampling}, since at the integer multiples of the frequency resolution $10Hz$, the $sinc$ function is zero, excluding the peaks at $\pm f_{0}$.
We highlight that this is not the same with the ideal FT spectrum, but just the result of frequency quantisation. If $f_{0}$ or $F_{s}$ are slightly perturbed
then the leakage will be inevitable. When zero-padding is performed and $K=1000$, then $\Delta f = 1Hz$ and the impact of the spectrum leakage is evident.
Figure \ref{fig:1_1_b} illustrated the DFT spectra of those experiments.
%
\begin{figure}[H]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/discete-fourier-transform-basics/assets/b/dft-f0_20-K_100}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/discete-fourier-transform-basics/assets/b/dft-f0_20-K_1000}
    \end{subfigure}
    \caption{$20Hz$ sampled sine wave DFT spectra and coherent sampling.}
    \label{fig:1_1_b}
\end{figure}

%% c)
\item
%
The experiment is repeated for a finite length sine wave at $f_{1}=24Hz$ and its spectra are provided at \ref{fig:1_1_c}.
As expected this time for both $K=100$ and $K=1000$ spectral leakage is unavoidable. However, especially in case of $K=100$
when $\Delta f = 10Hz$, we note that the peaks at $\pm f_{1} = \pm 24Hz$ are undetected and therefore the sequence is
incoherently sampled. Increasing the zero-padding samples, such that the frequency resolution $\Delta f$ is a factor of $f_{1}$
resolves this issue, therefore nothing can be done for the spectral leakage effect. Candidate values for $K$ then are:
$125, \frac{1000}{6}, 250, 500, 1000, \cdots$.
%
\begin{figure}[H]
    \centering
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/discete-fourier-transform-basics/assets/c/dft-f0_24-K_100}
    \end{subfigure}
    ~
    \begin{subfigure}{0.49\textwidth}
        \centering
        \includegraphics[height=1.5in]{report/spectrum-estimation/discete-fourier-transform-basics/assets/c/dft-f0_24-K_1000}
    \end{subfigure}
    \caption{$24Hz$ sampled sine wave DFT spectra and incoherent sampling.}
    \label{fig:1_1_c}
\end{figure}
\end{enumerate}


\section{Theory}
\label{sec:fundamentals}

The perceptron is the first mathematical model of a biological neuron.
Like a neuron, it has $N$ incoming connections, which represents the input, and $1$ outgoing connection, which represents the output (i.e. the class of the given example).
The output $S$ is a function of the input $\xi$, defined as the sign of the weighted sum of the input vector:
\begin{equation}
    S = sign(\mathsf{\bm{w}} \cdotp \xi) = \pm 1,
    \label{eq:perceptron-activation}
\end{equation}
where $\mathsf{\bm{w}}$ is an $N$-dimensional vector.
The $sign$ function simulates the activation function of the biological neuron and forces the possible output values to $\pm 1$.

The training process aims at learning the optimal weights vector $\mathsf{\bm{w}}$ to classify correctly all examples.
At time $t$, an example $\xi^t$ is presented to the perceptron:
if the output equals to the correct class $S^t$, $\mathsf{\bm{w}}$ is not changed;
if the output is wrong, a learning step is performed to correct $\mathsf{\bm{w}}$.
The procedure is repeated until the perceptron correctly classifies all the training dataset or until some maximum number of iterations is reached.

The Rosenblatt Perceptron training procedure is formally defined as:
\begin{gather}
    \label{eq:perceptron-weight-update}
    \mathsf{\bm{w}}(t+1) = \mathsf{\bm{w}}(t) + \frac{1}{N} \Theta \big[c - E^{v(t)}\big] \xi^{v(t)} S^{v(t)}, \\
    \Theta[c - E^{\mu}] =  \begin{cases} \label{eq:heaviside-fun}
        1 & E^{\mu} \leq c \\
        0 & E^{\mu} > c
    \end{cases},
\end{gather}
where $E^{v(t)} = \mathsf{\bm{w}}(t) \cdot \xi^{v(t)} S^{v(t)}$.
The weights are initialized as $\mathsf{\bm{w}}(0) = 0$.
The product $\xi^{v(t)} S^{v(t)}$ is called Hebbian term.

The performances of the perceptron are related to the storage capacity of the hyperplane defined by its weights.
For a random dataset with $P$ examples and $N$ dimensions (where $S^\mu = 1$ with probability $1/2$), the theoretical probability of linear separability $P_{l.s.}(P,N)$ is given by:
\begin{equation}
    P_{l.s.}(P,N) =
    \begin{cases}
        \hfil 1 & \text{for} \; P \leq N \\
        2^{1-P} \sum^{N-1}_{i=0} \binom{P - 1}{i} & \text{for} \; P > N
    \end{cases}
    \label{eq:prob-lin-sep}
\end{equation}

For $N \rightarrow \infty$, $P_{l.s.}(P,N)$ has a well-defined behavior:
\begin{equation} \label{eq:prob-lin-sep-alpha}
    P_{l.s.}(P,N) =
    \begin{cases}
        1 & \text{for} \; \alpha \leq 2 \\
        0 & \text{for} \; \alpha > 2
    \end{cases},
\end{equation}
where $\alpha = P / N$.


\chapter{The Tensor Product}
Consider the Cartesian product of a vector space $V$ and itself, $V \times V$. A \emph{tensor}~$T$ is a \emph{multilinear map} from this product and the field $\mathbb{F}$ over which the vector spaces are taken, where multilinear simply means that the map is linear in each component of the domain. Explicitly, it acts in the following way:
\[ T : V \times V \to \mathbb{F} : (\vec{v}, \vec{w}) \mapsto \langle \Lambda, \vec{v} \rangle\langle \Gamma, \vec{w} \rangle. \]
Since we already have tools which map vectors in $V$ to $\mathbb{F}$ in a linear fashion --- covectors from the dual space --- we define a tensor by using combinations of these covectors using the \emph{tensor product}~$\otimes$.
Consider the Cartesian produt $V \times V$; if we want to send the ordered pair $(\vec{v}, \vec{w})$ to $\mathbb{F}$, we take two covectors, $\beta$ and $\alpha$, and take their tensor product to get
\[ \beta \otimes \alpha : V \times V \to \mathbb{F} : (\vec{v},\vec{w}) \mapsto \langle \beta, \vec{v} \rangle\langle \alpha, \vec{w} \rangle. \]
Several things:
\begin{itemize}
    \item First, notice that the result of multilinear tensors are multiplied together, where the multiplication is the multiplication operation defined for that field.
    \item This is only a single type of tensor --- there are many others, including regular covectors and regular vectors.
    \item This tensor is an element of $V^* \times V^*$, while it takes an argument of $V \times V$.
\end{itemize}

\section{Addition \& the Tensor Product Space}
Consider again the vector space $V$ and its dual space $V^*$. Let's consider two tensors, $\alpha \otimes \beta$ and $\gamma \otimes \delta$, both elements of $V^* \times V^*$.
Since tensors are multilinear maps, we want tensor addition to act linearly.
We define tensor addition to act as follows:
\begin{align*}
    [\alpha \otimes \beta + \gamma \otimes \delta](\vec{v},\vec{w}) &= \alpha\otimes\beta (\vec{v},\vec{w}) + \gamma\otimes\delta(\vec{v},\vec{w}) \\
    &= \langle \alpha, \vec{v} \rangle\langle \beta, \vec{w} \rangle + \langle \gamma, \vec{v} \rangle\langle \delta,\vec{w} \rangle
\end{align*}
Likewise with addition, if we introduce scalar factors from $\mathbb{F}$, we define tensors such that
\[\lambda_1\alpha\otimes\lambda_2\beta(\vec{v},\vec{w}) = \lambda_1\lambda_2\langle \alpha, \vec{v} \rangle\langle \beta, \vec{w} \rangle.\]
Since the components of a tensor are members of the dual space, if we choose them to be the zero maps, we have created the `zero tensor'.
This, combined with the above two facts, means that the set of all possible tensors of a certain number of components (called rank) form a vector space.
This space is called the \emph{tensor product space}, and since all elements of vector spaces are vectors, this means that tensors are vectors \emph{and} vectors are tensors!\footnote{It's vector spaces all the way down!}

\section{Review \& Formalism}
We begin with a vector space $V$ over a field $\mathbb{F}$ with a basis $\{\vec{e}_\nu\}$, which implies the existence of a dual space $V^*$ with a basis $\{\vec{e}^\mu\}$.
This gives us a special type of tensor product space, $ V^* \otimes V^*$, whose elements are bilinear in $V \times V$.
Because $V^* \otimes V^*$ is a vector space, it has addition and scalar multiplication, and is also taken over the same field $\mathbb{F}$.
We select the basis of the vector and dual space such that
\[ \langle \vec{e}^\mu, \vec{e}_\nu \rangle = \delta^\mu_\nu. \]
Since the tensor product space is a vector space, it too has a basis.
As tensors in $V^* \otimes V^*$ are composed of linear maps from $V^*$, the basis for the tensor product space is $\vec{e}^\mu \otimes \vec{e}^\nu$.
Note that, in this basis, the index $\mu$ does not always equal $\nu$.
These tensors can be defined by
\[ A_\mu \vec{e}^\mu \otimes B_\nu \vec{e}^\nu : V \times V \to \mathbb{F} : (C^\alpha \vec{e}_\alpha, D^\beta \vec{e}_\beta) \mapsto A_\mu C^\mu B_\nu D^\nu. \]
Note that if our original vector space as four dimensional, the tensor product space will have dimension $4 \times 4 = 16$, which means that there are 16 basis vectors encapsulated by $\vec{e}^\mu \otimes \vec{e}^\nu$.
Because of the linearity of tensors, we can simplify the way we write a tensor into a single coefficient, using multiple indicies:
\[ A_\mu \vec{e}^\mu \otimes B_\nu \vec{e}^\nu \equiv T_{\mu\nu} \vec{e}^\mu \otimes \vec{e}^\nu. \]
Because Physicists and Mathematicians are fundamentally lazy people, and more importantly because \emph{any} tensors from the same tensor product space specified with the same vectors will differ only by the coefficient, we often adopt the convention of dropping the basis when discussing a specific tensor, and refer to it only by this coefficient; the above tensor would be called $T_{\mu\nu}$. These leading coefficients are often called the tensor's \emph{components}.


\subsection{The binomial data generating process}

\subsubsection{Introduction}

For linear regression our data generating process is:

\(y=\alpha + \beta x +\epsilon \)

For linear classification our data generating process is:

\(z=\alpha + \beta x +\epsilon \)

And set \(y\) to \(1\) if \(z>0\)

Or:

\(y=\mathbf I[\alpha+\beta x+\epsilon >0]\)

\subsubsection{Probability of each class}

The probability that an invididual with characteristics \(x\) is classified as \(1\) is:

\(P_1=P(y=1|x)\)

\(P_1=P(\alpha + \beta x+\epsilon >0)\)

\(P_1=\int \mathbf I [\alpha + \beta x+\epsilon >0]f(\epsilon )d\epsilon \)

\(P_1=\int \mathbf I [\epsilon >-\alpha-\beta x ]f(\epsilon )d\epsilon \)

\(P_1=\int_{\epsilon=-\alpha-\beta x}^\infty f(\epsilon )d\epsilon \)

\(P_1=1-F(-\alpha-\beta x) \)

\subsubsection{Example: The logistic function}

Depending on the probability distribution of \(epsilon \) we have different classifiers.

For the logistic case we then have

\(P(y=1|x)=\dfrac{e^{\alpha + \beta x}}{1+e^{\alpha + \beta x}}\)


\section{Important probability distributions}

\subsection*{Table}

\begin{figure*}
\resizebox{\linewidth}{!}{%
\begin{tabular}{ l  l  l  l l  l  l l  l l}

 Dist & Param & PDF & CDF & Mean & var & F($\theta$) & MLE & Likeleyhood $= \displaystyle  L_ n(X_1, \ldots , X_ n, \theta )$  & Log likelyhood \\
\hline

 %BERN
 Ber(p) coin flip
 &
 $p \in[0,1]$
 &
 $ p_x(k)=	\begin{cases} p,&\text{if k = 1}\\(1-p),&\text{if k = 0} \end{cases}$
 &
 None
 &
 p
 &
 p(1-p)
 &
 $I(p) = \frac{1}{p(1-p)}$
 &
 $\hat{p}_{MLE} = \frac{\sum^n_{i=1}(X_i)}{n}$
 &
 $ p^{\sum _{i = 1}^ n X_ i} (1 -p)^{{\color{blue}{n - }} \sum _{i = 1}^ n X_ i}$
 &
 $\ell_n (p)  = \ln  \left( p \right) \sum _{i=1}^{n}X_{{i}}+ \left( n-\sum _{i=1}^{n}X_{{i}} \right) \ln  \left( 1-p \right)$
 \\\hline

 %BINOM
 Binom(p) (n coin flips)
 &
 $p \in[0,1]$
 &
 $p_x(k) = {\binom{n}{k}} {p}^{k} (1-p )^{n-k}$, $k=0...n$
 &
 None
 &
 $n*p$
 &
 $n p(1-p)$
 &
 $I(p) = \frac{n}{p(1-p)}$
 &
 MLE
 &
 $\displaystyle   \left( \prod _{i = 1}^ n \binom {K}{X_ i} \right) \theta ^{\sum _{i = 1}^ n X_ i} (1 - \theta )^{nK - \sum _{i = 1}^ n X_ i }$
 &
 $\ell_n (\theta) = C + \left( \sum _{i = 1}^ n X_ i \right) \log \theta + \left( nK - \sum _{i = 1}^ n X_ i \right) \log (1 - \theta )$
 \\\hline

 $Geometric(p) \\ T trials for 1st success. $
 &
 $p \in[0,1]$
 &
 $p_T(t) = (1-p)^{t-1}, t=1,2..$
 &
 None
 &
 $\frac{1}{p}$
 &
 $\frac{1-p}{p^2}$
 &
 Fisher
 &
 MLE
 &
 LIKEHOD
 &
 LOG LIKEH

 \\\hline

 $ Multinomial(p_0...p_r,n) $
 &
 $n>0$ and $p_1, \ldots, p_r$
 &
 $p_x(x_1...x_n)=\frac{n!}{x_1!,\ldots,x_n!} p_1, \ldots, p_r$
 &
 None
 &
 $n p_i$
 &
$n * p_i(1-p_i)$
 &
 Fisher
 &
 MLE
 &
 \thead{$p_x(x)= \prod _{j=1}^{n}{p_{{j}}}^{T_{{j}}}$, where $T^j=\mathbbm{1}( X_i=j)$  \\ is the count how often an outcome is seen in trials.}
 &
 $\ell_n= \sum _{j=2}^{n}T_{{j}}\ln  \left( p_{{j}} \right)$

 \\\hline

 Poisson($\lambda$)  Discrete
 &
 $\lambda > 0$
 &
 $\mathbf{p_x}(k)=exp(-\lambda)\frac{\lambda^k}{k!}$ for $k=0,1, \ldots,$
 &
 CDF
 &
 $\lambda$
 &
 $\lambda$
 &
 $I(\lambda)= \frac{1}{\lambda}$
 &
 $\hat{\lambda}_{MLE} = \frac{1}{n} \sum^n_{i=1}(X_i)$
 &
 $ \prod _{i = 1}^ n \frac{\lambda^{\sum_{i=1}^{n} x_i}}{\prod _{i = 1}^ n x_i!} e^{-n\lambda}$
 &
 $\ell_n (\lambda)=  -n\lambda + log(\lambda)(\sum_{i=1}^n x_i)) - log(\prod _{i = 1}^ n x_i!)$

 \\\hline

 Exp(\lambda)  Dist
 &
 $\lambda$ >0
 &
 $ f_x(x)=
	\begin{cases}
		 \lambda exp(-\lambda x),&\text{if x >= 0}\\
		0,&\text{o.w.}\\
	\end{cases}
 $
 &
 $ \begin{cases}
		1-exp(-\lambda x),{if x >= 0}\\
		0,\quad{o.w.}\\
		P(X>a)= exp(-\lambda a)
	\end{cases}
 $
 &
 $\frac{1}{\lambda}$
 &
 $\frac{1}{\lambda^2}$
 &
 $I(\lambda)= \frac{1}{\lambda^2}$
 &
 $\hat{\lambda}_{MLE}= \frac{n}{\sum^{n}_{i=1}(X_i)}$
 &
 $\lambda^n\exp\left(-\lambda\sum_{i=1}^n X_i\right)$
 &
 $\ell_n (\lambda)= n ln(\lambda) - \lambda \sum_{i=1}^n (X_i)$
 \\\hline

 Shifted Exp (b,a) Dist
 &
 Parameters $\lambda, a \in \mathbb{R}$
 &
 $ f_x(x)=
	\begin{cases}
		 \lambda exp(-\lambda(x - a )),&{x >= a}\\
		0,&{x <= a}\\
	\end{cases}
$

 &
 $ 	\begin{cases}
		 1-exp(-\lambda(x-a)),\quad{if x >= a}\\
		0,\quad{x <= a}\\
	\end{cases}
$
 &
 $a + \frac{1}{\lambda}$
 &
 $\frac{1}{\lambda^2}$
 &
 F($\theta$)
 &
 $ \begin{cases}
 {\lambda } = \frac{1}{\overline{X}_ n - \hat{a}}\\
 {a} = \min(X_ i)
 \end{cases}
 $
 &
 $ \lambda ^ n \exp \left( -\lambda \sum _{i = 1}^ n (X_ i - a) \right) \mathbf{1}_{\min _{i = 1, \ldots , n}(X_ i) \geq a}.$
 &
 $\ell (\lambda , a) := n \ln \lambda - \lambda \sum _{i = 1}^ n X_ i + n \lambda a$
 \\\hline

  Uniform(a,b)
 &
 $a$ and $b$
 &
 $ \mathbf{f_x}(x)=
	\begin{cases}
		 \frac{1}{b-a},&\text{if a < x <b}\\
		0,&\text{o.w.}\\
	\end{cases}
$
 &
 $ \mathbf{F_x}(x)=
	\begin{cases}
		 0,&for x \leq a\\
		 \frac{x-a}{b-a},& x \in [a,b)\\
		1,&x \geq b\\
	\end{cases}
$
 &
 $\frac{a+b}{2}$
 &
 $\frac{(b-a)^2}{12}$
 &
 F($\theta$)
 &
 MLE
 &
 $\frac{\max_i (x_i \leq b)} {b^n}$
 &
 $ln(\frac{\max_i (x_i \leq b)} {b^n})$
 \\\hline

 Normal(\mu, \sigma)  Dist
 &
 $\mu$ and $\sigma^2 >0$
 &
 $f(x)= \frac{1}{\sqrt(2 \pi \sigma^2)} exp(-\frac{(x-\mu)^2}{2\sigma^2})$
 &
 $\Phi (z) = \int _{-\infty }^ z \frac{1}{\sqrt{2 \pi }} e^{-x^2/2} \,  dx$
 &
 $\mu$
 &
 $\sigma^2$
 &
 $I(\mu , \sigma ^2) = \begin{pmatrix}  \frac{1}{\sigma ^2} &  0 \\ 0 &  \frac{1}{2 \sigma ^4} \end{pmatrix}$
 &
 $\hat\mu = \bar X_ n, \quad \widehat{\sigma ^2} = \frac{1}{n} \sum _{i=1}^{n} (X_ i - \bar X_ n)^2$
 &
 $ \dfrac{1}{\left(\sigma\sqrt{2\pi}\right)^n}\exp{\left(-\dfrac{1}{2\sigma^2}\sum_{i=1}^n (X_i-\mu)^2 \right)}$
  &
  $\ell_n (\mu,\sigma^2)= -n log(\sigma\sqrt{2\pi})-\frac{1}{2\sigma^2}\sum_{i=1}^n (X_i-\mu)^2 $
  \\\hline

 Hypergeom  Dist
 & Param & PDF & CDF & Mean & var & F($\theta$) & MLE & Likeleyhood & Log likelyhood  \\\hline

 Negative Binom  Dist  & Param & PDF & CDF & Mean & var & F($\theta$) & MLE & Likeleyhood & Log likelyhood  \\\hline

 Beta  Dist  & Param & PDF & CDF & Mean & var & F($\theta$) & MLE & Likeleyhood & Log likelyhood  \\\hline

\end{tabular}
} %resize End

\end{figure*}

\subsection*{Univariate Gaussians}

Canonical exponential form:\\

Gaussians are invariant under affine transformation:\\

$aX+b \sim N(X+b,a^2\sigma^2)$\\

Sum of independent gaussians:\\

Let $X {\sim} N(\mu_X,\sigma_X^2)$ and $Y {\sim} N(\mu_Y,\sigma_Y^2)$\\

If $Y = X + Z$, then $Y \sim N(\mu_X + \mu_Y, \sigma_X + \sigma_Y)$\\

If $U = X - Y$, then $U \sim N(\mu_X - \mu_Y,\sigma_X + \sigma_Y)$\\

Symmetry:\\

If $X \sim\ N(0,\sigma^2),$ then $-X \sim N(0,\sigma^2)$\\

$\mathbb{P}(|X|>x) = 2\mathbb{P}(X>x)$\\

Standardization:\\

$Z= \frac{X-\mu}{\sigma} \sim N(0,1)$\\

$\mathbf{P}\left(X\leq t\right) = \displaystyle \mathbf{P}\left(Z\leq \frac{t-\mu}{\sigma}\right)$

Higher moments:\\

$\mathbb{E}[X^2] = \mu^2 + \sigma^2$\\
$\mathbb{E}[X^3] = \mu^3 + 3\mu\sigma^2$\\
$\mathbb{E}[X^4] = \mu^4 + 6\mu^2\sigma^2 +3\sigma^4$\\

Quantiles:\\

\subsection*{Uniform}

\subsection*{Cauchy}
continuous, parameter $m$,

$f_ m(x) = \frac{1}{\pi } \frac{1}{1 + (x - m)^2}$\\

$\mathbb{E}[X]=not defined!$\\
$Var(X)=not defined!$\\

$\text {med}(X) = P(X > M) = P(X < M)\\ = 1/2 = \displaystyle \int _{1/2}^{\infty } \frac{1}{\pi } \cdot \frac{1}{1 + (x-m)^2} \,  dx$

\subsection*{Chi squared}
The $\chi _ d^2$ distribution with $d$ degrees of freedom is given by the distribution of $Z_1^2 + Z_2^2 + \cdots + Z_ d^2,$ where $Z_1, \ldots , Z_ d \stackrel{iid}{\sim } \mathcal{N}(0,1)$

If $V \sim \chi^2_k:$\\

$\mathbb{E}= \mathbb{E}[Z_1^2] + \mathbb{E}[Z_2^2] + \ldots + \mathbb{E}[Z_d^2] = d$\\

$Var(V) = Var(Z_1^2) + Var(Z_2^2) + \ldots + Var(Z_d^2) = 2d$

\subsection*{Student's T Distribution}

$T_ n := \frac{Z}{\sqrt{V/n}}$ where $Z \sim \mathcal{N}(0,1)$, and $Z$ and $V$ are independent

\subsection*{Poisson}
Parameter $\lambda$. discrete, approximates the binomial PMF when $n$ is large, $p$ is small, and $\lambda = np$.\\

Canonical exponential form:\\

$ f_{\theta}(y) = \exp\big(y\theta - \underbrace{e^\theta}_{b(\theta)} \underbrace{- \ln y!}_{c(y, \phi)}\big)$\\
$\theta = \ln \lambda$\\
$\phi = 1$\\

Poisson process:\\
k arrivals in t slots
$\mathbf{p_x}(k,t) = \mathbb{P}(N_t=k)=e^{-\lambda t} \frac{(\lambda t)^k}{k!}$\\

$\mathbb{E}[N_t]=\lambda t$\\

$Var(N_t)=\lambda t$

\subsection*{Exponential}
$\mathbb{E}[X^2]=\frac{2}{\lambda^2}$
Canonical exponential form:\\
$f_{\theta}(y) = \exp\big(y\theta - \underbrace{(-\ln(-\theta))}_{b(\theta)} + \underbrace{0}_{c(y, \phi)}\big)$\\
$\theta = -\lambda = -\frac1{\mu}$\\
$\phi = 1$

\subsection{Bern}
Canonical exponential form:\\
$f_{\theta}(y)=\exp\big(y\theta - \underbrace{\ln(1 + e^\theta)}_{b(\theta)} + \underbrace{0}_{c(y, \phi)}\big) \quad$\\
$\theta = \ln\left(\frac{p}{1-p}\right)$\\
$\phi = 1$\\

\subsection*{Binomial}
Canonical exponential form:\\
$f_ p(y) =\\
exp (y \underbrace{(\ln (p)-\ln (1-p))}_{\theta } + \underbrace{n\ln (1-p)}_{-b(\theta )} +\underbrace{\ln(\binom {n}{y})}_{c(y,\phi )} )$

\subsection*{Pascal}

The negative binomial or Pascal distribution is a generalization of the geometric distribution. It relates to the random experiment of repeated independent trials until observing $m$ successes. I.e. the time of the kth arrival.

$Y_k=T_1+...T_k$\\

$T_i \sim iid Geometric(p)$\\

$\mathbb{E}[Y_k]=\frac{k}{p}$\\

$Var(Y_k)= \frac{k(1-p}{p^2}$

$p_{Y_k}(t) ={\binom{t-1}{ k-1}}p^k(1-p)^{t-k}$, $t=k,k+1,...$

\subsection*{Shifted Exponential}


\subsection{Introduction}

The outcome of a Bernoulli trial is either \(0\) or \(1\). We can describe it as:

\(P(1)=p\)

\(P(0)=1-p\)

With a single parameter \(p\).

\subsection{Moments of the Bernoulli distribution}

The mean of a Bernoulli trial is \(E[X]=(1-p)(0)+(p)(1)=p\).

The variance of a Bernoulli trial is \(E[(X-\mu)^2]=(1-p)(0-\mu)^2+(p)(1-\mu)^2=(1-p)p^2+p(1-p)^2]=p(1-p)\).