\chapter{Linear Regression}

\section{Introduction}
Linear regression is the “work horse” of statistics and (supervised) machine learning. When augmented with kernels or other forms of basis function expansion, it can model also nonlinear relationships. And when the Gaussian output is replaced with a Bernoulli or multinoulli distribution, it can be used for classification, as we will see below. So it pays to study this model in detail.

In the simplest approach,we can directly construct an appropriate function $y(\vec{x})$ whose values for new inputs $\vec{x}$ constitute the predictions for the corresponding values of $t$.More generally,from a probabilistic perspective,we aim to model the predictive distribution $p(t|x)$ because this expresses the uncertainty about the value of $t$ for each value of $x$.

\section{Representation}
The simplest linear model for regression is \textbf{linear regression} that involves a linear combination of the input variables
\begin{equation}
y(\vec{x},\vec{w}) = w_0+w_1x_1+...+w_Dx_D
\end{equation}
where $\vec{x}=(x_1,...,x_D)^T$.

Linear regression can be made to model non-linear relationships by replacing $\vec{x}$ with some non-linear function of the inputs, $\phi(\vec{x})$.Consider the linear combinations of fixed nonlinear functions of the input variables,of the form
\begin{equation}
y(\vec{x},\vec{w}) = w_0+\sum_{j=1}^{M-1}w_j\phi_j(x)
=\sum_{j=0}^{M-1}w_j\phi_j(x) = \vec{w}^T\phi(x)
\end{equation}
where $\phi_j(x)$ are known as \textbf{basis functions},
$\vec{w}=(w_0,...,w_{M-1})^T$ and $\phi=(\phi_0,...,\phi_{M-1})^T$.
This is known as \textbf{basis function expansion}. (Note that the model is still linear in the parameters $\vec{w}$, so it is still called linear regression; the importance of this will become clear below.) A simple example are polynomial basis functions, where the model has the form
\begin{equation}
\phi(x)=(1, x, \cdots, x^d)
\end{equation}

As before,we assume that the target variable is given by a deterministic function $y(\vec{x},\vec{w})$ with additive Gaussian noise so that
\begin{equation}
t=y(\vec{x},\vec{w})+\epsilon
\end{equation}
where $\epsilon$ is a zero mean Gaussian random variable with precision(inverse variance) $\beta$.Thus
\begin{equation}\label{eqn:linear regression representation}
p(t|\vec{x},\vec{w},\beta)=\mathcal{N}(t|h(\vec{x},\vec{w}),\beta^{-1})
\end{equation}
or
\begin{eqnarray}
& y(\vec{x}) = \vec{w}^T\vec{x}+\epsilon \\
& p(y|\vec{x},\vec{\theta})=\mathcal{N}(y|\vec{w}^T\vec{x}, \sigma^2) \\
\end{eqnarray}
where $\vec{w}$ (\textbf{weight vector}) and $\vec{x}$ are extended vectors, $\vec{x}=(1,x)$, $\vec{w}=(b,w)$ and $\epsilon$ has a \textbf{Gaussian} or \textbf{normal} distribution.$w_0$ is the intercept or \textbf{text}

%\begin{eqnarray}
%& y(\vec{x}) = \vec{w}^T\phi(\vec{x})+\epsilon \\
%& p(y|\vec{x},\vec{\theta})=\mathcal{N}(y|\vec{w}^T\phi(\vec{x}), \sigma^2)
%\end{eqnarray}

\section{Maximum likelihood estimations(least squares)}
Assume the training examples are \textbf{independently and identically distributed(IID)},we obtain the \textbf{likelihood function},which is a function of adjustable parameters $\vec{w}$ and $\beta$,in the form
\begin{align}\label{eqn:linear regression likelihood}
p(\vec{t}|\vec{X},\vec{w},\beta) =
\prod_{n=1}^{N}\mathcal{N}(t_n|\vec{w}^T\phi(\vec{x_n}),\beta^{-1})
\end{align}
we can write the \textbf{log-likelihood} (logarithm of the likelihood) function as follows:
\begin{equation}
\ell(\vec{\theta}) \triangleq \log p(\mathcal{D}|\vec{\theta})
\end{equation}
A common way to estimate the parameters of a statistical model is to compute the MLE(Maximum likelihood estimations),which is defined as
\begin{equation}
\vec{\hat{\theta}}=\arg\max_\theta{\log p(\mathcal{D}|\vec{\theta})}
\end{equation}

Instead of maximizing the log-likelihood, we can equivalently minimize the \textbf{negative log likelihood} or \textbf{NLL}:
\begin{equation}
\text{NLL}(\vec{\theta}) \triangleq -\ell(\vec{\theta})=-\log p(\mathcal{D}|\vec{\theta})
\end{equation}

The NLL formulation is sometimes more convenient, since many optimization software packages are designed to find the minima of functions, rather than maxima.

Now let us apply the method of MLE to the linear regression setting. Inserting the definition of the Gaussian into the above, we find that the log likelihood is given by
\begin{align}
\ell(\vec{\theta}) &= \log p(\vec{T}|\vec{w},\beta) \\
&=\sum_{n=1}^{N}\log \mathcal{N}(t_n|\vec{w}^T\phi(\vec{x_n}),\beta^{-1}) \\
&=\sum\limits_{i=1}^N \log\left[\dfrac{1}{\sqrt{2\pi}\sigma}\exp\left(-\dfrac{1}{2\sigma^2}(y_i-\vec{w}^T\vec{x}_i)^2\right)\right] \\
&=\dfrac{N}{2}\log \beta-\dfrac{N}{2}\log (2\pi)-\beta E_D(\vec{w}) \\
%&=-\dfrac{1}{2\sigma^2}\text{RSS}(\vec{w})-\dfrac{\vec{w}}{2}\log(2\pi\sigma^2)
\end{align}
where the sum-of-squares error function is defined by
\begin{equation}
E_D(\vec{w}) =
\dfrac{1}{2}\sum_{n=1}^{N}\{t_n-\vec{w}^T\phi(\vec{x_n}) \}^2
\end{equation}

There multiple ways to compute the optimal solution of likelihood function: in matrix calculus, sum of terms derivatives, geometry interpretation.

\subsection{Derivations of the MLE}
Denote $\vec{t}=(t_1,t_2,\cdots,t_N)^T$, $\vec{\Phi}=\left(\begin{array}{c}\vec{\phi}(x_1)^T \\ \vec{\phi}(x_2)^T \\ \vdots \\ \vec{\phi}(x_N)^T\end{array}\right)$.
$\vec{\Phi}$ is an $N\times M$ matrix,called the \textbf{design matrix}, whose elements are given by $\Phi_{nj}=\phi_j(\vec{x_n})$ so that
\begin{equation}
\vec{\Phi}=
\begin{bmatrix}
\phi_0(\vec{x_1}) &	\phi_1(\vec{x_1}) &...&\phi_{M-1}(\vec{x_1}) \\
\phi_0(\vec{x_2}) & \phi_1(\vec{x_2}) &...&\phi_{M-1}(\vec{x_2}) \\
\vdots &\vdots &\vdots&\vdots \\
\phi_0(\vec{x_N})& \phi_1(\vec{x_{N}})& \cdot &\phi_{M-1}(\vec{x_N})\\
\end{bmatrix}
\end{equation}

Note that the sum of terms can be expressed in matrix/vector multiplication.
\begin{align}
\because \sum_{n=1}^{N}t_n\phi(\vec{x}_n)^T &=\vec{t}^T\vec{\Phi}
\end{align}
\begin{align}
\begin{cases}
\vec{\Phi} = \begin{bmatrix}
&\vec{\Phi_1^T} \\
&\vec{\Phi_2^T} \\
& ... \\
&\vec{\Phi_n^T}
\end{bmatrix} \\
\vec{\Phi^T}\vec{\Phi} =
\begin{bmatrix}
\vec{\Phi_1} & \vec{\Phi_2} & ...&\vec{\Phi_n}
\end{bmatrix}
\cdot
\begin{bmatrix}
&\vec{\Phi_1^T} \\
&\vec{\Phi_2^T} \\
& ... \\
&\vec{\Phi_N^T}
\end{bmatrix}  = \sum\limits_{i=1}^{N}\vec{\phi_i}\vec{\phi_i^T}\\
\end{cases}
\end{align}

We see that maximizing the likelihood function under a conditional Gaussian noise distribution for a linear model is equivalent to $\vec{w}$ minimizing the sum-of-squares error function given by $E_D$, so this method is known as \textbf{least squares}.The gradient of the log likelihood function takes the form
\begin{equation}
\nabla\log p(\vec{t}|\vec{w},\beta)
=\sum_{n=1}^{N}\{t_n-\vec{w}^T\phi(\vec{x_n}) \}\phi(\vec{x_n})
\end{equation}
Setting the gradient respect to $\vec{w}$ to zero gives
\begin{align}
\nabla\log E_D &= 0 \\
\Rightarrow\log E_D^T &= 0 \\
\Rightarrow \sum_{n=1}^{N}\{t_n-\vec{w}^T\phi(\vec{x_n}) \}\phi(\vec{x_n})^T &= 0 \\
\Rightarrow \sum_{n=1}^{N}t_n\phi(\vec{x}_n)^T-\vec{w}^T(\sum_{n=1}^{N}\vec{\phi}(\vec{x}_n)\vec{\phi}(\vec{x}_n)^T) &= 0 \\
\Rightarrow \vec{t}^T \vec{\Phi} - \vec{w}^T\vec{\Phi}^T \vec{\Phi} &= 0
\end{align}
Solving for $\vec{w}$ we obtain
\begin{align}
\vec{w}_{ML}=\hat{\vec{w}}_{\mathrm{OLS}}=(\vec{\Phi}^T\vec{\Phi})^{-1}\vec{\Phi}^T\vec{t}
\end{align}
The corresponding solution $\hat{\vec{w}}_{\mathrm{OLS}}$ to this linear system of equations is called the \textbf{ordinary least squares} or \textbf{OLS} or \textbf{normal equation} solution. When $\mathcal{D}$ is small(for example, $N < 1000$), we can use the equation to compute $\vec{w}$ directly.
Make the bias parameter explicit,the error function
\begin{align}
E_D(\vec{w})=\dfrac{1}{2}\sum_{n=1}^{N}\{t_n-w_0-\sum_{j=1}^{M-1}w_j\phi_j(\vec{x}_n)\}^2
\end{align}
Setting the derivative with respect to $w_0$ equal to zero,we obtain
\begin{align}
w_0 = \bar{t}-\sum_{j=1}^{M-1}w_j\bar{\phi_j}
\end{align}
Then we have
\begin{align}
y&=\vec{w}^T\vec{\phi} \\
y&=\sum_{j=1}^{M}w_j \phi_j+w_0 \\
y&=\sum_{j=1}^{M}w_j \phi_j+\bar{t}-\sum_{j=1}^{M-1}w_j\bar{\phi_j} \\
y-\bar{y}&=\sum_{j=1}^{M}w_j(\phi_j-\bar{\phi}_j)
\end{align}
which indicates that we can normalize the features by subtracting the mean of $\vec{\Phi}$ while training to reduce computation complexity but obtain the same weight parameter.

\subsubsection{Matrix Calculus}
With regard to matrix calculus, when performing \textbf{gradient} based updates, we represent matrix derivatives in \textbf{denominator layout}. Thus the \textbf{dimension} of the matrix, which we are computing derivatives with respect to, will be consistent with its gradient matrix. Denominator layout convention will make it convenient for use to compute complex derivatives with chain rule and use the result to update old parameters directly, without transposing operation.

There are multiple ways to derive matrix differential calculus:
\begin{enumerate}
\item From differential definition, the limit of $\frac{df}{d\vec{x}}$ when $dx$ approaches 0. Take the difference $df = f(\vec{x} + d\vec{x}) - f(\vec{x})$, and compute $\lim_{\vec{h}\to 0} \frac{df}{d\vec{x}}$.
\item According to matrix multiplication definition, we can expand matrix cell to \textbf{sum of terms}. Then use 1 dimensional calculus, finally, rewrite to matrix form.
\item Construct \textbf{full derivative} from a sum of of \textbf{partial derivatives}. In this way, we can decompose a complex formula into several simple ones.
\end{enumerate}

We now state without proof some facts of matrix calculus (we won’t need all of these at this section).
\begin{eqnarray}
\frac{\partial}{\partial A}AB &=& B^T \\
\frac{\partial}{\partial A^T}f(A) &=& \left[\frac{\partial}{\partial A}f(A)\right]^T \label{eqn:matrix-1} \\
\frac{\partial}{\partial A}ABA^TC &=& CAB+C^TAB^T \label{eqn:matrix-2} \\
\frac{\partial}{\partial A}|A| &=& |A|(A^{-1})^T \\
trA &\triangleq& \sum\limits_{i=1}^n A_{ii} \nonumber
\end{eqnarray}

To see why $ \frac{\partial \vec{A}\vec{B}}{\partial \vec{A}} = \vec{B}^T $, we use dimension analysis. Denote $ f = f(\vec{C})$. Assume $\vec{A}$ is of dimension $m\times n$, $\vec{B}$ is of dimension $n\times p$, and of course, $\vec{C}$ is of dimension $n\times p$. Take the derivative of $f$ with respect to $\vec{A}$:
\begin{align}
\underbrace{\frac{\partial f}{\partial \vec{A}}}_{m\times n} &= \underbrace{\frac{\partial f}{\partial \vec{C}}}_{m \times p} \underbrace{\frac{\partial \vec{C}}{\partial \vec{A}}}_{p\times n} \\
\end{align}
And $\vec{B}^T$ is of size $p\times n$, the whole function mapping is linear. Also, we can explicitly write matrix elements in sum of terms to show that.

Another $\mathbf{vector-vector}$ differentiation identity, if $\mathbf{A}$ is not a function of $\vec{x}$ then
\begin{eqnarray}
\dfrac{d\vec{X}^TA\vec{X}}{d\vec{X}} \\
&=& \dfrac{d\vec{Y}^TA\vec{X}}{d\vec{X}} \\
&=& \dfrac{\partial\vec{Y}^TA\vec{X}}{\partial\vec{Y}} \dfrac{\partial\vec{Y}}{\partial\vec{X}} +  \dfrac{\partial\vec{Y}^TA\vec{X}}{\partial\vec{X}} \dfrac{\partial\vec{X}}{\partial\vec{X}}\\
&=& \dfrac{\partial \vec{X}^T \vec{A}^T\vec{Y}^T}{\partial\vec{Y}} \vec{I} +  \dfrac{\partial\vec{Y}^TA\vec{X}}{\partial\vec{X}} \vec{I} \\
&=& \vec{X}^T \vec{A}^T + \vec{Y}^T\vec{A}\\
&=& \vec{X}^T \vec{A}^T + \vec{X}^T\vec{A}\\
&=& \vec{X}^T(\vec{A}+\vec{A}^T)\\
&=& (\vec{A}+\vec{A}^T)\vec{X} \text{, In denominator layout}
\end{eqnarray}

Making use of \textbf{partial derivative}, \textbf{chain rule} still holds.

\subsubsection{Derivation directly from Matrix Calculus}
The previous derivation is based on the sum of terms arithmetic. Now we derive it from matrix calculus.

\begin{proof}
Let's drop constants with respect to $\vec{w}$ and NLL can be written as
\begin{eqnarray*}
\text{NLL}(\vec{w})
&=&\dfrac{1}{2}\sum\limits_{i=1}^N (\vec{t}_i-\vec{w}^T\vec{\phi}_i)^2 \\
&=&\dfrac{1}{2}(\vec{\Phi}\vec{w}-\vec{t})^T(\vec{\Phi}\vec{w}-\vec{t}) \\
&=&\frac{1}{2} (\vec{w}^T\vec{\Phi}^T\vec{\Phi}\vec{w}-\vec{w}^T\vec{\Phi}^T\vec{t}-\vec{t}^T\vec{\Phi}\vec{w}+\vec{t}^T\vec{t}) \\
&=&\frac{1}{2} (\vec{w}^T\vec{\Phi}^T\vec{\Phi}\vec{w}-\vec{w}^T\vec{\Phi}^T\vec{t}-\vec{t}^T\vec{\Phi}\vec{w}) \\
&=&\dfrac{1}{2} \vec{w^T} (\vec{\Phi}^T\vec{\Phi})\vec{w} -\vec{w}^T(\vec{\Phi}^T\vec{t}) \\
\Rightarrow
\frac{\partial \text{NLL}}{\partial\vec{w}} &=& \frac{1}{2} ((\vec{\Phi}^T\vec{\Phi}+\vec{\Phi}^T\vec{\Phi})\vec{w} -2\vec{\Phi}^T\vec{t}) \nonumber \\
&=& \frac{1}{2} (\vec{\Phi}^T\vec{\Phi}\vec{w} - \vec{\Phi}^T\vec{t}) \nonumber \\
\frac{\partial \text{NLL}}{\partial\vec{w}} &=& 0 \\
\Rightarrow \vec{\Phi}^T\vec{\Phi}\vec{w} - \vec{\Phi}^T\vec{t} &=& 0 \nonumber \\
\Rightarrow \vec{\Phi}^T\vec{\Phi}\vec{w} &=& \vec{\Phi}^T\vec{t} \label{eqn:normal-equation} \\
\Rightarrow \vec{w}}_{\mathrm{ML} &=& (\vec{\Phi}^T\vec{\Phi})^{-1}\vec{\Phi}^T\vec{t} \nonumber
\end{eqnarray*}

Equation \ref{eqn:normal-equation} is known as the \textbf{normal equation}.
\end{proof}

\subsection{Geometric interpretation}

See Figure \ref{fig:graphical-interpretation-of-OLS}.
\begin{figure}[hbtp]
\centering
    \includegraphics[scale=.50]{graphical-interpretation-of-OLS.png}
\caption{Graphical interpretation of least squares for $N=3$ examples and $D=2$ features. $\tilde{\vec{x}}_1$ and $\tilde{\vec{x}}_2$˜ are vectors in $\mathbb{R}^3$; together they define a 2D plane. $\vec{t}$ is also a vector in $\mathbb{R}^3$ but does not lie on this 2D plane. The orthogonal projection of $\vec{t}$ onto this plane is denoted $\hat{\vec{t}}$. The red line from $\vec{t}$ to $\hat{\vec{t}}$ is the residual, whose norm we want to minimize. For visual clarity, all vectors have been converted to unit norm.}
\label{fig:graphical-interpretation-of-OLS}
\end{figure}
Given that
\begin{eqnarray}
\vec{X} = \begin{bmatrix}
\vec{x_1}^T &\\
\vec{x_2}^T &\\
 ...        &\\
\vec{x_N}^T
\end{bmatrix}
= \begin{bmatrix}
\vec{\tilde{x_1}} & \vec{\tilde{x_1}} & ... &\vec{\tilde{x_D}}
\end{bmatrix}\\
\vec{t} = \begin{bmatrix}
t_1 \\
t_2 \\
... \\
t_n
\end{bmatrix}
\end{eqnarray}
We seek a vector $\hat{\vec{t}} \in \mathbb{R}^N$ that lies in the column linear space of $\vec{X}$ and is as close as possible to $\vec{t}$,i.e.,we want to find
\begin{eqnarray}
\hat{\vec{t}} \in span(\vec{X}) \\
\Rightarrow \hat{\vec{t}} = \vec{X}\vec{w} = w_1\vec{\tilde{x_1}}+\cdot\cdot\cdot+w_D\vec{\tilde{x_D}} \\
\vec{\hat{t}}=\arg\min\limits_{\hat{\vec{t}} \in \text{span} (\{\vec{\tilde{x_1}},...,\vec{\tilde{x_D}}\})}
\end{eqnarray}

To minimize the norm of the residual, $\vec{t}-\hat{\vec{t}}$, we want the residual vector to be orthogonal to every column of $\vec{X}$,so˜ $\tilde{\vec{x}}_j(\vec{t}-\hat{\vec{t}})=0$ for $j=1:D$. Hence
\begin{equation}\begin{split}
\tilde{\vec{x}}_j(\vec{t}-\hat{\vec{t}})=0 & \Rightarrow \vec{X}^T(\vec{t}-\vec{X}\vec{w})=0 \\
                                           & \Rightarrow \vec{w}=(\vec{X}^T\vec{X})^{-1}\vec{X}^T\vec{t}
\end{split}\end{equation}

\subsection{Sequential learning}
\textbf{Stochastic gradient descent(SGD)}, is also known as sequential gradient descent.
Batch techniques,such as maximum likelihood, involves processing the entire training set in one go(pass), which can be computationally costly for large data sets. For sufficiently large datasets, it's worthwhile to use \textbf{sequential} algorithms,known as \textbf{on-line} algorithms.

If the error function comprises a sum over data points $E = \sum_n En$, then after presentation of pattern n, the stochastic gradient descent algorithm updates the parameter vector w using
\begin{align}
E &=\sum_{n}E_n \\
\vec{w}^{\tau+1} &= \vec{w}^{\tau}-\eta\nabla E_n
\end{align}
where $\tau$ denotes the iteration number,the $\eta$ is a learning rate parameter.For the case of sum-of-squares error function,this gives
\begin{equation}
\vec{w}^{\tau+1} = \vec{w}^{\tau}-\eta(t_n-\vec{w}^{(\tau)T}\phi_n)\phi_n
\end{equation}
where $\phi_n=\phi(\vec{x_n})$.This is known as the \textbf{least-mean-squares} or the LMS algorithm.
Notice that the gradient vector $\nabla{E_n}$ here, is different with the one we used to derive the closed-form solution, which was using \textbf{numerator layout} to make it convenient to solve the equation.

\begin{align}
\because \dfrac{\partial}{\partial w_i}\text{NLL}(\vec{w})=& \sum\limits_{i=1}^N (\vec{w}^T\vec{x}_i-y_i)x_{ij} \\
\therefore w_j=& w_j - \alpha\dfrac{\partial}{\partial w_j}\text{NLL}(\vec{w}) \nonumber \\
                  =& w_j - \sum\limits_{i=1}^N \alpha(\vec{w}^T\vec{x}_i-y_i)x_{ij} \\
\therefore \vec{w}=& \vec{w}-\alpha(\vec{w}^T\vec{x}_i-y_i)\vec{x}
\end{align}

\section{Ridge regression(MAP)}
One problem with ML estimation is that it can result in over-fitting. In this section, we discuss a way to ameliorate this problem by using MAP estimation with a Gaussian prior.

\subsection{Basic idea}
We can encourage the parameters to be small, thus resulting in a smoother curve, by using a zero-mean Gaussian prior:
\begin{equation}
p(\vec{w})=\prod\limits_j \mathcal{N}(w_j|0,\tau^2)
\end{equation}
where $1/\tau^2$ controls the strength of the prior. The corresponding MAP estimation problem becomes
\begin{equation}
\arg\max_{\vec{w}} \sum\limits_{i=1}^N \log{\mathcal{N}(t_i|w_0+\vec{w}^T\vec{\phi}_i,\sigma^2)}+\sum\limits_{j=1}^D \log{\mathcal{N}(w_j|0,\tau^2)}
\end{equation}

This is equivalent to minimizing the following
\begin{align}\label{eqn:Ridge-regression-J}
J(\vec{w})&=E_D(\vec{w})+\lambda E_W(\vec{w}) \\
&=\dfrac{1}{N}\sum\limits_{i=1}^N (y_i-(w_0+\vec{w}^T\vec{x}_i))^2+\lambda\lVert\vec{w}\rVert^2 , \lambda \triangleq \dfrac{\sigma^2}{\tau^2}
\end{align}
where the first term is the MSE/ NLL as usual, and the second term, $\lambda \geq 0$, is the regularization coefficient that controls the complexity penalty.
Here $E_W(\vec{w})$ is one of the simplest forms of regularizer given by the sum-of-squares of the weight vector elements.
\begin{equation}
E_W(\vec{w})=\dfrac{1}{2}\vec{w}^T\vec{w}
\end{equation}
This particular choice of regularizer is known as \textbf{weight decay}.

The corresponding solution is given by
\begin{equation}\label{eqn:Ridge-regression-solution}
\hat{\vec{w}}_{\mathrm{ridge}}=(\lambda\vec{I}_D+\vec{\Phi}^T\vec{\Phi})^{-1}\vec{\Phi}^T\vec{t}
\end{equation}

This technique is known as \textbf{ridge regression},or \textbf{penalized least squares}. In general, adding a Gaussian prior to the parameters of a model to encourage them to be small is called $\ell_2$ \textbf{regularization} or \textbf{weight decay}. Note that the offset term $w_0$ is not regularized, since this just affects the height of the function, not its complexity.

We will consider a variety of different priors in this book. Each of these corresponds to a different form of \textbf{regularization}. This technique is very widely used to prevent overfitting.

\subsection{Multiple outputs}
Now consider the case where we wish to predict $K>1$ target variables,which we denote collectively by the target vector $\vec{t}$.To use the same set of basis functions to model all the components of the target vector so that
\begin{equation}
\vec{y}(\vec{x},\vec{w}) = \vec{W}^T\phi(x)
\end{equation}
where $\vec{y}$ is a $K$-dimensional column vector,$\vec{W}$ is an $M\times K$ matrix of parameters.

The conditional distribution of the target vector is an isotropic Gaussian
\begin{equation}
p(\vec{t}|\vec{x},\vec{W},\beta) =
\mathcal{N}(\vec{t}|\vec{W}^T\phi(x,\beta^{-1}\vec{I}))
\end{equation}

The log likelihood function is then given by
\begin{align}
\ln p(\vec{T}|\vec{X},\vec{W},\beta)
&=\sum_{n=1}^{N}\ln \mathcal{N}(\vec{t_n}|\vec{W}^T\phi(x_n),\beta^{-1}\vec{I}) \\
&=\dfrac{NK}{2}\ln(\dfrac{\beta}{2\pi})
 -\dfrac{\beta}{2}\sum_{n=1}^{N}\parallel \vec{t_n}-\vec{W}^T\phi(x_n)\parallel^2
\end{align}
We maximize this function with respect to $\vec{W}$,giving
\begin{equation}
\vec{W}_{ML} = (\vec{\Phi}^T\vec{\Phi})^{-1}\vec{\Phi}^T\vec{T}
\end{equation}

\subsection{Numerically stable computation *}

\begin{equation}\label{eqn:Ridge-regression-SVD}
\hat{\vec{w}}_{\mathrm{ridge}}=\vec{V}(\vec{Z}^T\vec{Z}+\lambda\vec{I}_N)^{-1}\vec{Z}^T\vec{y}
\end{equation}

\subsection{Connection with PCA *}

\subsection{Regularization effects of big data}
Regularization is the most common way to avoid overfitting. However, another effective approach — which is not always available — is to use lots of data. It should be intuitively obvious that the more training data we have, the better we will be able to learn.

In domains with lots of data, simple methods can work surprisingly well (Halevy et al. 2009). However, there are still reasons to study more sophisticated learning methods, because there will always be problems for which we have little data. For example, even in such a data-rich domain as web search, as soon as we want to start personalizing the results, the amount of data available for any given user starts to look small again (relative to the complexity of the problem).

\section{The Bias-Variance Decomposition}
\subsubsection{representation}
Data representation:
N observations of x,wiritten
$X \equiv (x_1,x_2,...,x_n)^T$
together with corresponding observations of the values of t,denoted
$t \equiv (t_1,t_2,...t_N)^T$.
\begin{equation}
t_i = f(\vec{x_i}) + \epsilon
\end{equation}
where the noise $ \epsilon $ has zero mean and variance $ \sigma^2 $.
Find a function \[ \hat{f}(x) \] that approximates the true function \[ t = f(\vec{x}) \] as well as possible.Make "as well as possible" precise  by measuring the mean squared error between y and $ \hat{f}(x) $,we want $ (t - \hat{f}(x))^2 $ to be minimal.
Hypothesis function(model representation):
$y(x,\bold w)= w_0+w_1x+w_2x^2+...+w_Mx^M = \sum_{j=0}^{M}w_jx^j$
M is the order of the polynomial,and $x^j$ denotes $x$ raised to the power of $j$.The polynomial coefficients $w_0,...w_M$ are
collectively denoted by the vector $\mathbf{w}$.

Error Function(Sum of squares of errors between predictions $y(x_n,w)$ for each data point $x_n$ and the corresponding target values $t_n$,so that we minimize:
\begin{equation}
E(\bold w)=\frac{1}{2}\sum_{n=1}^{N}\{y(x_n,\bold w)-t_n \}^2
\end{equation}
root-mean-squre(RMS) error defined by
\begin{equation}
E_{RMS} = \sqrt[2]{2E(\bold w^*)/N}
\end{equation}
Penalized(regularized) error function
\begin{equation}
\widetilde{E}(\textbf{w}) = \frac{1}{2}\sum_{n=1}^{N}\{y(x_n,\textbf{w}-t_n\}^2 + \frac{1}{2} \parallel \textbf{w} \parallel^2
\end{equation}
where $\parallel \textbf{w} \parallel^2 \equiv \textbf{w}^T\textbf{w}=w_0^2+w_1^2+...+w_M^2$

\subsubsection{Loss function for regression}
\begin{equation}
\mathbb{E}[\mathit{L}] = \iint\mathit{L}(t,y(\textbf{x}))p(\textbf{x},t)d\textbf{x}dt
\end{equation}
A common choice of loss function in squared loss given by
\begin{eqnarray}
\mathit{L}(t,y(\textbf{x})) = \{y(\textbf{x}) - t\}^2 \\
\mathbb{E}[\mathit{L}] = \iint\{y(\textbf{x}) - t\}^2 p(\textbf{x},t)d\textbf{x}dt.
\end{eqnarray}
Minimize $ \mathbb{E}[\mathit{L}] $ by using the calculus of variations to give
\begin{equation}
\frac{\delta\mathbb{E}[\mathit{L}]}{\delta y(\mathbf{x}))} = 2 \int\{ y(\mathbf{x} -t) \}p(\mathbf{x},t)dt = 0
\end{equation}
Solving for $ y(\textbf{x}) $ and using the sum and product rules of probability,we obtain
\begin{equation}
y(\textbf{x}) = \frac{\int tp(\textbf{x},t)dt}{p(\textbf{x})} = \int tp(t|\textbf{x})dt = \mathbb{E}[t|\textbf{x}]
\end{equation}

Let's derive this result in a slightly different way.Armed with knowledge that the optimal solution is the
conditional expectation,we can expand the square term as follows
\begin{equation}
\{y(\textbf{x} -t)\}^2
= \{y(\textbf{x}) - \mathbb{E}[t|\textbf{x}] + \mathbb{E}[t|\textbf{x}] - t )\}^2
= \{ y(\textbf{x}) - \mathbb{E}[t|\textbf{x}] \}^2 + 2\{ y(\textbf{x}) - \mathbb{E}[t|\textbf{x}] \}\{ \mathbb{E}[t|\textbf{x}]-t \} + \{ \mathbb{E}[t|\textbf{x}] -t \}^2
\end{equation}
where,$ \mathbb{E}[t|\textbf{x}] $ denote $ \mathbb{E}_{t}[t|\textbf{x}] $.Substitute into the loss function and perform the integral over t,we see the cross-term vanishes
\begin{align}\label{eqn:squared loss function}
\mathbb{E}[\mathit{L}]
&= \iint\{y(\textbf{x}) - t\}^2 p(\textbf{x},t)d\textbf{x}dt                    \\
&= \int \{ y(\textbf{x}) -\mathbb{E}[t|\textbf{x}] \}^2 p(\textbf{x})d\textbf{x} +
\int\{ \mathbb{E}[t|\textbf{x}] - t \}^2 p(\textbf{x})d\textbf{x}               \\
&= \int \{ y(\textbf{x}) -h(\textbf{x}) \}^2 p(\textbf{x})d\textbf{x} +
\int\{ h(\textbf{x}) - t \}^2 p(\textbf{x})d\textbf{x}                          \\
&= \int \{ y(\textbf{x}) -h(\textbf{x}) \}^2 p(\textbf{x})d\textbf{x} +
\int\{ h(\textbf{x}) - t \}^2 p(\textbf{x},t)d\textbf{x}dt                      \\
\end{align}

\subsubsection{Decomposition}
For a popular choice,we use squared loss function,for which the optimal prediction is given by the conditional
expectation,which we denote by h(\textbf{x}) and which is given by
\begin{equation}
h(\textbf{x}) = \mathbb{E}[t|\textbf{x}] = \int tp(t|\textbf{x})dt
\end{equation}

Consider the integrand of the first term of \ref{eqn:squared loss function},which for particular data set D takes the form
\begin{equation}
\{ y(\textbf{x};D) - h(\textbf{x}) \} ^2
\end{equation}
This quantity will be dependent on the particular data set D,so we take its average over the ensemble of data sets.
If we add and subtract the quantity $ \mathbb{E_D}[y(\textbf{x};D)] $ inside the braces,and then expand,we obtain
\begin{align}
\{ y(\textbf{x};D) - h(\textbf{x}) \} ^2 \\
=&\{y(\textbf{x};D) - \mathbb{E_D}[y(\textbf{x};D)]
	+ \mathbb{E_D}[y(\textbf{x};D)] -h(\textbf{x})  \}^2         \\
=& \{ y(\textbf{x};D) -\mathbb{E}_{\mathbb{D}}[y(\textbf{x};D)]    \}^2
	+ \{ \mathbb{E_D}[y(\textbf{x};D)] - h(\textbf{x})\}^2
	+ 2\{ y(\textbf{x};D) - \mathbb{E_D}[y(\textbf{x};D)]\}\{ \mathbb{E_D}[y(\textbf{x};D)] -h(\textbf{x})\} \\
=& \underbrace{\{ y(\textbf{x};D) -\mathbb{E}_{\mathbb{D}}[y(\textbf{x};D)]    \}^2}_\text{\color{red}{variance}}
	+ \underbrace{\{ \mathbb{E_D}[y(\textbf{x};D)] - h(\textbf{x})\}^2}_\text{\color{blue}$(bias)^2$}
	+0\\
\end{align}
The decomposition of the expected squared loss
\begin{equation}
\text{expected loss} = (bias)^2 + variance + noise
\end{equation}
where
\begin{eqnarray}
(bias)^2 = ... \\
variance = ... \\
noise = ...
\end{eqnarray}

The function $ y(\textbf{x}) $ we seek to determine enters only the first term,which will be minimized when $ y(\textbf{x}) $ is equal to $ \mathbb{E}[t|\textbf{x}] $,in which case this term will vanish.The second term is the variance of distribution of t,averaged over $ \textbf{x} $,representing the intrinsic variabilility of the target data and can be regarded as noise.It's the irreducible minimum value of the loss function.

More sophisticated loss function,Minkowski loss
\begin{equation}
\mathbb{E}[\mathit{L_q}] = \iint| y(\textbf{x}) - t |^q p(\textbf{x},t)d\textbf{x}dt
\end{equation}

\section{Bayesian linear regression}
Hold-out data can be used to determine model complexity but it will be
computationally expensive and wasteful of valuable data.We therefore turn to a Bayesian treatment of linear regression,which will avoid the over-fitting problem of maximum likelihood,and which will also lead to automatic methods of determining model complexity using the training data.

\subsection{Parameter distribution}
First introduce a prior probability distribution over the model parameters.The likelihood function $p(\vec{t}|\vec{w})$ defined by \ref{eqn:linear regression likelihood} is the exponential of a quadratic function of $\vec{w}$,so the corresponding conjugate prior is
Gaussian
\begin{equation}
p(\vec{w}) = \mathcal{N}(\vec{w}|\vec{m_0},\vec{S}_0)
\end{equation}

The posterior distribution is proportional to the product of the likelihood and the prior.And the posterior will also be Gaussian due to the choice of conjugate Gaussian prior,which is derived in \ref{sec:Gaussian distribution}.
\begin{equation}\label{eqn:Bayes linear regression posterior}
p(\vec{w}|\vec{t}) = \mathcal{N}(\vec{w}|\vec{m}_N,\vec{S}_N)
\end{equation}
where
\begin{align}
\vec{m}_N &=\vec{S}_N(\vec{S}_0^{-1}\vec{m}_0+\beta\vec{\Phi}^T\vec{t})\\
\vec{S}_N^{-1} &= \vec{S}_0^{-1}+\beta\vec{\Phi}^T\vec{\Phi}
\end{align}
Thus the maximum posterior weigh vector is simply given by $\vec{w}_{MAP}=\vec{m}_{N}$.If $N=0$ then the posterior distribution reverts to the prior.Furthermore,if data points arrive sequentially,then the posterior distribution at any stage acts as the prior distribution,such that the new posterior is again given.

A zero-mean isotropic Gaussian governed by a single precision parameter $\alpha$ so that
\begin{equation}
p(\vec{w}|\alpha) = \mathcal{N}(\vec{w}|\vec{0},\alpha^{-1}\vec{I})
\end{equation}
for which the posterior is given by
\begin{align}
\vec{m}_N &=\beta\vec{S}_N\vec{\Phi}^T\vec{t} \\
\vec{S}_N^{-1}&=\alpha\vec{I}+\beta\vec{\Phi}^T\vec{\Phi}
\end{align}

The log of posterior distribution is given by the sum of the log likelihood and the log of prior and,as a function of $\vec{w}$,takes the form
\begin{equation}
\log p(\vec{w}|\vec{t}) =
-\dfrac{\beta}{2}\sum_{n=1}^{N}\{t_n-\vec{w}^T\phi(\vec{x}_n) \}^2-\dfrac{\alpha}{2}\vec{w}^T\vec{w}+const
\end{equation}
Maximization of this posterior w.r.t $\vec{w}$ is equivalent to minimization of the sum-of-squares error function with the addition of a quadratic regularization term,corresponding with $\lambda=\alpha/\beta$.

\subsection{Predictive distribution}
Evaluate the \textbf{predictive distribution} defined by
\begin{equation}
p(t|\vec{t},\alpha,\beta) = \int p(t|\vec{w},\beta)p(\vec{w}|\vec{t},\alpha,\beta)d\vec{w}
\end{equation}
The right hand quantity can be interpreted as the marginalization by integration of the joint distribution of $p(t,\vec{w}|\vec{t},\alpha,\beta)$ over $\vec{w}$.And the joint distribution can be solved by the Bayesian theorem for Gaussian.
The conditional distribution $p(t|\vec{x},\vec{w},\beta)$ of the target variable is given by \ref{eqn:linear regression representation},and the posterior weight distribution is given by \ref{eqn:Bayes linear regression posterior}.This involves the \textbf{convolution} of two Gaussian distributions.The predictive distribution take the form
\begin{equation}
p(t|x,\vec{t},\alpha,\beta) = \mathcal{N}(t|\vec{m}_N^T\phi(x),\sigma_N^2(\vec{x}))
\end{equation}
where the variance $\sigma_N^2(\vec{x})$ is given by
\begin{equation}
\sigma_N^2(\vec{x})=\dfrac{1}{\beta}+\phi(\vec{x})^T\vec{S}_N\phi(\vec{x})
\end{equation}
The first term represents the noise on the data whereas the second term reflects the uncertainty associated with the parameters $\vec{w}$.
$\sigma_{N+1}^2(\vec{x})\leq \sigma_N(\vec{x})$.In the limit $N\rightarrow \infty$,the second term goes to zero.

Note that if both $\vec{w}$ and $\beta$ are treated as unknown,then we can introduce a conjugate prior distribution $p(\vec{w},\beta)$ given by Gaussian-gamma distribution,leading to a Student's t-distribution predictive distribution.

\subsection{Equivalent kernel}
The posterior mean solution \ref{} has an interpretation that will set stage for kernel methods,including Gaussian processes.The predictive mean:
\begin{equation}
y(\vec{x},\vec{w})=\vec{m}_N^T\vec{\phi}(\vec{x})
=\beta\vec{\phi}(\vec{x})^T\vec{S}_N\vec{\Phi}^T\vec{t}
=\sum_{n=1}^{N}\beta\vec{\phi}(\vec{x})^T\vec{S}_N\vec{\phi}(\vec{x}_n)t_n
=\sum_{n=1}^{N}\mathit{k}(\vec{x},\vec{x}_n)t_n
\end{equation}
where
\begin{equation}
k(\vec{x},\vec{x'})
=\beta\vec{\phi}(\vec{x})^T\vec{S}_N\vec{\phi}(\vec{x'})
\end{equation}
is known as the \textbf{smoother matrix} or \textbf{equivalent kernel}.Regression functions,such as this,which make predictions by taking linear combinations of the training set target values are known \textbf{linear smoothers}.The kernel functions are localized around $x$(local evidence weight more than distant evidence).

The covariance between $y(\vec{x})$ and $y(\vec{x'})$ is given by
\begin{align}
cov[y(\vec{x}),y(\vec{x'})] &=cov[\vec{\phi}(\vec{x})^T\vec{w},\vec{w}^T\vec{\phi}(\vec{x'})] \\
&=\phi(\vec{x})^T\vec{S}_N\phi(\vec{x'})=\beta^{-1}\mathit{k}(\vec{x},\vec{x'})
\end{align}
We see that the predictive mean at nearby points will be highly correlated.

The formulation of linear regression in terms of kernel functions suggests an alternative approach,called \textbf{Gaussian process}:Instead of introducing a set of basis functions,which implicitly determines an equivalent kernel,we can instead define a localized kernel directly and use this to make predictions for new input vectors $\vec{x}$,given the observed training set.

The effective kernel defines the weights by which the training set target values are combined in order to make a prediction at a new value of $x$,and it can be shown that these weights sum to one,
\begin{align}
\sum\limits_{n=1}^{N}\mathcal{k}(\vec{x},\vec{x}_n) = 1
\end{align}
for all values of $\vec{x}$.

General kernel functions share an important property,namely that it can be expressed in the form an inner product with respect to a vector $\varPsi(\vec{x}) $ of nonlinear functions,so that
\begin{align}
\mathcal{k}(\vec{x},\vec{z}) = \varPsi(\vec{x})^T\varPsi(\vec{z})
\end{align}
where $\varPsi(\vec{x})=\beta^{1/2}\vec{S}_n^{1/2}\phi(\vec{x})$

\section{Bayesian Model Comparison}
The over-fitting associated with maximum likelihood can be avoided by \textbf{marginalizing(summing or integrating)} over the model parameters instead of making point estimates of their values,no need for validation.

Suppose we wish to compare a set of $L$ models $\{\mathcal{M}_i\},i=1,...L$ over observed data $\mathcal{D}$.Evaluate the posterior distribution
\begin{align}
p(\mathcal{M}_i|\mathcal{D})\propto p(\mathcal{M}_i)p(\mathcal{D}|\mathcal{M}_i)
\end{align}
The prior allows us to express a preference for different models.$p(\mathcal{D}|\mathcal{M}_i)$ is the \textbf{model evidence},which expresses the preference shown by the data for different models.It is also called the \textbf{marginal likelihood}.The ratio of model evidences $p(\mathcal{D}|\mathcal{M}_i)/p(\mathcal{D}|\mathcal{M}_j)$ for two models is \textbf{Bayes factor}.
The predictive distribution
\begin{align}
& p(t|\vec{x},\mathcal{D})p(\mathcal{M}_i|\mathcal{D})&=p(t|\vec{x},\mathcal{M}_i,\mathcal{D})p(\mathcal{M}_i|\mathcal{D}) \\
&\Rightarrow \sum\limits_{i=1}^{L}p(t|\vec{x},\mathcal{D})p(\mathcal{M}_i|\mathcal{D})&=\sum\limits_{i=1}^{L}p(t|\vec{x},\mathcal{M}_i,\mathcal{D})p(\mathcal{M}_i|\mathcal{D}) \\
&\Rightarrow p(t|\vec{x},\mathcal{D})&=\sum\limits_{i=1}^{L}p(t|\vec{x},\mathcal{M}_i,\mathcal{D})p(\mathcal{M}_i|\mathcal{D}) \\
\end{align}
This is an example of a \textbf{mixture distribution} in which the overall predictive distribution is obtained by averaging the predictive distributions of individual models weighted by the posterior probabilities $p(\mathcal{M}_i|\mathcal{D})$ of those models.

To use the single most probable model alone to make predictions is known as \textbf{model selection}.Model evidence
\begin{align}
p(\mathcal{D}|\mathcal{M}_i)=\int p(\mathcal{D}|\mathcal{\vec{w}})p(\vec{w}|\mathcal{M}_i)d\vec{w}
\end{align}
because
\begin{align}
p(\vec{w}|\mathcal{D},\mathcal{M}_i)=\dfrac{p(\mathcal{D}|\vec{w},\mathcal{M}_i)p(\vec{w}|\mathcal{M}_i)}{p(\mathcal{D}|\mathcal{M}_i)}
\end{align}

Assume the posterior distribution over parameters is sharply peaked around the most probable value $w_{MAP}$,with width $\Delta w_{posterior}$,then we can approximate the integral by the value of integrand at its maximum times the width of the peak.And further assume that the prior is flat with width $\Delta w_{prior}$ so that $p(w)=1/\Delta w_{prior}$,then we have
\begin{align}
p(\mathcal{D})=\int p(\mathcal{D}|w)p(w)dw \simeq p(\mathcal{D}|w_{MAP})p(w_{MAP})\Delta w_{posterior} \simeq p(\mathcal{D}|w_{MAP})\dfrac{\Delta w_{posterior}}{\Delta w_{prior}}
\end{align}
\begin{SCfigure*}
	\caption{model evidence approximation}
	\includegraphics{prml/Figure3.12}
\end{SCfigure*}

and so taking logs
\begin{align}
\ln p(\mathcal{D}) \simeq \ln p(\mathcal{D}|w_{MAP}) +\ln(\dfrac{\Delta w_{posterior}}{\Delta w_{prior}})
\end{align}
If we have a set of $M$ parameters,assuming that all parameters have the same ratio of $\Delta w_{posterior}/\Delta w_{prior}$,we obtain
\begin{align}
\ln p(\mathcal{D}) \simeq \ln p(\mathcal{D}|w_{MAP}) +M\ln(\dfrac{\Delta w_{posterior}}{\Delta w_{prior}})
\end{align}
As we increase the complexity of the model,the first term representing the fit to the data,increases(?),whereas the second term will decrease due to the dependence on $M$.The optimal model is given by a trade-off between these two competing terms.
\begin{SCfigure*}
	\caption{Model complexity increases}
	\includegraphics{prml/Figure3.13}
\end{SCfigure*}
Average the Bayes factor over the distribution of data sets
\begin{align}
\int p(\mathcal{D}|\mathcal{M}_1)\ln\dfrac{p(\mathcal{D}|\mathcal{M}_1)}{p(\mathcal{D}|\mathcal{M}_2)}
\end{align}
This quantity is $Kullback-Leibler$ divergence,and will be bigger than or equal to $0$ if $\mathcal{M}_1$ is the correct model.

\section{Model Evidence}


\section{energy in one slab for 1D}

How to find the energy contained in a 1D slab:

Physically, there is a slab in the x-direction from $x=0$ to $x=a$. The wave propating in the positive x-direction has amplitude $A$ and the opposite direction has amplitude $B$.

Method: match the wave and derivative at the boundary.

Electric field:
\begin{equation}
 E(x) = A e^{i k n x} + B e^{-i k n x}
\end{equation}

Energy density is $\epsilon |E|^2$, where $\epsilon$ is the dielectric constant and is related to the index of refraction $n$ by $\epsilon=n^2$. The total energy in a slab of size $a$ is
\begin{equation}
 energy = {\cal E} = \epsilon \int_0^a |E|^2 dx
\end{equation}
The integrand is
\begin{equation}
 |E|^2 = (A e^{i k n x} + B e^{-i k n x})(A^* e^{i k n x} + B^* e^{-i k n x})
\end{equation}
Expanding,
\begin{equation}
 |E|^2 = |A|^2+ AB^* e^{2 i k n x} + A^* B e^{-2 i k n x} + |B|^2
\label{eq:expanded_energy}
\end{equation}

How do we know that A and B should be complex? Recall that the field is given by $E=A+B$, and the derivative of the field is
\begin{equation}
 \frac{1}{k} \frac{\partial E}{ \partial x} = i (A-B)
\end{equation}
Solving for $A$ and $B$,
\begin{equation}
 \begin{gathered}
  A= \frac{1}{2} (E-i\frac{1}{k} \frac{\partial E}{ \partial x}) \\
  B= \frac{1}{2} (E+i\frac{1}{k} \frac{\partial E}{ \partial x})
 \end{gathered}
\end{equation}
Thus $A$ and $B$ are both complex.

Returning to Eq.~\ref{eq:expanded_energy} and re-grouping terms,
\begin{equation}
 {\cal E} = \epsilon \int_0^a |E|^2 dx = n^2 [(|A|^2+|B|^2)a + AB^* \frac{1}{2ikn}(e^{2 i k n x}-1) - A^*B\frac{1}{2ikn}(e^{-2 i k n x}-1)]
\label{eq:energy_integral}
\end{equation}
Even though the terms contain complex pieces, we expect the result to be real since energy is observable and doesn't have a phase.

If $z=x+iy$, then
\begin{equation}
 z+z^* = x+iy +x-iy = 2x = 2 Re(z)
\end{equation}
Thus Eq.~\ref{eq:energy_integral} is real.

The middle term of Eq.~\ref{eq:energy_integral} can be expanded as
\begin{equation}
 AB^* \frac{1}{2ikn}(e^{2 i k n x}-1) = \frac{AB^* e^{ i k n x}}{kn} \left( \frac{e^{ i k n x} - e^{-i k n x}}{2i}\right)
\end{equation}

And then applying $z+z^*=2 Re(z)$, the energy in one 1D slab is

\begin{equation}
 {\cal E} =  n^2 \left[(|A|^2+|B|^2)a + 2 Re \left(  \frac{AB^* e^{ i k n x} sin(kna)}{kn}\right)\right]
\end{equation}


%----------------------------------------------------------------------------------------
%   ELECTROSTATIC EQUILIBRIUM
%----------------------------------------------------------------------------------------

\section{Electrostatic equilibrium}
\subsection{Jacobian Matrix}
Let us consider the interval $[-1, 1]$ and two electrostatic charges fixed at the positions $-1$ and $1$. We assume that there exists $N$ charges positioned at $x_1, x_2,..., x_N$ and that these charges can move freely in the interval $[-1, 1]$.

Findind the positions that essen the amout energy of the charges means:
\[\nabla E(x_1, x_2, ..., x_n) = \begin{pmatrix}\frac{\partial E(x_1,...,x_N)}{\partial x_i}\end{pmatrix}_{i \in [1,N]} = 0\]

Noticing that :
\[\frac{\partial E(x_1,...,x_N)}{\partial x_i} = \frac{1}{x_i + 1} + \frac{1}{x_i - 1} +  \displaystyle\sum_{j=1,j\ne i}^{N}\frac{1}{(x_i - x_j)}\]

Thus, the jacobian matrix of $\nabla E(x_1, x_2, ..., x_n)$ is defined as:
\begin{center}
  \fbox{
    $J = \begin{pmatrix}\frac{\partial^2 E}{\partial x_i \partial x_j}\end{pmatrix}_{i\in [1,N], j\in[1,N]}$}
\end{center}
where

\[\begin{pmatrix}\frac{\partial^2 E}{\partial x_i \partial x_j}\end{pmatrix}  =  \left\{
\begin{array}{ll}
  \frac{1}{(x_i - x_j)^2} & \mbox {if $i\ne j$}\\
  \frac{-1}{(x_i +1)^2 } - \frac{1}{(x_i - 1)^2 } - \displaystyle\sum_{j=1,j\ne i}^{N}\frac{1}{(x_i  - x_j)^2} & \mbox{else}
\end{array}\right.\]

\subsection{Equation solving}
The Newton-Raphson method is applied in order to find the roots of non-linear equation system.

The equilibrium positions are exactly the roots of successive derivatives of Legendre Polynomials, which are:

\begin{center}
  \fbox{$P_n' = \frac{1}{2^n n!}\frac{d^{n+1}}{dx^{n+1}}((x^2 - 1 )^n)$}
\end{center}
%%Idéalement, il faudait ajouter un graphique qui compare les racines et les les polynomes de legendre et de même concernant l'étude de minimum et maximum


\section{Exercises}
\begin{enumerate}
  \item Write the following sets explicitly:
    \begin{enumerate}[label=(\roman*)]
      \item $\left\{ x\in \mathbb{N}\mid1<x\leq7\right\}$
      \item $\left\{ x\in \mathbb{Z}\mid x<5\right\}$
      \item $\left\{ x\in \mathbb{R}\mid x^{2}=-1 \right\}$
      \item $\left\{ x\in \mathbb{N} \wedge x\in \mathbb{Q} \right\}$
      \item $\left\{ x\in \mathbb{R} \mid x^{2}-3x-4=0 \right\}$
      \item $\left\{ x\in\mathbb{R}\mid x<5\wedge x \geq 2\right\}$
    \end{enumerate}

	\item Determine the relation between the sets:
    \begin{enumerate}[label=(\roman*)]
      \item $A=\left\{ 1,\ 2,\ 3\right\},\ B=\left\{ 1,\ 2 \right\}$
      \item $A=\varnothing,\ B=\left\{ 2,\ -5,\ \pi \right\}$
      \item $A=\mathbb{Z},\ B=\left\{ \pm x \mid x\in\mathbb{N} \cup \right\{0\left\} \right\}$
			\item $A=\left\{\pi, \eu, \sqrt{2}\right\},\ B=\mathbb{Q}$
    \end{enumerate}

	\item Write all elements in $S^{2}\times W$, where $S=\{\alpha,\beta,\gamma\}$ and $W=\{x,y,z\}$. Find a condition that guarantees $S^{2}\times W = W\times S^{2}$.

	\item How many different injective functions $f:\{1,2\}\to\{1,2\}$ exist? How many injective functions $f:\{1,2,3\}\to\{1,2,3\}$ exist? How many inject functions $f:\{1,2,\dots,n\}\to\{1,2,\dots,n\}$ exist for a given $n\in\mathbb{N}$?

	\item For each of the real functions below, find a set on which it is surjective (use a graphing calculator if you are not familiar with the shape of a function):
		\[
			x^{2},\ x^{3}-5,\ \eu^{-x^{2}/2},\ \sin(x),\ \sin(x)+\cos(x),\ x\eu^{x}.
		\]

	\item Given two sets $A,B$ such that $|B|=|A|-1$, can a bijective function $f:A\to B$ exist? Explain your answer.
	\item MORE EXERCISES TO BE WRITTEN\ldots
\end{enumerate}


\lab{Importance Sampling and Monte Carlo Simulations}{Importance Sampling and Monte Carlo Simulations}
\objective{Use importance sampling to reduce the error and variance of Monte Carlo Simulations.}
\label{lab:montecarlo2}

\section*{Introduction} % =====================================================

The traditional methods of Monte Carlo integration as discussed in the previous lab are not always the most efficient means to estimate an integral. For example, assume we were trying to find the probability that a randomly chosen variable $X$ from the standard normal distribution is greater than $3$. We know that one way to solve this is by solving the following integral:

\begin{equation} \label{eq:integral}
P(X > 3) = \int_{3}^{\infty} f_X(t)\,dt = \frac{1}{\sqrt{2\pi}}\int_{3}^{\infty} e^{-t^2/2}\,dt
\end{equation}

If we define the function $h: \mathbb{R} \rightarrow \mathbb{R}$ as

$$h(t) = \begin{cases}
1 & \text{ if } t > 3 \\
0 & \text{ if } t \leq 3
\end{cases}, $$
we can rewrite this integral as

\begin{equation*}
\int_{3}^{\infty} f_X(t)\,dt = \int_{-\infty}^{\infty} h(t)f_X(t)\,dt.
\end{equation*}

By the Law of the Unconscious Statistician (see Volume 2 \S 3.5), we can restate the integral above as

$$\int_{-\infty}^{\infty} h(t)f_X(t)\,dt = E[h(X)].$$
Being able to write integrals as expected values is an essential tool in this lab.

\section*{Monte Carlo Simulation} % ===========================================

In the last section, we expressed the probability of drawing a number greater than $3$ from the normal distribution as an expected value problem. We can now easily estimate this same probabilty using Monte Carlo simulation.
Given a random i.i.d. sample $x_1, x_2, \cdots , x_N$ generated by $f_X$, we can estimate $E[h(X)]$ using
\begin{equation} \label{eq:estimator}
\widehat{E}_n[h(X)] = \frac{1}{N}\sum_{i = 1}^{N}h(x_i)
\end{equation}

Now that we have defined the estimator, it is now quite manageable to approximate Equation \ref{eq:integral}. By the Weak Law of Large Numbers (see Volume 2 \S 3.6), the estimate will get closer and closer to the actual value as we use more and more sample points.

\begin{comment}
\begin{lstlisting}
>>> h = lambda x : x > 3
>>> N = 10**7
>>> x = np.random.normal(size=N)
>>> 1./N * np.sum(h(x))
0.0013644
\end{lstlisting}
\end{comment}

\begin{problem} \label{prob:mc}
Write a function in Python that estimates the probability that a random draw from the standard normal distribution is greater than 3 using Equation \ref{eq:estimator}. Your function should accept a parameter \li{n} for the number of samples to use in your approximation. Your answer should approach $0.0013499$ for sufficiently large samples.
\end{problem}

Though this approach gets the job done, it turns out that this isn't very efficient. Since the probability of drawing a number greater than $3$ from the standard normal distribution is so unlikely, it turns out we need many sample points to get a good approximation.

\section*{Importance Sampling} % ==============================================

Importance sampling is one way to make Monte Carlo simulations converge much faster. We choose a different distribution to sample our points to generate more \emph{important} points. With our example, we want to choose a distribution that would generate more numbers around $3$ to get a more reliable estimate. The theory behind importance sampling boils down to the following result. In these equations, the random variable $X$ is generated by $f_X$ and the random variable $Y$ is generated by $g_Y$. We will refer to $X$ and $Y$ in this way for the remainder of the lab.

\begin{equation} \label{eq:importance}
\begin{split}
E[h(X)] & = \int_{-\infty}^{\infty} h(t)f_X(t)\,dt \\
& = \int_{-\infty}^{\infty} h(t)f_X(t)\left ( \frac{g_Y(t)}{g_Y(t)} \right )\,dt \\
& = \int_{-\infty}^{\infty} \left ( \frac{h(t)f_X(t)}{g_Y(t)} \right )g_Y(t)\,dt \\
& = E\left [ \frac{h(Y)f_X(Y)}{g_Y(Y)}\right ]
\end{split}
\end{equation}

The corresponding estimator is

\begin{equation}\label{eq:imp_estimator}
\begin{split}
\widehat{E}[h(X)] & = \widehat{E}\left [ \frac{h(Y)f_X(Y)}{g_Y(Y)}\right ] \\
& = \frac{1}{N}\sum_{i = 1}^{N}\frac{h(y_i)f_X(y_i)}{g_Y(y_i)}
\end{split}
\end{equation}

The function $f_X$ is the p.d.f. of the \emph{target distribution}. The function $g_Y$ is the p.d.f. of the \emph{importance distribution}. The fraction $\frac{f_X(X)}{g_Y(X)}$ is called the \emph{importance weight}. This allows us to draw a sample from any distribution with p.d.f. $g_Y$ as long as we multiply $h(X)$ by the importance weight.

\subsection*{Choosing the Importance Distribution} % --------------------------

There is no correct choice for the importance distribution. It may be possible to find the distribution that allows the simulation to converge the fastest, but oftentimes, we don't need a perfect answer. Close to perfect is good enough.

We will solve the same problem as in Problem \ref{prob:mc} using importance sampling. We will choose $g_Y$ to be the normal distribution with $\mu = 4$ and $\sigma = 1$.
We have chosen this distribution for $g_Y$ because it will give us more points closer to and greater than 3. Note that it is not necessary to choose an importance distribution of the same type.

\begin{figure}[H]
% MISSING FIGURE!!
% \includegraphics[width=\textwidth]{importance_distribution.pdf}
\caption{In our problem, we choose an importance distribution that will generate more samples that are greater than 3. Though not a perfect choice, choosing a normal distribution with $\mu = 4$ and $\sigma = 1$ will suffice.}
\label{fig:importance}
\end{figure}

\begin{lstlisting}
>>> from scipy import stats
>>> h = lambda x : x > 3
>>> f = lambda x : stats.norm().pdf(x)
>>> g = lambda x : stats.norm(loc=4,scale=1).pdf(x)

# Sample from the N(4,1).
>>> N = 10**7
>>> X = np.random.normal(4,scale=1,size=N)

# Calculate estimate.
>>> 1./N * np.sum(h(X)*f(X)/g(X))
0.00134921134631
\end{lstlisting}

\begin{figure}[H]
\includegraphics[width=\textwidth]{figures/MCvsIS.png}
\caption{Comparison of error between standard method Monte Carlo and Importance Sampling method of Monte Carlo.}
\label{fig:compare}
\end{figure}

\begin{problem} \label{prob:gamma}
A tech support hotline receives an average of 2 calls per minute. What is the probability that they will have to wait at least 10 minutes to receive 9 calls? Implement your estimator using importance sampling. Calculate estimates using 5000, 10000, 15000, $\cdots$, 500000 sample points. Return an array of estimates. Your answers should approach $0.00208726$.

Hint: In Volume 2 \S 3.5, the gamma distribution is defined as, $$f_X(x) = \frac{b^{a}x^{a-1}e^{-xb}}{\Gamma(a)}.$$ The version of the gamma distribution in \li{scipy.stats} is determined by the shape ($a$) and the scale ($\theta$) of the distribution. $$f_X(x) = \frac{x^{a-1}e^{-x/\theta}}{\Gamma(a)\theta^a}$$
You can switch between these representations this with the fact that $\theta = 1/b$.
\end{problem}

\begin{problem}
In this problem, we will visualize the benefits of importance sampling. Create a plot of the error of the traditional methods of Monte Carlo integration and the importance sampling methods of Monte Carlo for Problem \ref{prob:gamma}. What do you observe? Your plot should resemble Figure \ref{fig:compare}.

Hint: The following code solves Problem \ref{prob:gamma} using traditional methods of Monte Carlo integration:
\begin{lstlisting}
h = lambda x : x > 10
MC_estimates = []
for N in xrange(5000,505000,5000):
    X = np.random.gamma(9,scale=0.5,size=N)
    MC = 1./N*np.sum(h(X))
    MC_estimates.append(MC)
MC_estimates = np.array(MC_estimates)
\end{lstlisting}

Hint: To determine the error of your approximations, the following code returns the actual value of the probability:
\begin{lstlisting}
1 - stats.gamma(a=9,scale=0.5).cdf(10)
\end{lstlisting}
\end{problem}

Now that we have visualized the benefits of importance sampling, note that we can achieve the same results as traditional Monte Carlo with a fraction of the samples.

\section*{Generalizing the Principles of Importance Sampling} % ===============

The examples we have explored to this point in the lab were merely educational. Since we have a simple means of calculating the correct answer to Problem \ref{prob:gamma}, it doesn't make much sense to use methods of Monte Carlo in this situation. However, as discussed in the previous lab, there are not always closed-form solutions to the integrals we want to compute.

We can extend the same principles we have discussed thus far to solve many types of problems. For a more general problem, we can implement importance sampling by doing the following:
\begin{enumerate}
\item Define a function $h$ where, $h(t) = \begin{cases}
1 & \text{ if condition is met }  \\
0 & \text{ otherwise}
\end{cases} $.
\item Define a function $f_X$ which is the p.d.f. of the target distribution.
\item Define a function $g_Y$ which is the p.d.f. of the importance distribution.
\item Use these functions in conjunction with Equation (\ref{eq:imp_estimator}).
\end{enumerate}

\begin{problem}
The joint normal distribution of $N$ independent random variables with mean 0 and variance 1 is
\[
f_X(\x) = \frac{1}{\sqrt{(2 \pi)^N}} e^{-(\x^T\x)/2}.
\]
The integral of $f_X(\x)$ over a box is the probability that a draw from the distribution will be in the box.
However, $f_X(\x)$ does not have a symbolic antiderivative.

\begin{comment}
\item The integral of this function on $B = [-1,1]\times [-1,1]\times[-1,1] \subset \mathbb{R}^3$ can be computed in SciPy with the following code.
\begin{lstlisting}
>>> import scipy.stats as stats

# Define the bounds of the box to integrate over
>>> mins = np.array([-1, -1, -1])
>>> maxs = np.array([1, 1, 1])

# Each variable has mean 0
>>> means = np.zeros(3)

# The covariance matrix of N independent random variables
#    is the NxN identity matrix.
>>> covs = np.eye(3)

# Compute the integral
>>> value, inform = stats.mvn.mvnun(mins, maxs, means, covs)
\end{lstlisting}
Then \li{value} is the integral of $f(\x)$ on $B$.
Use SciPy to integrate $f(\x)$ on $\Omega=[-0.5, 0.75]\times[0,1]\times[0, 0.5]\times[0,1] \subset \mathbb{R}^4$.
\end{comment}

Use what you have learned about importance sampling to estimate the probability that a given random variable in $\mathbb{R}^2$ generated by $f_X$ will be less than -1 in the x-direction and greater than 1 in the y-direction.

Treat $f_X$ as the p.d.f. of your target distribution.
Use the function \li{stats.multivariate_normal} to create a multivariate normal distribution to serve as your importance distribution.
For more information on how to use this function, consult the documentation for
\li{stats.multivariate_normal}.
\end{problem}

\section*{Unnormalized Target Densities} % ====================================

The methods discussed so far are only applicable if the target density is normalized, or in other words, has an integral of 1. If the target density is not normalized, Equation \ref{eq:importance} becomes

\begin{equation*} \label{eq:unnormalized}
\begin{split}
E[h(X)] & = \frac{\int h(t)f(t)\,dt}{\int f(t)\,dt} \\
& = \frac{\int h(t)f(t) \left ( \frac{g_Y(t)}{g_Y(t)} \right )\,dt}{{\int f(t)} \left ( \frac{g_Y(t)}{g_Y(t)} \right )\,dt} \\
& = \frac{\int \left ( \frac{h(t)f(t)}{g_Y(t)} \right ) g_Y(t)\,dt}{\int \left ( \frac{f(t)}{g_Y(t)} \right ) g_Y(t)\,dt} \\
& = \frac{E\left [ \frac{h(Y)f(Y)}{g_Y(Y)}\right ]}{E\left [ \frac{f(Y)}{g_Y(Y)}\right ]}
\end{split}
\end{equation*}

The corresponding estimator becomes
\begin{equation*}
\begin{split}
\widehat{E}_n[h(X)] & = \frac{\widehat{E}\left [ \frac{h(Y)f(Y)}{g_Y(Y)}\right ]}{\widehat{E}\left [ \frac{f(Y)}{g_Y(Y)}\right ]} \\
& = \frac{\frac{1}{N}\sum_{i = 1}^{N}\frac{h(y_i)f(y_i)}{g_Y(y_i)}}{\frac{1}{N}\sum_{i = 1}^{N}\frac{f(y_i)}{g_Y(y_i)}} \\
\end{split}
\end{equation*}


In this chapter, we will discuss a number of activation functions.  We
say an activation function $\alpha$
satisfies the basic activated-approximation-property (AAP) if for any $g\in C^1[-1,1]$, we have
\begin{equation}
\label{AAP}
\min_{a_i,b_i,c,w_i\in\mathbb R^1}\max_{t\in[-1,1]}\bigg|g(t)-\sum_{i=1}^k(a_i\alpha(w_it+b_i)-c\bigg|
\le \frac{C}{k}\max_{t\in[-1,1]}|g'(t)|
\end{equation}
for some constant $C$ independent of $k$ and $g$.

\section{Cardinal B-splines}
The {\it cardinal B-splines} are given by the following recurrent
relationship
\begin{equation}
  \label{cardinal}
M_d(x)=\frac{x}{d}M_{d-1}(x)  + \frac{d+1-x}{d}M_{d-1}(x-1)
\end{equation}
\subsection{$d=0$}
\begin{equation}
  \label{cardinal}
M_0(x)=
\left\{
  \begin{array}{ll}
0 & x<0 \\
1 & 0\le x<1    \\
0 & x > 1
  \end{array}
\right.
\end{equation}
The Heaviside function is defined by
\begin{equation}
  \label{a0}
\alpha_0(x)=
\left\{
  \begin{array}{ll}
0 & x<0; \\
1 & x \ge 1.
  \end{array}
\right.
\end{equation}
\begin{lemma}
  \begin{equation}
  \label{a0M0}
\alpha_0(x)=\sum_{j=0}^\infty M_0(x-j)
  \end{equation}
  \begin{equation}
  \label{M0a0}
M_0(x)=\alpha_0(x)-\alpha_0(x-1).
  \end{equation}
\end{lemma}

\subsection{$d=1$}
\begin{equation}
  \label{M1M0}
M_1(x)=xM_0(x)+(2-x)M_0(x-1)
\end{equation}
We note that
\begin{equation}
  \label{M1}
M_1(x)=
\left\{
\begin{array}{cl}
0 & x<0; \\
x & 0\le x < 1\\
2-x & 1\le x \le 2\\
0 & x>2.
  \end{array}
\right.
\end{equation}
We define (see Fig. \ref{alpha1})
\begin{equation}
  \label{a1}
\alpha_1(x)=
\left\{
\begin{array}{cl}
M_1(x) &  x < 1\\
1 & x\ge 1
  \end{array}
\right.
\end{equation}
\begin{figure}[!htb]
	\center{\includegraphics[width=10cm] {figures/alpha1.png}}
	\caption{The activation $\alpha_1$}
	\label{alpha1}
\end{figure}

\begin{lemma}
  \begin{equation}
    \label{a1a0}
\alpha_1(x)=x\alpha_0(x)+(1-x)\alpha_0(x-1).
  \end{equation}
\end{lemma}

\begin{lemma}
  \begin{equation}
  \label{a1M1}
\alpha_1(x)=\sum_{j=0}^\infty M_1(x-j)
  \end{equation}
\end{lemma}

\begin{lemma}
  \begin{equation}
    \label{M1a1}
M_1(x)=\alpha_1(x)-\alpha_1(x-1).
  \end{equation}
\end{lemma}
Consider the following
$$
\alpha_1(x)-\alpha_1(2x-1).
$$
We note that the so-called ReLU is defined as follows:
\begin{equation}
  \label{ReLU}
ReLU(x)=
\left\{
\begin{array}{cl}
0 & x<0; \\
x & x\ge 0
  \end{array}
\right.
\end{equation}
and
\begin{equation}
\alpha_1(x)=ReLU(x)-ReLU(x-1).
\end{equation}

\begin{equation}
\alpha_1({1 \over h_1} x) - \alpha_1({1 \over h_2} (x - h_1) )
\end{equation}

\newpage
\subsection{$d=2$}
\begin{equation}
  \label{M2}
M_2(x)=\frac{x}{2}M_{1}(x)  + \frac{3-x}{2}M_{1}(x-1)
\end{equation}
Note that
$$
M_2(0)=0, M_2(1)={1\over 2}, M_2({3\over2})={3\over 4},
M_2(2)={1\over2}, M_2(3)=0.
$$
Now we define
\begin{equation}
  \label{a2}
\alpha_2(x)=
\left\{
\begin{array}{cl}
{4\over3}M_2(x) &  x < {3\over 2}\\
1 & x\ge {3\over2}
  \end{array}
\right.
\end{equation}
Note that
$$
\alpha_2(0)=0, \alpha_2(1)={1\over 3}, \alpha_2({3\over2})=\alpha_2(2)=\alpha_2(3)=1.
$$
\begin{lemma}
 \begin{equation}
    \label{M2a2}
M_2(x)={3\over 2}(\alpha_2(x)-\alpha_2(x-1)).
  \end{equation}
\end{lemma}
\begin{proof}
 We need to check some details ...
\end{proof}

\subsection{$d=3$}
The presentation below follows the file lect-spline.pdf found in the following web page
\begin{verbatim}
https://www.geos.ed.ac.uk/~yliu23/docs/lect_spline.pdf
\end{verbatim}
Take $x_0=0$ and $h=1$, we get
$$
B_0(x)=
\left\{
  \begin{array}{ll}
0 & x\le -2\\
\\
{1\over 6}(2+x)^3 & -2\le x\le -1\\
\\
{2\over 3}-{1\over2}x^2(2+x) & -1\le x\le 0\\
\\
{2\over 3}-{1\over2}x^2(2-x) & 0\le x\le 1\\
\\
{1\over 6}(2-x)^3 & 1\le x\le 2\\
\\
 0 & x\ge 2.
  \end{array}
\right.
$$
Let
$$
B_k(x)=B_0(x-k)
$$
A cubic spline function in $[0,N]$ can be written as
$$
S(x)=\sum_{k=-1}^{N+1}a_kB_0(x-k).
$$
The following identity holds:
\begin{equation}
  \label{eq:1}
\sum_{k=-1}^{N+1}B_0(x-k) =1, \quad\forall x\in [0,N]
\end{equation}
We propose the following activation function
$$
\sigma(x)=
\left\{
  \begin{array}{ll}
B_1(x) & x\le 1 \\
1 & x\ge 1
  \end{array}
\right.
$$

\section{Approximation properties}
We consider an interval $I =(0,1)$ and a partition
\begin{equation}
  \label{1d-partition}
-1=t_0  < t_1<\ldots<t_k=1.
\end{equation}
As a special case of uniform partition, we take
\begin{equation}
\label{tk}
t_i=t_0+ih, I_i=(t_{i-1}, t_i)\quad i=1:k, h={2\over k}.
\end{equation}
Consider the basis function
$$
M_{0,i}(t)=M_0(\frac{t-t_{i-1}}{h}).
$$
Given
$$
v: (-1,1)\mapsto \mathbb R^1
$$
The interpolation is defined as
\begin{equation}
  \label{interp0}
(\Pi_0v)(t) =\sum_{i=1}^kv_iM_{0,i}(t)
=v_1+\sum_{i=2}^k(v_i-v_1)M_{0,i}(t)
\end{equation}
with
$$
v_i={1\over h}\int_{I_i}v(t).
$$
By \eqref{M0a0}, we have
\begin{eqnarray}
(\Pi_0v)(t)
&=&v_1+\sum_{i=2}^k(v_i-v_1)(\alpha_{0}(\frac{t-t_{i-1}}{h})-\alpha_{1}(\frac{t-t_{i}}{h}))\\
&=&v_1+\sum_{i=2}^k(v_i-v_1)(\alpha_{0}(\frac{t-t_{i-1}}{h})-\alpha_{0}(\frac{t-t_{i}}{h}))\\
&=&v_1+\sum_{i=1}^{k-1}(v_{i+1}-v_1)\alpha_{0}(\frac{t-t_{i}}{h})-
\sum_{i=2}^k(v_i-v_1)\alpha_{0}(\frac{t-t_{i}}{h})\\
&=&v_1+\sum_{i=1}^{k}(v_{i+1}-v_i)\alpha_{0}(\frac{t-t_{i}}{h}).
\end{eqnarray}
with
$$
v_{k+1}=v_{1}.
$$
\begin{theorem}
  \label{M0-error}
  \begin{equation}
  \label{Pi0-error}
\|v-\Pi_0v\|_{0,\infty}\le \frac{c_0}{k}\|v'\|_{0,\infty}
\end{equation}
\end{theorem}
\begin{proof}
Easy.
\end{proof}
In general, we consider the following space of Splines:
\begin{theorem}  \label{Md-error}

\subsection{General $d$}
Let $S^{d,k}$ be the spline space generated by the B-spline $M_d$ from
the partition \eqref{1d-partition}, we have
  \begin{equation}
  \label{Pid-error}
\|v-\Pi_{d,k}v\|_{0,\infty}\le \frac{c_d}{k^r}\|v^{(r)}\|_{0,\infty},
\quad 1\le r\le d+1.
\end{equation}
\end{theorem}
\begin{proof}
	You can find this proof in \cite{de1978practical} in theorem $XII.3$ in page 176.
It needs to be checked.  Li Lin might know where a proof can be found
in the literature.
\end{proof}

\subsection{Sigmoidal function}
The so-called sigmoidal function is defined as follows:
\begin{equation}
  \label{sigmoidal}
\sigma(t) = \frac{1}{1 + e^{-t}}.
\end{equation}
This popular activation provides a smooth approximation of the Heaviside function $\alpha_0$ as follows:
\begin{equation}
  \label{sig}
\lim_{a\to \infty}\sigma(at) = \alpha_0(t), \quad t\neq 0.
\end{equation}
For $t>0$
$$
\alpha_0(t)- \sigma(at)
=1-\frac{1}{1+e^{-at}}\le e^{-at}.
$$
For $t<0$
$$
\sigma(at)-\alpha_0(t)
=\frac{1}{1+e^{-at}}\le e^{-a|t|}.
$$
We have in general
$$
|\sigma(at)-\alpha_0(t)|
\le e^{-a|t|}.
$$
We consider the following interpolation
\begin{equation}
  \label{interp0-1}
v(t)=v_1+\sum_{i=1}  (v_i-v_1)
\end{equation}

%\newpage
\section{Special activation functions}

        \begin{itemize}
	\item An general activation function(must be nonlinear) is
$$\sigma: \mathbb{R} \to  \mathbb{R}.$$

\item The Heaviside function is
$$
H(x ) = \begin{cases}
0 \quad &\text{if} ~ x \le 0, \\
1 \quad &\text{if} ~ x > 0.
\end{cases}
$$
The biggest problem for this activation function is that this function is not continuous which will cause
huge difficult in training phase.

\item The sigmoid function:
$$s(x) = \frac{1}{1 + e^{-x}}
\rightarrow
\begin{cases}
0,~~x\rightarrow -\infty,\\
1,~~x\rightarrow +\infty.
\end{cases}
.
$$
This function can be seen as the smooth approximation of Heaviside function.
This activation function was very popular in shallow neural network in about 1990s. Now, this
activation function is also often used in RNN or some NLP tasks.

\item Currently, the most used activation function in DNN  and CNN is ``Rectified Linear Unit'' (ReLU):
$$
{\rm ReLU}(x) = \max(0, x).
$$
There are many interesting properties of ${\rm ReLU}$ function:
\begin{enumerate}
	\item ${\rm ReLU}$ is a piecewise linear function. Thus, DNN with this activation
	function is always a piecewise linear function.

	\item The connection of ${\rm ReLU}$ and Heaviside.
	\begin{equation}
	\frac{d}{dx} {\rm ReLU}(x) = H(x).
	\end{equation}

	\item Recently, there are huge research works about the approximation properties of DNN
	with ${\rm ReLU}$ activation function see ~\cite{he2018relu,wang2018exponential,yarotsky2017error}.
\end{enumerate}

	\item The new activation function from ReLU is:
	$$
	\tau(x) = r(x) - r(x-1) = \begin{cases}
	0 \quad &\text{if} ~ x \le 0, \\
	x \quad &\text{if} ~  0 < x \le 1, \\
	1  \quad &\text{if} ~ x > 1.
	\end{cases}
	$$
        Here we need to note that, because $-r(x) \neq r(ax + b)$, so
        if we use $\tau$ as $f_{out}$, the layers will be $J+1$ for
        ReLU as activation function and $f_{out} = id$.
	\end{itemize}

Here is a simple diagram for a general DNN structure:
\begin{figure}[!h]
	\center{\includegraphics[width=12cm,height=6cm] {ANN.png}}
	\caption{A General Structure of DNN}
\end{figure}


% !TeX spellcheck = en_US
\subsection{Log encoding}
This encoding utilize the fact that each propositional variable can only take one of two values, i.e. either True or False.

\subsubsection{Encoding the variables}
First we encode each possible value from the domain with a distinct binary representation. Depending on the maximum number of bits needed to encoded all the domain values, an equal number of propositional variables for each CSP variable is generated. So if the size of domain was $m$ and the number of CSP variables was $n$, we need $n \lceil \log_2 m \rceil$ propositional variable on the SAT side.

Notice that encoding each value from the CSP variable's domain means that the size of this domain must be exactly equal to $2^i$, which is not necessary the case. So extra possible binary representations can emerge (implied in the formula by the ceiling function, i.e. if $ \lceil \log_2 m \rceil > \log_2 m $). To avoid assigning these invalid values to any CSP variable we have to include extra clauses to ensure that a variable can't be assigned a value outside its domain.

On the other hand, unlike the direct encoding we do not need any additional clauses to ensure that each CSP variable is given a value at all (ALO clauses) nor to ensure that each CSP variable is given only one value (AMO clauses).

The same example from above is used to demonstrate the additional needed clauses. The CSP constraints are not listed explicitly but rather illustrated by the borders on the map.

\begin{figure}[H]
	\centering
	\includegraphics[width=0.85\linewidth]{assets/log_variables}
	\captionsetup{justification=centering,margin=2cm}
	\caption{Encoding CSP variables using log encoding}
	\label{fig:log_variables}
\end{figure}

Notice that the log encoding generates less propositional variables than direct encoding. Anyway, this fact does not improve the performance of DPLL on the generated SAT instance because more steps are required to untangle the problem as discussed later. \ref{subsec:log_proposition}

\subsubsection{Encoding the constraints}
As discussed before, constraints could be unary, binary or n-ary. Since any n-ary constraint can be expanded to multiple binary constraints, the main focus will be on encoding only the unary and the binary constraints.

Encoding unary constraints is straight forward. For example to ensure that a CSP variable $X$ can't be assigned a certain value from its domain encoded in the binary form $01$, we only need to add the following clause to the SAT formula: $\neg (\neg X_1 \wedge X_0) \equiv X_1 \vee \neg X_0$. Remember that the subscript of the propositional variables $X_0, X_1$ corresponds to the position of the bit in the binary form of the CSP value.

Encoding binary constraints can be done in similar manner. For example to encode the CSP constraint $A \neq D$, where $A$ and $D$ can be either green \textcolor{green}{$\blacksquare$} or blue \textcolor{blue}{$\blacksquare$} (i.e. their domain is $\{ \textcolor{green}{\blacksquare}, \textcolor{blue}{\blacksquare} \}$) with the binary representations \textcolor{green}{$01$} and \textcolor{blue}{$10$} respectively, we need to include the following clauses in our SAT:
\begin{figure}[H]
	\centering
	\includegraphics[width=0.7\linewidth]{assets/binary_constraints_log_encoding}
	\captionsetup{justification=centering,margin=2cm}
	\caption{Encoding binary CSP constraint using log encoding}
	\label{fig:binary_constraints_log_encoding}
\end{figure}

\subsubsection{Proposition}\label{subsec:log_proposition}
In spite of the fact that log encoding generate fewer variables on the SAT side than direct encoding, it is easy to see that DPLL applied on log encoded CSP is dominated by FC applied to the original CSP problem (assuming equivalent branching heuristics).

\subsubsection{Proof idea}
The proof idea is similar to the proof mentioned earlier for the direct encoding case \ref{subsec:direct_encoding_proof}. We process by comparing the search trees of each algorithm. First we show that both trees must have at least equal number of branches. By Induction on each CSP variable $X$, FC algorithm will branch upon it. On the SAT side DPLL will consider all propositional variables corresponding to the same CSP variable ($X_i \dots X_{\lceil \log_2 m \rceil}$), where $m$ is the size of the domain.

Further more, we can exploit the natural logarithmic behavior of log encoding on the search tree to show that DPLL is \textbf{strictly} dominated by FC. This can be done using a simple example. Consider a CSP consists of two variables $A$ and $B$ with domain of size 3 and no constraints other than rolling out all assignments. FC will require three branches for each variable to show that this problem is unsatisfiable. Since the size of the domain equals 3, we need at least two bits to represent each CSP variable. Therefore, DPLL must branch at least $2 * 2 * 2 = 8$ times to show that the problem is unsatisfiable (the first 2 represents for the number of variables, the second one for the number of bits, and the last one to indicate branching on each case True or False as discussed before).

\begin{figure}[H]
	\centering
	\includegraphics[width=0.85\linewidth]{assets/log_dominated_by_fc}
	\captionsetup{justification=centering,margin=2cm}
	\caption{Seach trees of DPLL and FC for trivial CSP}
	\label{fig:log_dominated_by_fc}
\end{figure}


\section{Application: Markov matrices}

There are applications of great importance which feature a special type of matrix.  Matrices whose columns consist of non-negative numbers that sum to one are
called \textbf{Markov matrices}\index{Markov matrices}. An important application of Markov matrices is
in population migration, as illustrated in the following definition.

\begin{definition}{Migration matrices}{migration-matrix}
Let $m$ locations be denoted by the numbers $1,2,\ldots,m$. Suppose
it is the case that each year the proportion of residents in location
$j$ which move to location $i$ is $a_{ij}$. Also suppose no one
escapes or emigrates from without these $m$ locations. This last
assumption requires $\sum_{i}a_{ij}=1$, and means that the matrix $A$,
such that $A = \mat{a_{ij} }$, is a Markov matrix. In this context,
$A$ is also called a \textbf{migration matrix}\index{migration matrix}.
\end{definition}

Consider the following example which demonstrates this situation.

\begin{example}{Migration matrix}{migration-matrix}
Let $A$ be a Markov matrix given by
\begin{equation*}
A =
\begin{mymatrix}{rr}
0.4 & 0.2 \\
0.6 & 0.8
\end{mymatrix}
\end{equation*}
Verify that $A$ is a Markov matrix and describe the entries of $A$ in terms of population migration.
\end{example}

\begin{solution}
The columns of $A$ are comprised of non-negative numbers which sum to $1$. Hence, $A$ is a Markov matrix.

Now, consider the entries $a_{ij}$ of $A$ in terms of population. The
entry $a_{11} = 0.4$ is the proportion of residents in location one
which stay in location one in a given time period.  Entry $a_{21} =
0.6$ is the proportion of residents in location 1 which move to
location 2 in the same time period. Entry $a_{12} = 0.2$ is the
proportion of residents in location 2 which move to location
1. Finally, entry $a_{22} = 0.8$ is the proportion of residents in
location 2 which stay in location 2 in this time period.

Considered as a Markov matrix, these numbers are usually identified
with probabilities. Hence, we can say that the probability that a
resident of location one will stay in location one in the time period
is $0.4$.
\end{solution}

Observe that in Example~\ref{exa:migration-matrix} if there was initially say 15
thousand people in location 1 and 10 thousands in location 2, then
after one year there would be $0.4 \times 15 + 0.2 \times 10 = 8$
thousands people in location 1 the following year, and similarly
there would be $0.6 \times 15 + 0.8 \times 10 = 17$
thousands people in location 2 the following year.

More generally let $X_n=\mat{x_{1n}, \ldots, x_{mn}} ^{T}$ where $x_{in}$ is the
population of location $i$ at time period $n$. We call $X_n$ the \textbf{state vector at period $n$}\index{state vector}. In particular, we call $X_0$ the initial state vector. Letting $A$ be the migration matrix, we compute the population in each location $i$ one time period later by $AX_n$. In order to find the population of location $i$ after $k$
years, we compute the $i\th$ component of $A^{k}X$. This discussion is summarized in the following theorem.

\begin{theorem}{State vector}{state-vector}
Let $A$ be the migration matrix of a population and let $X_n$ be the vector whose entries give the population of each location at time period $n$. Then $X_n$ is the state vector at period $n$ and it follows that
\[
X_{n+1} = A X_n
\]
\end{theorem}

The sum of the entries of $X_n$ will equal the sum of the entries of the initial
vector $X_{0}$. Since the columns of $A$ sum to $1$, this sum is preserved for every
multiplication by $A$ as demonstrated below.
\begin{equation*}
\sum_{i}\sum_{j}a_{ij}x_{j}=\sum_{j}x_{j}\paren{\sum_{i}a_{ij}}
=\sum_{j}x_{j}
\end{equation*}

Consider the following example.

\begin{example}{Using a migration matrix}{using-migration-matrix}
Consider the migration matrix
\begin{equation*}
A =
\begin{mymatrix}{rrr}
0.6 & 0 & 0.1 \\
0.2 & 0.8 & 0 \\
0.2 & 0.2 & 0.9
\end{mymatrix}
\end{equation*}
 for locations $1,2$, and $3$. Suppose initially there are $100$
residents in location $1$, $200$ in location $2$ and $400$ in location $3$. Find the
population in the three locations after $1,2$, and $10$ units of time.
\end{example}

\begin{solution}
Using Theorem~\ref{thm:state-vector} we can find the population in each location using the equation $X_{n+1} = AX_n$. For the population after $1$ unit, we calculate $X_1 = AX_0$ as follows.
\begin{eqnarray*}
X_1 &=& AX_0 \\
\begin{mymatrix}{r}
x_{11} \\
x_{21} \\
x_{31}
\end{mymatrix}
&=&
\begin{mymatrix}{rrr}
0.6 & 0 & 0.1 \\
0.2 & 0.8 & 0 \\
0.2 & 0.2 & 0.9
\end{mymatrix}
\begin{mymatrix}{r}
100 \\
200 \\
400
\end{mymatrix} \\
&=&
\begin{mymatrix}{r}
100 \\
180 \\
420
\end{mymatrix}
\end{eqnarray*}
Therefore after one time period, location $1$ has $100$ residents, location $2$ has $180$, and location $3$ has $420$. Notice that the \textbf{total} population is unchanged, it simply migrates within the given locations.
We find the locations after two time periods in the same way.
\begin{eqnarray*}
X_2 &=& AX_1 \\
\begin{mymatrix}{r}
x_{12} \\
x_{22} \\
x_{32}
\end{mymatrix}
&=&
\begin{mymatrix}{rrr}
0.6 & 0 & 0.1 \\
0.2 & 0.8 & 0 \\
0.2 & 0.2 & 0.9
\end{mymatrix}
\begin{mymatrix}{r}
100 \\
180 \\
420
\end{mymatrix} \\
&=&
\begin{mymatrix}{r}
102 \\
164 \\
434
\end{mymatrix}
\end{eqnarray*}

We could progress in this manner to find the populations after $10$ time periods. However from our above discussion, we can simply calculate $(A^{n}X_0) _{i}$,
where $n$ denotes the number of time periods which have passed. Therefore, we compute the populations in each location after $10$ units of time as follows.
\begin{eqnarray*}
X_{10} &=& A^{10}X_0 \\
\begin{mymatrix}{r}
x_{1 10} \\
x_{2 10} \\
x_{3 10}
\end{mymatrix}
&=&
\begin{mymatrix}{rrr}
0.6 & 0 & 0.1 \\
0.2 & 0.8 & 0 \\
0.2 & 0.2 & 0.9
\end{mymatrix} ^{10}\begin{mymatrix}{r}
100 \\
200 \\
400
\end{mymatrix} \\
&=&  \begin{mymatrix}{c}
115.\,\allowbreak 085\,829\,22 \\
120.\,\allowbreak 130\,672\,44 \\
464.\,\allowbreak 783\,498\,34
\end{mymatrix}
\end{eqnarray*}
Since we are speaking about populations, we would need to round these numbers to provide a logical
answer. Therefore, we can say that after $10$ units of time, there will be $115$ residents in location one, $120$ in location two,
and $465$ in location three.
\end{solution}

A second important application of Markov matrices is the concept of random walks\index{random walk}. Suppose a walker has $m$ locations to choose from, denoted $1, 2,\ldots, m$. Let $a_{ij}$ refer to the probability that the person will travel \textbf{to}  location $i$ \textbf{from} location $j$. Again, this requires that
\[
\sum_{i=1}^{k}a_{ij}=1
\]
In this context, the vector $X_n=\mat{x_{1n}, \ldots, x_{mn}} ^{T}$ contains the probabilities $x_{in}$ the walker ends up in location $i, 1\leq i \leq m$ at time $n$.

\begin{example}{Random walks}{random-walks}
Suppose three locations exist, referred to as locations $1, 2$ and $3$. The Markov matrix of probabilities $A = [a_{ij}]$ is given by
\[
\begin{mymatrix}{rrr}
0.4 & 0.1 & 0.5 \\
0.4 & 0.6 & 0.1 \\
0.2 & 0.3 & 0.4
\end{mymatrix}
\]
If the walker starts in location $1$, calculate the probability that he ends up in location $3$ at time $n = 2$.
\end{example}

\begin{solution}
Since the walker begins in location $1$, we have
\[
X_{0} = \begin{mymatrix}{c}
1 \\
0 \\
0
\end{mymatrix}
\]
The goal is to calculate $x_{32}$. To do this we calculate $X_{2}$, using $X_{n+1} = AX_{n}$.
\begin{eqnarray*}
X_{1} &=& A X_{0} \\
&=& \begin{mymatrix}{rrr}
0.4 & 0.1 & 0.5 \\
0.4 & 0.6 & 0.1 \\
0.2 & 0.3 & 0.4
\end{mymatrix} \begin{mymatrix}{c}
1 \\
0 \\
0
\end{mymatrix} \\
&=&
\begin{mymatrix}{r}
0.4 \\
0.4 \\
0.2
\end{mymatrix}
\end{eqnarray*}
\begin{eqnarray*}
X_{2} &=& A X_{1} \\
&=& \begin{mymatrix}{rrr}
0.4 & 0.1 & 0.5 \\
0.4 & 0.6 & 0.1 \\
0.2 & 0.3 & 0.4
\end{mymatrix} \begin{mymatrix}{c}
0.4 \\
0.4 \\
0.2
\end{mymatrix} \\
&=&
\begin{mymatrix}{r}
0.3 \\
0.42 \\
0.28
\end{mymatrix}
\end{eqnarray*}
This gives the probabilities that our walker ends up in locations 1, 2, and 3. For this example we are interested in location 3, with a probability on $0.28$.
\end{solution}

Returning to the context of migration, suppose we wish to know how many residents will be in a certain location
after a very long time. It turns out that if some power of the
migration matrix has all positive entries, then there is a vector $X_s$ such that $A^{n}X_{0}$ approaches $X_s$ as $n$ becomes very large. Hence as more time passes and $n$ increases, $A^{n}X_{0}$ will become closer to the vector $X_s$.

Consider Theorem~\ref{thm:state-vector}. Let $n$ increase so that $X_n$ approaches $X_s$. As $X_n$ becomes closer to $X_s$, so too does $X_{n+1}$. For sufficiently large $n$, the statement $X_{n+1} = AX_n$ can be written as $X_s = AX_s$.

This discussion motivates the following theorem.

\begin{theorem}{Steady state vector}{steady-state}
Let $A$ be a migration matrix. Then there exists a \textbf{steady state vector} written $X_s$ such that
\[
X_s = AX_s
\]
where $X_s$ has positive entries which have the same sum as the entries of $X_0$.

As $n$ increases, the state vectors $X_n$ will approach $X_s$.
\end{theorem}

Note that the condition in Theorem~\ref{thm:steady-state} can be written as $(I - A)X_s=0$, representing a homogeneous system of equations.

Consider the following example. Notice that it is the same example as the Example~\ref{exa:using-migration-matrix} but here it will
involve a longer time frame.

\begin{example}{Populations over the long run}{long-run-population}
Consider the migration matrix
\begin{equation*}
A
=
\begin{mymatrix}{rrr}
0.6 & 0 & 0.1 \\
0.2 & 0.8 & 0 \\
0.2 & 0.2 & 0.9
\end{mymatrix}
\end{equation*}
 for locations $1,2$, and $3$. Suppose initially there are 100
residents in location 1, 200 in location 2 and 400 in location 4. Find the
population in the three locations after a long time.
\end{example}

\begin{solution}
By Theorem~\ref{thm:steady-state} the steady state vector $X_s$ can be found by solving the system $(I-A)X_s = 0$.

Thus we need to find a solution to
\begin{equation*}
\paren{\begin{mymatrix}{rrr}
1 & 0 & 0 \\
0 & 1 & 0 \\
0 & 0 & 1
\end{mymatrix} -\begin{mymatrix}{rrr}
0.6 & 0 & 0.1 \\
0.2 & 0.8 & 0 \\
0.2 & 0.2 & 0.9
\end{mymatrix}} \begin{mymatrix}{c}
x_{1s} \\
x_{2s}\\
x_{3s}
\end{mymatrix} =\begin{mymatrix}{c}
0 \\
0 \\
0
\end{mymatrix}
\end{equation*}
The augmented matrix and the resulting {\rref} are given by
\begin{equation*}
\begin{mymatrix}{rrr|r}
0.4 & 0 & -0.1 & 0 \\
-0.2 & 0.2 & 0 & 0 \\
-0.2 & -0.2 & 0.1 & 0
\end{mymatrix}
\roweq\ldots\roweq
\begin{mymatrix}{rrr|r}
1 & 0 & -0.25 & 0 \\
0 & 1 & -0.25 & 0 \\
0 & 0 & 0 & 0
\end{mymatrix}
\end{equation*}
Therefore, the eigenvectors are
\begin{equation*}
t\begin{mymatrix}{c}
0.25 \\
0.25 \\
1
\end{mymatrix}
\end{equation*}

The initial vector $X_0$ is given by
\begin{equation*}
\begin{mymatrix}{r}
100 \\
200 \\
400
\end{mymatrix}
\end{equation*}

Now all that remains is to choose the value of $t$ such that
\begin{equation*}
0.25t+0.25t+t=100+200+400
\end{equation*}
Solving this equation for $t$ yields $t=\vspace{.05in} \frac{1400}{3}$. Therefore the population in the long run
is given by
\begin{equation*}
\vspace{.05in} \frac{1400}{3}\begin{mymatrix}{c}
0.25 \\
0.25 \\
1
\end{mymatrix} = \begin{mymatrix}{c}
116. 666\,666\,666\, 666\,7 \\
116. 666\,666\,666\, 666\,7 \\
466. 666\,666\,666\, 666\,7
\end{mymatrix}
\end{equation*}

Again, because we are working with populations, these values need to be rounded. The steady state vector $X_s$ is given by
\[
\begin{mymatrix}{c}
117 \\
117 \\
466
\end{mymatrix}
\]
\end{solution}

We can see that the numbers we calculated in Example~\ref{exa:using-migration-matrix} for the populations
after the $10\th$ unit of time are not far from the long term values.

Consider another example.

\begin{example}{Populations after a long time}{population-after-long-time}
Suppose a migration matrix is given by
\begin{equation*}
A = \begin{mymatrix}{ccc}
\vspace{.05in} \frac{1}{5} & \vspace{.05in} \frac{1}{2} & \vspace{.05in}
\frac{1}{5} \\
\vspace{.05in} \frac{1}{4} & \vspace{.05in} \frac{1}{4} & \vspace{.05in}
\frac{1}{2} \\
\vspace{.05in} \frac{11}{20} & \vspace{.05in} \frac{1}{4} & \vspace{.05in}
\frac{3}{10}
\end{mymatrix}
\end{equation*}
 Find the comparison between the populations in the three
locations after a long time.
\end{example}

\begin{solution}
In order to compare the populations in the long term, we want to find the steady state vector $X_s$.
Solve
\begin{equation*}
\paren{\begin{mymatrix}{rrr}
1 & 0 & 0 \\
0 & 1 & 0 \\
0 & 0 & 1
\end{mymatrix} -\begin{mymatrix}{ccc}
\vspace{.05in} \frac{1}{5} & \vspace{.05in} \frac{1}{2} & \vspace{.05in}
\frac{1}{5} \\
\vspace{.05in} \frac{1}{4} & \vspace{.05in} \frac{1}{4} & \vspace{.05in}
\frac{1}{2} \\
\vspace{.05in} \frac{11}{20} & \vspace{.05in} \frac{1}{4} & \vspace{.05in}
\frac{3}{10}
\end{mymatrix}} \begin{mymatrix}{c}
x_{1s} \\
x_{2s} \\
x_{3s}
\end{mymatrix} =\begin{mymatrix}{c}
0 \\
0 \\
0
\end{mymatrix}
\end{equation*}
The augmented matrix and the resulting {\rref} are given by
\begin{equation*}
\begin{mymatrix}{rrr|r}
\vspace{.05in} \frac{4}{5} & -\vspace{.05in} \frac{1}{2} & -\vspace{.05in}
\frac{1}{5} &  0 \\
-\vspace{.05in} \frac{1}{4} & \vspace{.05in} \frac{3}{4} & -\vspace{.05in}
\frac{1}{2} &  0 \\
-\vspace{.05in} \frac{11}{20} & -\vspace{.05in} \frac{1}{4} & \vspace{.05in}
\frac{7}{10} &  0
\end{mymatrix}
\roweq\ldots\roweq
\begin{mymatrix}{rrr|r}
1 & 0 & -\vspace{.05in} \frac{16}{19} & 0 \\
0 & 1 & -\vspace{.05in} \frac{18}{19} & 0 \\
0 & 0 & 0 & 0
\end{mymatrix}
\end{equation*}
and so an eigenvector is
\begin{equation*}
\begin{mymatrix}{c}
16 \\
18 \\
19
\end{mymatrix}
\end{equation*}

Therefore, the proportion of population in location 2 to location 1 is given by $\vspace{.05in} \frac{18}{16}$.
The proportion of population 3 to location 2 is given by $\vspace{.05in} \frac{19}{18}$.
\end{solution}


\section*{Chapter 3: Ancient Greek Number Theory}

\paragraph{Exercise 3.6}
Prove that if $n$ and $m$ are coprime, then $\DivSum{nm} =
\DivSum{n} \, \DivSum{m}$

\begin{proof}
Let the prime factorization of $nm$ be
$nm = p_1^{\alpha_1} \dots p_k^{\alpha_k}$.
Being $\GCD{n}{m} = 1$, if $\Divides{p_i}{n}$ then $\NotDivides{p_i}{m}$
(and viceversa), $1 \leq i \leq k$. In consequence, if the prime factorization
of $n$ is $n = q_1^{\beta_1} \dots q_l^{\beta_l}$, then any $q_i$ cannot appear
in the prime factorization of $m$. That is, if the prime
factorization of $m$ is $m = r_1^{\gamma_1} \dots r_s^{\gamma_s}$,
then $q_i \neq r_j$, $1 \leq i \leq l$, $1 \leq j \leq s$. Thus,
\begin{eqnarray*}
\DivSum{nm} &=& \DivSum{p_1^{\alpha_1} \dots p_k^{\alpha_k}} \\
            &=& \prod_{i = 1}^{k}{\frac{p_i^{\alpha_i + 1} - 1}{p_i - 1}} \\
            &=& \prod_{i = 1}^{l}{\frac{q_i^{\beta_i + 1} - 1}{q_i - 1}} \,
                \prod_{j = 1}^{s}{\frac{r_i^{\gamma_i + 1} - 1}{r_i - 1}} \\
            &=& \DivSum{q_1^{\beta_1} \dots q_l^{\beta_l}} \,
                \DivSum{r_1^{\gamma_1} \dots r_s^{\gamma_s}} \\
            &=& \DivSum{n} \, \DivSum{m}
\end{eqnarray*}
\end{proof}

\paragraph{Exercise 3.7}
Prove that every even perfect number is a triangular number.

\begin{proof}
Let $k$ be an even perfect number. Then, by the Euclid-Euler theorem,
$k = 2^{n-1} (2^n - 1)$ for some $n \in \Nat$, where $2^n - 1$ is prime.
Thus,
\begin{eqnarray*}
k &=& 2^{n-1} (2^n - 1)\\
  &=& (2^n - 1) (2^n / \, 2) \\
  &=& \frac{(2^n - 1) 2^n}{2} \\
  &=& \triangle_{2^n - 1}
\end{eqnarray*}

\end{proof}

\paragraph{Exercise 3.8}
Prove that the sum of the reciprocals of the divisors of a perfect number is
always 2.

\begin{proof}
Let $n$ be a perfect number with divisors $d_1,\dots,d_k$. By definition of
perfect number, we have that
$$\DivSum{n} = d_1 + \dots + d_k = 2n$$
which implies that
$$2 = \frac{d_1 + \dots + d_k}{n} = \frac{d_1}{n} + \dots + \frac{d_k}{n}$$
Since $\Divides{d_i}{n}$, $1 \leq i \leq k$, $n = d_i q_i$. But
$\Divides{q_i}{n}$ as well, and so $q_i = d_j$. Then, $d_i / n = 1 / d_j$.
In consequence, every summand on the right-hand side of the previous equation
can be rewritten as the reciprocal of some divisor of $n$, and so
$$2 = \frac{d_1}{n} + \dots + \frac{d_k}{n} = \frac{1}{d_1} + \dots + \frac{1}{d_k}$$

\end{proof}