\chapter{Sufficient statistics}
In this chapter the theory of sufficient statistics are introduced.
First we begin with understanding of a statistic. Information about this subject can be found in \cite{casella2002statistical}. A statistic is a function that returns a summary of the data. Examples of this can be mean value and standard deviation of the sample. Lets assume our data is from a distribution depending on some parameters $\Theta$. The statistics we are interested in are those who contain information about the parameters $\Theta$. This leads us to sufficient statistics. A sufficient statistic is a statistic that captures all information about the parameters $\Theta$ and discard the rest. From \cite{casella2002statistical} we have the following.
\begin{defn}
(Sufficient statistic definition) A statistic $T(\boldsymbol{X})$ is sufficient statistic for $\Theta$ if the conditional distribution of sample $\boldsymbol{X}$ gives the value for $T(\boldsymbol{X})$ does not depend on $\Theta$.
\end{defn}
To find sufficient statistics can be difficult, however one can use the factorization theorem to find the statistics. The theorem is as follows.
\begin{theorem} % s276
(Factorization theorem) Let $f(\boldsymbol{x}| \Theta)$ denote the joint pdf or pmf of a sample $\boldsymbol{X}$. A statistic $T(\boldsymbol{X})$ is a sufficient statistic for $\Theta$ if and only if there exist functions $g(t|\Theta)$ and $h(\boldsymbol{x})$ such that, for all sample points $\boldsymbol{x}$ and all parameter points $\Theta$,
\begin{equation*}
f(\boldsymbol{x}|\Theta) = g(T(\boldsymbol{x})| \Theta)h(\boldsymbol{x}).
\end{equation*}
\end{theorem}
$h(\boldsymbol{x})$ should not be dependent on $\Theta$. Then the remaining part will be $g(T(\boldsymbol{x})| \Theta)$ and from this we can see the sufficient statistic.

\section{Sufficient statistics in NHPP}
\label{sec:sufstat}
To find the sufficient statistics for our NHPP one can use the likelihood function as shown in equation \ref{eq:like} and from \cite{lee1980testing}. The likelihood function can be found by
\begin{equation*}
P(t_1,..., t_n) = P(t_1,...,t_n|N(\tau)=n)P(N(\tau)=n).
\end{equation*}
From equation \ref{eq:NHPPdens} the probability density of a event $t_i$ given number of events $n$ is
\begin{equation*}
P(t_i|n) = \frac{\lambda(t_i)}{\Lambda(\tau)}.
\end{equation*}
Hence the joint probability given  $n$
\begin{equation*}
P(t_1,...,t_n| N(\tau) = n) = n! \prod_{i = 1}^{n} \frac{\lambda(t_i)}{\Lambda(\tau)}.
\end{equation*}
The term $n!$ is included because the time runs are ordered.
The probability of $N=n$ events is as shown in equation \ref{eq:NumNHPP} with $t = \tau$. The joint probability of  $\boldsymbol{t}$ becomes
\begin{equation*}
P(t_1,...,t_n) = e^{-\Lambda(\tau)} \frac{\Lambda(\tau)^n}{n!} n! \prod_{i=1}^n \frac{\lambda(t_i)}{\Lambda(\tau)} = e^{-\Lambda(\tau)} \prod_{i=1}^n \lambda(t_i).
\end{equation*}
From this the log likelihood function becomes
\begin{equation}
l = -\Lambda(\tau) + n\log(a) + n\log(b) + (b-1)\sum_{i=1}^{n} \log(t_i) + c\sum_{i=1}^{n} t_i.
\label{eq:loglike}
\end{equation}
From the factorization theorem we can set $u(t) = 1$. Then we get the sufficient statistics $(n,\displaystyle\sum_{i=1}^{n} t_i, \displaystyle\sum_{i=1}^{n} \log(t_i))$.
The last statistic can be rewritten to $\displaystyle\prod_{i=1}^{n} t_i$.


\subsection{Translation into a Graph}

\begin{frame}
  \frametitle{Translation into a Graph}

  \scriptsize

  The constraint $\quad x - y \leq c \quad$ says that

  \begin{center}
    ``the distance between $x$ and $y$ is at most $c$''
  \end{center}

  This can be encoded as $(x,y;c)$

  \begin{center}
    \input{arc}
  \end{center}

  \pause
  \vfill
  So, a set $\babst{\mu}$ can be encoded as a graph. Concrete example:

  \begin{columns}

    \begin{column}{.3\textwidth}
      \ra{1.3}
      $$
      \begin{array}{rcr}
	x - y & \leq & 8   \\
	y - z & \leq & -1  \\
	z - x & \leq & -6  \\
	z - w & \leq & 2   \\
	w - x & \leq & -10 \\
	w - t & \leq & 0   \\
	t - x & \leq & 3
      \end{array}
      $$
    \end{column}

    \begin{column}{.6\textwidth}
      \begin{center}
	\input{example}
      \end{center}
    \end{column}

  \end{columns}
  \vfill
  $G( V, E )$: $V = \{ x, y, z, w, t \}$, $E = \{ (x,y;8), (y,z;-1), (z,x;-6), (z,w;2), (w,x;-10), \ldots \}$

\end{frame}

\begin{frame}
  \frametitle{Translation into a Graph}

  \scriptsize

    \begin{theorem}[Translation]
      \label{the:idl}
      $\babst{\mu}$ is \Idl-unsatisfiable
      \begin{center}
      iff
      \end{center}
      there is a {\bf negative cycle} in the corresponging graph $G(V,E)$
    \end{theorem}

  \pause
  \vfill
  E.g.:

  \begin{columns}

    \begin{column}{.3\textwidth}
      \ra{1.3}
      $$
      \begin{array}{rcr}
	\coloneat{x - y}{3-} & \coloneat{\leq}{3-} & \coloneat{8}{3-}   \\
	\coloneat{y - z}{3-} & \coloneat{\leq}{3-} & \coloneat{-1}{3-}  \\
	\colfouat{z - x}{3-} & \colfouat{\leq}{3-} & \colfouat{-6}{3-}  \\
	\coloneat{z - w}{3-} & \coloneat{\leq}{3-} & \coloneat{2}{3-}   \\
	\coloneat{w - x}{3-} & \coloneat{\leq}{3-} & \coloneat{-10}{3-} \\
	\colfouat{w - t}{3-} & \colfouat{\leq}{3-} & \colfouat{0}{3-}   \\
	\colfouat{t - x}{3-} & \colfouat{\leq}{3-} & \colfouat{3}{3-}
      \end{array}
      $$
    \end{column}

    \begin{column}{.6\textwidth}
      \begin{center}
	\begin{overlayarea}{.5\textwidth}{3cm}
	  \only<2|handout:0>{\input{example}}
	  \only<3->{\input{example_hl}}
	\end{overlayarea}
      \end{center}
    \end{column}

  \end{columns}

\end{frame}

\begin{frame}
  \frametitle{Translation into a Graph}

  We use the following lemma to aid the proof

  \begin{lemma}[Farka's Lemma for \Idl]
    $\babst{\mu}$ is unsatisfiable iff there exists a subset
    $\babst{\nu} = \{\ \coltwo{x_1} - x_2 \leq c_1,\ x_2 - x_3 \leq c_2,\ \ldots,\ x_n - \coltwo{x_1} \leq c_n\ \}$ of $\babst{\mu}$
    such that $c_1 + \ldots + c_n < 0$
  \end{lemma}
  \pause
  \vfill
  The proof of Theorem is now trivial
  \begin{proof}
  $\babst{\mu}$ is unsatisfiable iff \colfou{by Farka's Lemma for \Idl} \\
  there exists $\babst{\nu} \subseteq \babst{\mu}$ with $c_1 + \ldots + c_n < 0$ iff \colfou{by our translation} \\
  there exists a negative cycle in $G(V,E)$
  \end{proof}

\end{frame}


\subsection{Pauli matrices}

Pauli matrices are \(2\times 2\) matrices which are unitary and hermitian.

That is, \(P^*=P^{-1}\).

And \(P^*=P\).

\subsubsection{The Pauli matrices}

The matrices are:

\(\sigma_1 =\begin{bmatrix} 0&1  \\ 1&0  \end{bmatrix}\)

\(\sigma_2 =\begin{bmatrix} 0&-i \\ i&0  \end{bmatrix}\)

\(\sigma_3 =\begin{bmatrix} 1&0  \\ 0&-1 \end{bmatrix}\)

The identity matrix is often considered alongside these as:

\(\sigma_0 =\begin{bmatrix} 1&0  \\ 0&1  \end{bmatrix}\)

\subsubsection{Pauli matrices are their own inverse}

\(\sigma_i^2 =\sigma_i\sigma_i\)

\(\sigma_i^2 =\sigma_i\sigma_i^*\)

\(\sigma_i^2 =\sigma_i\sigma_i^{-1}\)

\(\sigma_i^2 =I\)

\subsubsection{Determinants and trace of Pauli matrices}

\(\det \sigma_i =-1\)

\(Tr (\sigma_i) =0\)

As the sum of eigenvalues is the trace, and the product is the determinant, the eigenvalues are \(1\) and \(-1\).


\subsection{Interest}

\subsubsection{Interest}

An alternative measure for finding rules is to use interest.

\(Interest(A\rightarrow B)=\dfrac{P(A\land B)}{P(A)P(B)}\)

\(Interest(A\rightarrow B)=\dfrac{supp(A\land B)}{P(supp(A))P(supp(B))}\)

If this is \(1\), then they are independent.

If this is greater than \(1\), they are positively dependent.

If this is less than \(1\), they are negatively dependent.


%\subsection{Bilinear element}

%\noindent\textbf{Bilinear element}

Continuous bilinear finite element discretization of
\eqref{laplace} on the right mesh in
Fig. \ref{fig:2dpartition}. The discrete space for linear finite element is
$$
\mathcal V_h=\{v_h: v_h|_K\in \{1,\ x,\ y,\ xy \} \text{ and } v_h \text{ is globally continuous}\}.
$$
%It is easy to see that on the element $K$ with four vertice $(x_i, y_j)$, $(x_i,y_{j+1})$, $(x_{i+1},y_j)$ and $(x_{i+1},y_{j+1})$, the nodal basis functions
%%associate with each $(x_i,y_j)$
%(satisfying \eqref{NodalBasis}) are given by
%\begin{equation}
%  \label{BilinearNodalBasis}
%  \begin{array}{llll}
%\phi_{i,j}(x,y)&=  \frac{(x_{i+1}-x)(y_{j+1}-y)}{h^2},
%&\phi_{i, j+1}(x,y)&= \frac{(x_{i+1}-x)(y-y_j)}{h^2}, \\
%\phi_{i+1,j}(x,y)&=  \frac{(x-x_i)(y_{j+1}-y)}{h^2},
%& \phi_{i+1,j+1}(x,y)&= \frac{(x-x_i)(y-y_j)}{h^2}.
%\end{array}
%\end{equation}
For bilinear element case, we have
\begin{equation}
\begin{split}
(\nabla \mathbf u_h, \nabla \mathbf v_h)&=\sum\limits_{i,j=1}^{n}\int_{E_{i,j}}\nabla \mathbf u_h, \nabla \mathbf v_h dxdy\\
&=\sum\limits_{i,j=1}^{n}\int_{E_{i,j}} \left(\frac{(u_{i+1,j}-u_{i,j})(y_{j+1}-y)}{h^2}
+\frac{(u_{i,j+1}-u_{i+1,j+1})(y-y_j)}{h^2}\right)\\
&~\qquad\qquad\left(\frac{(v_{i+1,j}-v_{i,j})(y_{j+1}-y)}{h^2}
+\frac{(v_{i,j+1}-v_{i+1,j+1})(y-y_j)}{h^2}\right)\\
&~~\quad\qquad+\left(\frac{(u_{i,j+1}-u_{i,j})(x_{i+1}-x)}{h^2}
+\frac{(u_{i+1,j}-u_{i+1,j+1})(x-x_i)}{h^2}\right)\\
&~\qquad\qquad \left(\frac{(v_{i,j+1}-v_{i,j})(x_{i+1}-x)}{h^2}
+\frac{(v_{i+1,j}-v_{i+1,j+1})(x-x_i)}{h^2}\right)dxdy\\
&=(A\ast u, v)_{l^2}.
\end{split}
\end{equation}
where $A=\left(
\begin{matrix}
-1&-1&-1\\
-1&8&-1\\
-1&-1&-1\\
\end{matrix}
\right)$
and $A\ast u$ is given by \eqref{2d-fe1}.

And we have
\begin{equation}
  \label{2d-fe1}
A\ast u=8u_{ij}-(u_{i+1,j}+u_{i-1,j}+u_{i,j+1}+u_{i,j-1}+u_{i+1,j+1}+u_{i-1,j-1}+u_{i-1,j+1}+u_{i+1,j-1})=f_{i,j},
\end{equation}
and
$
u_{i,j}=0~~\hbox{if}~~i ~~\hbox{or}~~ j\in \{0, n+1\}.
$

497-6DL/8 Convolutional Multigrid Method/8.7-bilinear.tex


\subsubsection{Area of a Closed Region}
\noindent
The area inside of $D$ is
\begin{align*}
	A &= \iint\limits_{D}{\mathrm{d}x\mathrm{d}y} \\
	&= \iint\limits_{D}{\left(\nabla \times \vec{F}\right)\mathrm{d}x\mathrm{d}y}
\end{align*}
if $\nabla \times \vec{F} = 1$. One such vector field is $\vec{F} = \langle -y/2, x/2 \rangle$.
\begin{equation*}
	= \oint_{C}{(-y/2)\mathrm{d}x + (x/2)\mathrm{d}y}
\end{equation*}
by Green's Theorem for Circulation.
\begin{equation*}
	= \frac{1}{2}\oint\limits_{C}{xy^\prime - yx^\prime}
\end{equation*}
So, if we have some counter-clockwise oriented parametric function $(x(t), y(t))$ where $t_0 \leq t \leq t_1$ that parameterizes $C$, then
\begin{equation*}
	A = \frac{1}{2}\int_{t_0}^{t_1}{\left(x\frac{\mathrm{d}y}{\mathrm{d}t} - y\frac{\mathrm{d}x}{\mathrm{d}t}\right)\mathrm{d}t}
\end{equation*}

\noindent
We can also choose different vector field where $\nabla \times \vec{F} = 1$ so that
\begin{equation*}
	 A = \oint\limits_{C}{xy^\prime} = \oint\limits_{C}{yx^\prime}
\end{equation*}

\noindent
For example, let's compute the area of a circle with radius $R$.
We'll parameterize the circle as $(R\cos{t}, R\sin{t}), t \in [0,2\pi]$.\\
\begin{align*}
	A &= \frac{1}{2}\int_{0}^{2\pi}{((R\cos{t})(R\cos{t}) - (R\sin{t})(-R\sin{t}))\mathrm{d}t} \\
	&= \frac{1}{2}\int_{0}^{2\pi}{R^2\mathrm{d}t} \\
	&= \frac{1}{2}R^2 2\pi \\
	&= \pi R^2
\end{align*}


\subsubsection{Linear Factors}
\noindent
This is the the most basic type where the degree of the numerator is less than the degree of the denominator and the denominator factors into all linear factors with no repeated roots. In this case we can write
\begin{equation*}
	\frac{P(x)}{Q(x)} = \frac{A_1}{(x-a_1)} + \ldots + \frac{A_n}{(x-a_n)}.
\end{equation*}

\noindent
Multiplying each side by $Q(x)$,
\begin{equation*}
	P(x) = A_1(x-a_2) \ldots (x-a_n) + \ldots + A_n(x-a_1) \ldots (x-a_{n-1}).
\end{equation*}

\noindent
We can then find each $A_i$ by evaluating both sides at $x=a_i$, since every term except the ith has an $(x-a_i)$ factor that will go to 0. So,
\begin{equation*}
	A_i = \frac{P(a_i)}{(x-a_i) \ldots (x-a_{i-1})(x-a_{i+1}) \ldots (x-a_n)}.
\end{equation*}

\begin{example}
	Find the partial fraction decomposition of the following expression:
	\begin{equation*}
		\frac{2x-1}{x^3-6x^2+11x-6}.
	\end{equation*}
\end{example}
\noindent
Factoring,
\begin{equation*}
	x^3 - 6x^2 + 11x - 6 = (x-1)(x-2)(x-3).
\end{equation*}
So,
\begin{equation*}
	\frac{2x-1}{x^3-6x^2+11x-6} = \frac{A_1}{x-1}+\frac{A_2}{x-2}+\frac{A_3}{x-3}.
\end{equation*}
Multiplying each side by the denominator,
\begin{equation*}
	2x-1 = A_1(x-2)(x-3)+A_2(x-1)(x-3)+A_3(x-1)(x-2).
\end{equation*}
At $x=1$,
\begin{equation*}
	1 = A_1(1-2)(1-3) \implies A_1 = \frac{1}{2}.
\end{equation*}
At $x=2$,
\begin{equation*}
	3 = A_2(2-1)(2-3) \implies A_2 = -3.
\end{equation*}
At $x=3$,
\begin{equation*}
	5 = A_3(3-1)(3-2) \implies A_3 = \frac{5}{2}.
\end{equation*}
So,
\begin{equation*}
	\frac{2x-1}{x^3-6x^2+11x-6} = \frac{1/2}{x-1} + \frac{-3}{x-2} + \frac{5/2}{x-3},
\end{equation*}
just as was shown in the previous example.


\subsection{Right and left inverses}

So far, we have only talked about the inverses of square matrices. But
what about matrices that are not square? Can they be invertible? It
turns out that non-square matrices can never be invertible. However,
they can have left inverses or right inverses.

\begin{definition}{Left and right inverses}{left-and-right-inverse}
  Let $A$ be an $m\times n$-matrix and $B$ an $n\times m$-matrix.  We
  say that $B$ is a \textbf{left inverse}%
  \index{inverse!left inverse}%
  \index{left inverse}%
  \index{matrix!left inverse}%
  \index{matrix!inverse!left inverse} of $A$ if
  \begin{equation*}
    BA=I.
  \end{equation*}
  We say that $B$ is a \textbf{right inverse}%
  \index{inverse!right inverse}%
  \index{right inverse}%
  \index{matrix!right inverse}%
  \index{matrix!inverse!right inverse} of $A$ if
  \begin{equation*}
    AB=I.
  \end{equation*}
  If $A$ has a left inverse, we also say that $A$ is
  \textbf{left invertible}. Similarly, if $A$ has a right inverse, we
  say that $A$ is \textbf{right invertible}.
\end{definition}

\begin{example}{Right inverse}{right-inverse}
  Let
  \begin{equation*}
    A = \begin{mymatrix}{rrr}
      1 & 0 & 0 \\
      0 & 1 & 0 \\
    \end{mymatrix}
    \quad\mbox{and}\quad
    B = \begin{mymatrix}{rr}
      1 & 0 \\
      0 & 1 \\
      0 & 0 \\
    \end{mymatrix}.
  \end{equation*}
  Show that $B$ is a right inverse, but not a left inverse, of $A$.
\end{example}

\begin{solution}
  We compute
  \begin{equation*}
    AB
    ~=~ \begin{mymatrix}{rrr}
      1 & 0 & 0 \\
      0 & 1 & 0 \\
    \end{mymatrix}
    \begin{mymatrix}{rr}
      1 & 0 \\
      0 & 1 \\
      0 & 0 \\
    \end{mymatrix}
    ~=~ \begin{mymatrix}{rrr}
      1 & 0 \\
      0 & 1 \\
    \end{mymatrix}
    ~=~ I,
  \end{equation*}
  \begin{equation*}
    BA
    ~=~ \begin{mymatrix}{rr}
      1 & 0 \\
      0 & 1 \\
      0 & 0 \\
    \end{mymatrix}
    \begin{mymatrix}{rrr}
      1 & 0 & 0 \\
      0 & 1 & 0 \\
    \end{mymatrix}
    ~=~ \begin{mymatrix}{rrr}
      1 & 0 & 0 \\
      0 & 1 & 0 \\
      0 & 0 & 0 \\
    \end{mymatrix}
    ~\neq~ I.
  \end{equation*}
  Therefore, $B$ is a right inverse, but not a left inverse, of $A$.
\end{solution}

Recall from Definition~\ref{def:invertible-matrix} that $B$ is called
an \textbf{inverse}%
\index{inverse!of a matrix}%
\index{matrix!inverse} of $A$ if it is both a left inverse and a right
inverse. A crucial fact is that invertible matrices are always square.

\begin{theorem}{Invertible matrices are square}{invertible-square}
  Let $A$ be an $m\times n$-matrix.
  \begin{itemize}
  \item If $A$ is left invertible, then $m\geq n$.
  \item If $A$ is right invertible, then $m\leq n$.
  \item If $A$ is invertible, then $m=n$.
  \end{itemize}
  In particular, only square matrices can be invertible.
\end{theorem}

\begin{proof}
  To prove the first claim, assume that $A$ is left invertible, i.e.,
  assume that $BA=I$ for some $n\times m$-matrix $B$. We must show
  that $m\geq n$. Assume, for the sake of obtaining a contradiction,
  that this is not the case, i.e., that $m<n$. Then the matrix $A$ has
  more columns than rows. It follows that the homogeneous system of
  equations $A\vect{x}=\vect{0}$ has a non-trivial solution; let
  $\vect{x}$ be such a solution. We obtain a contradiction by a
  similar method as in
  Example~\ref{exa:non-invertible-matrix}. Namely, we have
  \begin{equation*}
    \vect{x} ~=~ I\,\vect{x} ~=~ (BA)\vect{x} ~=~ B(A\vect{x}) ~=~ B\vect{0} ~=~
    \vect{0},
  \end{equation*}
  contradicting the fact that $\vect{x}$ was non-trivial.  Since we
  got a contradiction from the assumption that $m<n$, it follows that
  $m\geq n$.

  The second claim is proved similarly, but exchanging the roles of
  $A$ and $B$.  The third claim follows directly from the first two
  claims, because every invertible matrix is both left and right
  invertible.
\end{proof}

Of course, not all square matrices are invertible. In particular, zero
matrices are not invertible, along with many other square matrices.


\section{The quadratic formula}

\begin{outcome}
  \begin{enumerate}
  \item Use the Quadratic Formula to find the complex roots of a
    quadratic equation.
  \end{enumerate}
\end{outcome}

The roots (or solutions) of a quadratic equation $ax^{2}+bx+c=0$ where $a,b,c$ are real numbers are
obtained by solving the familiar quadratic formula
given by
\index{quadratic formula}
\begin{equation*}
x=
\frac{-b\pm \sqrt{b^{2}-4ac}}{2a}
\end{equation*}

When working with real numbers, we cannot solve this formula if
$b^{2}-4ac<0$. However, complex numbers allow us to find square roots
of negative numbers, and the quadratic formula remains valid for
finding roots of the corresponding quadratic equation.   In this case
there are exactly two distinct (complex) square roots of $b^{2}-4ac$, which are
$i\sqrt{4ac-b^{2}}$ and $-i\sqrt{4ac-b^{2}}$.

Here is an example.

\begin{example}{Solutions to quadratic equation}{quadratic-equation}
Find the solutions to $x^{2}+2x+5=0$.
\end{example}

\begin{solution}
In terms of the quadratic equation above, $a=1$, $b=2$, and $c=5$.
Therefore, we can use the quadratic formula with these values, which becomes
\begin{equation*}
x=
\frac{-b\pm \sqrt{b^{2}-4ac}}{2a}
=
\frac{-2 \pm \sqrt{(2)^{2} - 4 (1)(5)}}{2(1)}
\end{equation*}
Solving this equation, we see that the solutions are given by
\begin{equation*}
x=\frac{-2i\pm \sqrt{4-20}}{2}=\frac{-2\pm 4i}{2}=-1\pm 2i
\end{equation*}

We can verify that these are solutions of the original equation.
We will show $x = -1+2i$ and leave $x = -1-2i$ as an exercise.

\begin{eqnarray*}
x^{2}+2x+5
&=& (-1+2i)^2 + 2(-1+2i) + 5 \\
&=& 1 - 4i - 4 -2 + 4i + 5 \\
&=& 0
\end{eqnarray*}

Hence $x = -1+2i$ is a solution.
\end{solution}

What if the coefficients of the quadratic equation are actually complex
numbers? Does the formula hold even in this case? The answer is yes. This is
a hint on how to do Problem~\ref{exer-complex3} below, a special case of the
fundamental theorem of algebra, and an ingredient in the proof of some
versions of this theorem.

Consider the following example.

\begin{example}{Solutions to quadratic equation}{quadratic-equation-complex}
Find the solutions to $x^{2}-2ix-5=0$.
\end{example}

\begin{solution}
In terms of the quadratic equation above, $a=1$, $b=-2i$, and $c=-5$.
Therefore, we can use the quadratic formula with these values, which becomes
\begin{equation*}
x=
\frac{-b\pm \sqrt{b^{2}-4ac}}{2a}
=
\frac{2i \pm \sqrt{(-2i)^{2} - 4 (1)(-5)}}{2(1)}
\end{equation*}
Solving this equation, we see that the solutions are given by
\begin{equation*}
x=\frac{2i\pm \sqrt{-4+20}}{2}=\frac{2i\pm 4}{2}=i\pm 2
\end{equation*}

We can verify that these are solutions of the original equation.
We will show $x = i + 2$ and leave $x = i-2$ as an exercise.

\begin{eqnarray*}
x^{2}-2ix-5
&=& (i+2)^2 - 2i (i+2) - 5 \\
&=& -1 + 4i + 4 + 2 - 4i - 5 \\
&=& 0
\end{eqnarray*}

Hence $x = i+2$ is a solution.
\end{solution}

We conclude this section by stating an essential theorem.

\begin{theorem}{The fundamental theorem of algebra}{fund-theorem}
Any polynomial of degree at least $1$ with complex coefficients has a root which is a complex number.
\end{theorem}


\section{Boolean Functions}

\subsection{Binary Fields and Functions}
Let $\field{}$ denote the finite field with two elements. For a positive integer $n$ let $\field{n}$ denote the vector space over $\field{}$ of dimension $n$. An $n$-bit \emph{Boolean function} is a function mapping $\field{n}$ to $\field{}$. The set of all $n$-bit Boolean functions is denoted by $\BF{n}$. The \emph{value vector} $\valvec{f}$ of a Boolean function $f$ is the vector of length $2^n$ consisting of the values of $f$ on all possible inputs in the lexicographic order.
$\fzero,\fone$ denote the two constant functions.

For $n \in \ZZplus$ let $\fielde{n}$ denote the field with $2^n$ elements. Such field is defined as the set of polynomials with coefficients from $\field{}$ and degree at most $n-1$. The field addition is the usual addition of polynomials, and the field multiplication is the multiplication of polynomials modulo a fixed irreducible polynomial of degree $n$. It can be summarized by the isomorphism
$$
\fielde{n} \simeq \field{}[X]/P(x),
$$
where $P(x)$ is an irreducible polynomial, i.e. $P(x)$ cannot be factored into polynomials of strictly lower degree.

\subsection{Vectors and Weights}
Elements in vectors are indexed starting from 1. For a vector $v$ from $\field{n}$ it is written $v = (v_1, \ldots, v_n)$. $|X|$ denotes the size of the vector/set $X$. The \emph{weight} of a vector $v$ is the number of nonzero entries in it and is denoted $\wt(v)$. \emph{Weight} of a Boolean function is the weight of its value vector. An $n$-bit Boolean function is said to be \emph{balanced}, if its weight is equal to $2^{n-1}$.

The \emph{correlation} of a vector $v \in \field{t}$ is defined as
\begin{align*}
& \cor(v) \eqdef 2\cdot\wt(v)/t-1,~~ -1 \le \cor(v) \le 1.
\end{align*}
The \emph{correlation}
of a Boolean function $f\colon \field{n} \to \field{}$ is defined as the correlation
of its value vector $\valvec{f}$:
$$\cor(f) \eqdef \cor(\valvec{f}) = \wt(f)/2^{n-1}-1.$$

For any $n \in \ZZplus$, $\idvec{n} \in \field{n}$ denotes the all-one vector $(1,1,\ldots,1)$. For $j \in \ZZplus,1\le j \le n$, the $j$-th unit vector $e_j$ is the vector having 1 at position $j$ and 0 otherwise. $e_1, \ldots, e_n$ form a linear basis of $\field{n}$.

\subsection{Bit-wise Arithmetic}
Let $\land,\lor,\oplus,\lnot$ denote the Boolean operations \tand{}, \tor{}, \txor{} and \tnot{} respectively. The corresponding operations on $\field{n}$ are defined  component-wise, e.g.
$$
(x_1, \ldots, x_n) \land (y_1, \ldots, y_n) \eqdef (x_1 \land y_1, \ldots, x_n \land y_n).
$$
The operation of addition modulo $2^w$ is denoted $\boxplus$, and $w$ should be clear from the context; the bits in a vector are ordered in the decreasing order of significance (see~\SecRef{implicit}). The rotations a vector $x$ to the left and to the right are denoted by $\lll$ and $\ggg$ respectively.

For $x, y \in \field{n}$ the \emph{inner product} of $x$ and $y$ is defined as
$$
\inprod{x,y} \eqdef \bigoplus_{i=1}^n x_i y_i \in \field{}.
$$
This notion is generalized to more arguments. Let $x_1,\dots,x_d \in \field{n}$. Then define
$$
\inprod{x_1,\dots,x_d} \eqdef \bigoplus_{i=1}^{n} \prod_{j=1}^d x_{j,i} \in \field{}.
$$

For $x, y \in \field{n}$, $x^y$ is defined as (note $0^0 = 1$)
$$
x ^ y \eqdef x_1^{y_1}x_2^{y_2}\ldots x_n^{y_n} \eqdef
x \lor \lnot y = \idvec{n} \oplus y \land (x \oplus \idvec{n}) \in \field{}.
$$

Let $\preceq$ be the partial relation on $(\field{n})^2$ defined by $x \preceq y$ if and only if, for all $i \in \{1,\ldots,n\}$, $x_i \leq y_i$. I remark that
$$
x \preceq y ~\Leftrightarrow~
y^x = 1 ~\Leftrightarrow~
(\lnot x)^{\lnot y} = 1.
$$

\SubSecDef{implicit}{Implicit Isomorphisms}
For any $n,m \in \ZZplus$, the vector spaces $\field{n+m}$ and $\field{n}\times \field{m}$ are considered to be the same with an implicit isomorphism splitting an $(n+m)$-bit vector $v \in \field{n+m}$ into two components: $n$ leftmost bits $l\in \field{n}$ and $m$ rightmost bits $r \in \field{m}$.

For any $n \in \ZZplus$, the vectors from $\field{n}$ can be implicitly represented as integers, such that the leftmost bits correspond to the most significant bits. Let $v \in \field{n}$. Then, by abuse of notation, it can be written:
$$
v = (v_1, \ldots, v_n) \in \field{n},
~~\Leftrightarrow~~
v = \sum_{i=1}^n v_i2^{n-i} \in \ZZ_n.$$
A hexadecimal vector notation may be used and indicated by a monospace font, for example
$$
163 \in \ZZn{256} = \hex{a3} \in \field{8} = (1, 0, 1, 0, 0, 0, 1, 1) \in \field{8}.
$$

Another implicit isomorphism is allowed between the vector space $\field{n}$ and the polynomial ring $\field{}[X]$:
$$
v = (v_1, \ldots, v_n) \in \field{n}
~~\Leftrightarrow~~
\sum_{i=1}^n v_i X^{n-i} \in \field{}[X].
$$
For example,
$$
\hex{a3} \in \field{8} = (X^7 + X^5 + X + 1) \in \field{}[X]
$$
Assuming that an irreducible polynomial $P(x)$ defining
$$
\fielde{n} \simeq \field{}[X]/(P(x))
$$
is clear from the context, the multiplication operation in the finite field is denoted $\fmult$. The division in the finite field is denoted by $\fdiv$.

\subsection{Algebraic Normal Form}
Any Boolean function $f\colon \field{n} \to \field{}$ has a unique representation of the form
$$
f(x) = \bigoplus_{u \in \field{n}} a_u x^u,~
f(x) \in \field{}[x_1,\ldots,x_n]/(x_1^2 + x_1, \ldots, x_n^2 + x_n)
$$
called the \emph{algebraic normal form (ANF)}. Here $x^u$ is a shorthand for $x_1^{u_1}\ldots x_n^{u_n}$ and such products are called \emph{monomials}. Let $\coef{u}{f}\in \field{}$ denote the coefficient of the monomial $x^u$ in the ANF of $f$. It can be computed by the \Mobius{} transform:
$$
\coef{u}{f} \eqdef a_u = \bigoplus_{z \in \field{n}, z \preceq u} f(z).
$$

The \emph{algebraic degree} of a Boolean function $f$ is the maximum Hamming weight of all $u$ such that $a_u=1$. Equivalently, it is the maximum degree of a monomial in the ANF of $f$.  It is denoted $\deg{f}$. The zero-function is said to have the algebraic degree $-\infty$. The set of all Boolean functions with $n$ input bits and degree at most $d$ is denoted by $\BF{n,d}$. A Boolean function of algebraic degree at most 1 is called an \emph{affine} function. An affine Boolean function $f$ is said to be \emph{linear} if $f(0) = 0$. Any affine Boolean function $f\colon \field{n} \to \field{}$ can be expressed as $f(x) = \inprod{a,x} + c$ for unique $a\in \field{n}$ and $c \in \field{}$, where $c = 0$ if and only if $f$ is linear.

\subsection{Derivatives}
For a Boolean function $f\colon \field{n} \to \field{}$ and a vector $\alpha \in \field{n}$, I denote the function $\delta_\alpha f\colon \field{n} \to \field{}$ to be the \emph{derivative} of $f$ with respect to $\alpha$, given by
$$
\delta_\alpha f(x) \eqdef f(x) \oplus f(x\oplus \alpha).
$$
It is well known that $\deg{\delta_\alpha f} \le \max(-1, \deg{f} - 1)$ for any Boolean function $f$ and any $\alpha$, see~\cite{Lai1994}. The derivation can be iterated multiple times resulting in a \emph{higher-order derivative}. For $d$ linearly independent vectors $\alpha_1, \ldots, \alpha_{d} \in \field{n}$ it holds that
$$
\delta_{\alpha_1}\ldots \delta_{\alpha_{d}}f(x) = \bigoplus_{z \in \Span(\alpha_1, \ldots, \alpha_{d})} f(x \oplus z).
$$
If the vectors $\alpha_1, \ldots, \alpha_{d}$ are linearly dependent, then the derivative is equal to zero.

\section{Vectorial Boolean Functions}
A \emph{Vectorial} Boolean function $S$ is a function mapping $\field{n}$ to $\field{m}$ for some positive integers $n,m$. When $n$ is relatively small, such functions are often called \emph{S-Boxes}. Each output bit of a vectorial Boolean function naturally defines a Boolean function. The corresponding $m$ Boolean functions are called \emph{coordinates} of $S$. For any nonzero $a \in \field{m}$ the mapping $x \mapsto \inprod{a, S(x)}$ is called a \emph{component} of $S$ and is denoted by $S_a$. A component is a linear combination of coordinates of $S$. The function $S$ is said to be \emph{balanced}, if each $y \in \field{m}$ has exactly $2^{n-m}$ preimages. In particular, $S$ is a bijection if and only if $m = n$ and $S$ is balanced.

A vectorial function $S\colon \field{n} \to \field{m}$ can be given by the vector of its values using the following notation:
$$
\lookup{S} \eqdef (S(0), S(1), \ldots, S(2^n-1)), ~\text{where}~ S(x) \in \field{m}.
$$

The \emph{algebraic degree} of a vectorial Boolean function is defined to be the maximum algebraic degree of its coordinates.

For any $n \in \ZZplus$ the following maps are defined:
\eq{
    &\Left\colon \field{n} \times \field{n} \to \field{n},~~(a,b) \mapsto a,\\
    &\Right\colon \field{n} \times \field{n} \to \field{n},~~(a,b) \mapsto b,\\
    &\Swap\colon \field{n} \times \field{n} \to \field{n} \times \field{n},~~(a,b) \mapsto (b,a).
}

\subsection{Linear maps}
The vectors from $\field{n}$ are considered as column vectors. The transpose of a vector or matrix $v$ is denoted $v^{\top}$. The $n\times n$ identity matrix is denoted $\idmat{n}$.

A vectorial Boolean function $S\colon \field{n} \to \field{m}$ is called \emph{linear} (resp. \emph{affine}) if all its coordinates are linear (resp. affine). If $S$ is affine, then it can be expressed as $S(x) = A \times x \oplus b$ for a unique $m \times n$ matrix $A$ over $\field{}$ and $b = S(0) \in \field{m}$, where $b = 0$ if and only if $S$ is linear.

For $m,n \in \ZZplus$, the set of all $m \times n$ matrices over $\field{}$ is denoted $\linmap{n}{m}$. Any such matrix $M$ defines a linear map from $\field{n}$ to $\field{m}$, given by $x \mapsto M\times x$. The set of all bijective linear maps are denoted $\linbij{n} \subseteq \linmap{n}{n}$. The set of all bijective affine maps is denoted $\affbij{n}$.

\subsection{Equivalence Notions}
There are several important notions of \emph{equivalence} between vectorial Boolean functions. Let $S_1,S_2\colon \field{n} \to \field{m}$ be vectorial Boolean functions. Let
$$\Gamma_1 = \pset{(x, S_1(x)) \mid x \in \field{n}} \subseteq \field{n+m},$$
$$\Gamma_2 = \pset{(x, S_2(x)) \mid x \in \field{n}} \subseteq \field{n+m}$$
be the functional graphs of $S_1$ and $S_2$ respectively.

\begin{itemize}
    \item $S_1, S_2$ are \emph{linear} (resp. \emph{affine}) equivalent if there exist linear (resp. affine) mappings $A,B$ such that $S_2 = B \circ S_1 \circ A$.

    \item $S_1, S_2$ are \emph{extended-affine} equivalent (EA-equivalent) if there exist affine mappings $A,B,C$ such that $S_2 = B \circ S_1 \circ A \oplus C$.

    \item $S_1, S_2$ are \emph{CCZ-equivalent} if there exists an affine mapping $L$ such that $\Gamma_2 = L(\Gamma_1) \eqdef \pset{L(x) \mid x \in \Gamma_1}$, i.e. the functional graphs of $S_1$ and $S_2$ are affine equivalent.
\end{itemize}

\section{Set Indicators and Subspaces}
Let $V \subseteq \field{n}$. The \emph{indicator} of the set $V$ is defined as
\begin{align*}
& \Ind_V\colon \field{n} \to \field{},\\
& \Ind_V(x) :=
    \begin{cases}
    1 &\text{if } x \in V, \\
    0 &\text{if } x \notin V.
    \end{cases}
\end{align*}
The \emph{degree} of the set $V$ is defined as the algebraic degree of its indicator:
$$
\deg{V} \eqdef \deg{\Ind_V}.
$$
In the case of \emph{multiset} over $\field{n}$, only the elements with an even multiplicity are considered.

A set $V \subseteq \field{n}$ is said to be a \emph{linear subspace} if $V$ is closed under the addition in $\field{n}$ (i.e., under the \txor{} operation). A set $U \subseteq \field{n}$ is said to be an \emph{affine subspace} if there exists $a \in \field{n}$ such that $V \eqdef a \oplus U \eqdef \pset{a \oplus u \mid u \in U}$ is a linear subspace. It is then said that $U = a \oplus V$ is a \emph{coset} of the linear subspace $V$. Such $a$ may not be unique, but the corresponding linear subspace is unique.

Let $U$ be any affine subspace. The \emph{dimension} of $U$ is the maximum number of linearly independent vectors in the linear part of $U$; it is denoted $\dim{U}$. Furthermore, $U$ has $2^{\dim{U}}$ elements. $U$ can be viewed a solution to a system of $k \eqdef n-\dim{U}$ linear equations defined by affine functions $l_1,\ldots,l_k$:
$$
U = \{x \in \field{n} \mid l_1(x) = 0, \ldots, l_k(x) = 0\}.
$$
It follows that the indicator of $U$ is affine equivalent to a monomial function of degree $n - \dim{U}$, i.e. it has the following form:
$$
\Ind_U(x) = (l_1(x)+1)\cdot \ldots \cdot (l_k(x)+1).
$$

Consider a Boolean function $f\colon \field{n} \to \field{}, f \ne 0$ and let $d = \deg{f}$. The minimum possible weight of $f$ is equal to $2^{n-d}$, i.e.
$$\wt(f) \ge 2^{n-\deg{f}}.$$
%The equality holds if and only if $f$ is an indicator of an affine subspace, i.e. is affine equivalent to a monomial function.


\section{Strange Phenomenon in Training with Gradient Descent}

We want to use gradient descent to solve the Poisson equations.
\begin{equation}
  \label{BilinearNodalBasis}
  \left\{
  \begin{array}{ll}
-u''=f, \quad on [0,1], \\
u(0)=u(1)=0.
\end{array}
\right.
\end{equation}

For a given $f=\pi^2sin(\pi x) $, we can calculate the true solution as $u=sin(\pi x)$. We also want to use gradient descent to calculate the numerical solution to compare the error.

We divide the unit interval into $n$ intervals equally and denote the boundary points as $x_i=\frac{i}{n+1}, i=0,1,...,n+1$. Then the interpolation of the true solution is $u_i=sin(\pi x_i)$.

To compute the numerical solution, we first define
\begin{equation}
J(v_h)=\frac12\int_0^1|v_h'|^2dx-\int_0^1fv_hdx.
\end{equation}
Let
$$
\displaystyle v_h=\sum_{i=1}^n\nu_i\varphi_i,
$$
then
$$
J(v_h)=I(\nu)=\frac12\nu^TA\ast\nu-b^T\nu
$$
and
$$
\nabla I(\nu) =A\ast \nu -b.
$$
Here $A=(-1,2,-1)/h$
At the same time, let $\displaystyle u_h=\sum_{i=1}^n\mu_i\varphi_i,$
\begin{equation}\label{min}
\displaystyle u_h=\argmin_{v_h\in V_h} J(v_h)\Leftrightarrow \mu=\argmin_{\nu \in R^n} I(\nu)
\end{equation}

Therefore, we choose a random $u_0$, then in $i$-th iteration we update
\begin{equation}
u^i=u^{i-1}+\eta(b-A * u^{i-1}).
\end{equation}
We plot the graph of the $H_1$-norm of the error, $||u^i-u||$, vs the number of iteration $i$.

When we choose $k=4$ and $ \eta=\frac{h}{4},\frac{h}{8},\frac{h}{16},\frac{h}{32}$ respectively, the graphs of the $H_1$-norm of the error, $||u^i-u||$, vs the number of iteration $i$ are as follows.
\begin{figure}[!htbp]
	\begin{center}
		\includegraphics[width=0.45\textwidth]{./figures/h-4}  \quad
		\includegraphics[width=0.45\textwidth]{./figures/h-8} \\
		\includegraphics[width=0.45\textwidth]{./figures/h-16} \quad
		\includegraphics[width=0.45\textwidth]{./figures/h-32}
	\end{center}
	\caption{error vs number of iteration for $\eta=\frac{h}{4},\frac{h}{8},\frac{h}{16},\frac{h}{32}$}
\end{figure}

There is a spike in  the graphs, where the error decreases very fast below $10^{-8}$ and then increase. Moreover, as $ \eta$ decreases, the number of iteration where the spike happens also increases.

One explanation is that it's a special case when the interpolation of the true solution is a eigenvector of the matrix $A$. What we can do is to decrease the $ \eta$ accordingly to a very small value.


\chapter{Logistic Regression}

\section{Representation}
Logistic regression can be binomial or multinomial. The \textbf{binomial logistic regression} model has the following form
\begin{equation}
p(y|\vec{x},\vec{w})=\mathrm{Ber}(y|\mathrm{sigm}(\vec{w}^T\vec{x}))
\end{equation}
where $\vec{w}$ and $\vec{x}$ are extended vectors, i.e., $\vec{w}=(b, w_1, w_2,\cdots, w_D)$, $\vec{x}=(1, x_1, x_2,\cdots, x_D)$.

\section{Optimization}
\label{sec:binomial-LR-Optimization}

\subsection{MLE}
\begin{align}
\ell(\vec{w}) &= \log\left\{\prod\limits_{i=1}^N{\left[\pi(\vec{x}_i)\right]^{y_i}\left[1-\pi(\vec{x}_i)\right]^{1-y_i}}\right\} \nonumber \\
              & \quad \text{, where } \pi(\vec{x}) \triangleq P(y=1|\vec{x},\vec{w}) \nonumber \\
           &= \sum\limits_{i=1}^N\left[y_i\log\pi(\vec{x}_i)+(1-y_i)\log(1-\pi(\vec{x}_i))\right] \label{eqn:cross-entropy-error} \nonumber \\
		   &= \sum\limits_{i=1}^N\left[y_i\log\dfrac{\pi(\vec{x}_i)}{1-\pi(\vec{x}_i)}+\log(1-\pi(\vec{x}_i))\right] \nonumber \\
		   &= \sum\limits_{i=1}^N\left[y_i(\vec{w}\cdot\vec{x}_i)-\log(1+\exp(\vec{w}\cdot\vec{x}_i))\right] \nonumber \\
J(\vec{w}) & \triangleq \mathrm{NLL}(\vec{w})= -\ell(\vec{w}) \nonumber \\
           & = -\sum\limits_{i=1}^N\left[y_i(\vec{w}\cdot\vec{x}_i)-\log(1+\exp(\vec{w}\cdot\vec{x}_i))\right]
\end{align}

Equation \ref{eqn:cross-entropy-error} is also called the \textbf{cross-entropy} error function (see Equation \ref{eqn:cross-entropy}).

Unlike linear regression, we can no longer write down the MLE in closed form. Instead, we need to use an optimization algorithm to compute it, see Appendix \ref{chap:Optimization-methods}. For this, we need to derive the gradient and Hessian.

In the case of logistic regression, one can show that the gradient and Hessian of this are given by the following
\begin{align}
\vec{g}(\vec{w}) &= \dfrac{\mathrm{d} J}{\mathrm{d} \vec{w}} = \sum\limits_{i=1}^N \left[\pi(\vec{x}_i) - y_i \right]\vec{x}_i = \vec{X}(\vec{\pi}-\vec{y})\\
\vec{H}(\vec{w}) &= \dfrac{\mathrm{d} \vec{g}^T}{\mathrm{d} \vec{w}}= \dfrac{\mathrm{d}}{\mathrm{d} \vec{w}} (\vec{\pi}-\vec{y})^T\vec{X}^T \nonumber \\
        &= \dfrac{\mathrm{d}}{\mathrm{d} \vec{w}} \vec{\pi}^T\vec{X}^T \nonumber \\
		&= (\pi(\vec{x}_i)(1-\pi(\vec{x}_i))\vec{x}_i, \cdots,)\vec{X}^T \nonumber \\
		&= \vec{X}\vec{S}\vec{X}^T, \quad \vec{S} \triangleq \mathrm{diag}(\pi(\vec{x}_i)(1-\pi(\vec{x}_i))\vec{x}_i)
\end{align}

\subsubsection{Iteratively reweighted least squares (IRLS)}
\label{sec:IRLS}
TODO

\subsection{MAP}
Just as we prefer ridge regression to linear regression, so we should prefer MAP estimation for logistic regression to computing the MLE.
$\ell_2$ regularization

we can use $\ell_2$ regularization, just as we did with ridge regression. We note that the new objective, gradient and Hessian have the following forms:
\begin{align}
J'(\vec{w}) & \triangleq \mathrm{NLL}(\vec{w})+\lambda \vec{w}^T\vec{w} \\
\vec{g}'(\vec{w}) &= \vec{g}(\vec{w})+\lambda\vec{w} \\
\vec{H}'(\vec{w}) &= \vec{H}(\vec{w})+\lambda\vec{I}
\end{align}

It is a simple matter to pass these modified equations into any gradient-based optimizer.

\section{Multinomial logistic regression}

\subsection{Representation}
\textbf{Multinomial logistic regression} model is also called a \textbf{maximum entropy classifier}, which has the following form
\begin{align}
p(y=c|\vec{x},\vec{W}) & =\dfrac{\exp(\vec{w}_c^T\vec{x})}{\sum_{c=1}^C \exp(\vec{w}_c^T\vec{x})}
\end{align}

\subsection{MLE}
Let $\vec{y}_i=(\mathbb{I}(y_i=1),\mathbb{I}(y_i=1),\cdots, \mathbb{I}(y_i=C))$, $\vec{\mu}_i=(p(y=1|\vec{x}_i,\vec{W}),p(y=2|\vec{x}_i,\vec{W}),\cdots, p(y=C|\vec{x}_i,\vec{W}))$, then the log-likelihood function can be written as
\begin{align}
\ell(\vec{W}) & =\log\prod\limits_{i=1}^N\prod\limits_{c=1}^C \mu_{ic}^{y_{ic}}=\sum\limits_{i=1}^N\sum\limits_{c=1}^C y_{ic}\log \mu_{ic} \\
     & = \sum\limits_{i=1}^N\left[\left(\sum\limits_{c=1}^C y_{ic}\vec{w}_c^T\vec{x}_i\right)-\log\left(\sum\limits_{c=1}^C \exp(\vec{w}_c^T\vec{x}_i)\right)\right]
\end{align}

Define the objective function as NLL
\begin{equation}
J(\vec{W})=\mathrm{NLL}(\vec{W})=-\ell(\vec{W})
\end{equation}

Define $\vec{A} \otimes \vec{B}$ be the \textbf{kronecker product} of matrices $\vec{A}$ and $\vec{B}$.If $\vec{A}$ is an $m \times n$ matrix and $\vec{B}$ is a $p \times q$ matrix, then $\vec{A} \otimes \vec{B}$ is the $mp \times nq$ block matrix
\begin{equation}
\vec{A} \otimes \vec{B} \triangle \left(\begin{array}{ccc}
a_{11}\vec{B} & \cdots & a_{1n}\vec{B} \\
\vdots & \vdots & \vdots \\
a_{m1}\vec{B} & \cdots & a_{mn}\vec{B}
\end{array}\right)
\end{equation}

The gradient and Hessian are given by
\begin{align}
\vec{g}(\vec{W}) & =\sum\limits_{i=1}^N (\vec{\mu}-\vec{y}_i) \otimes \vec{x}_i \\
\vec{H}(\vec{W}) & =\sum\limits_{i=1}^N (\mathrm{diag}(\vec{\mu}_i)-\vec{\mu}_i\vec{\mu}_i^T) \otimes (\vec{x}_i\vec{x}_i^T)
\end{align}
where $\vec{y}_i=(\mathbb{I}(y_i=1),\mathbb{I}(y_i=1),\cdots, \mathbb{I}(y_i=C-1))$ and $\vec{\mu}_i=(p(y=1|\vec{x}_i,\vec{W}),p(y=2|\vec{x}_i,\vec{W}),\cdots, p(y=C-1|\vec{x}_i,\vec{W}))$ are column vectors of length $C-1$.

Pass them to any gradient-based optimizer.

\subsection{MAP}
The new objective
\begin{align}
J'(\vec{W}) & =\mathrm{NLL}(\vec{w})-\log{p(\vec{W})} \\
            & \quad \text{, where } p(\vec{W}) \triangleq \prod\limits_{c=1}^C \mathcal{N}(\vec{w}_c|\vec{0},\vec{V}_0) \nonumber \\
   & = J(\vec{W})+\dfrac{1}{2}\sum\limits_{c=1}^C \vec{w}_c\vec{V}_0^{-1}\vec{w}_c \\
\end{align}

Its gradient and Hessian are given by
\begin{align}
\vec{g}'(\vec{w}) & =\vec{g}(\vec{W})+\vec{V}_0^{-1}\left(\sum\limits_{c=1}^C \vec{w}_c\right) \\
\vec{H}'(\vec{w}) & =\vec{H}(\vec{w})+\vec{I_C} \otimes \vec{V}_0^{-1}
\end{align}

This can be passed to any gradient-based optimizer to find the MAP estimate. Note, however, that the Hessian has size $((CD)×(CD))$, which is $C$ times more row and columns than in the binary case, so limited memory BFGS is more appropriate than Newton’s method.

\section{Bayesian logistic regression}
It is natural to want to compute the full posterior over the parameters, $p(\vec{w}|\mathcal{D})$, for logistic regression models. This can be useful for any situation where we want to associate confidence intervals with our predictions (e.g., this is necessary when solving contextual bandit problems, discussed in Section TODO).

Unfortunately, unlike the linear regression case, this cannot be done exactly, since there is no convenient conjugate prior for logistic regression. We discuss one simple approximation below; some other approaches include MCMC (Section TODO), variational inference (Section TODO), expectation propagation (Kuss and Rasmussen 2005), etc. For notational simplicity, we stick to binary logistic regression.

\subsection{Laplace approximation}

\subsection{Derivation of the BIC}

\subsection{Gaussian approximation for logistic regression}
\label{sec:Gaussian-approximation-for-logistic-regression}

\subsection{Approximating the posterior predictive}

\subsection{Residual analysis (outlier detection) *}

\section{Online learning and stochastic optimization}
Traditionally machine learning is performed \textbf{offline}, however, if we have \textbf{streaming data}, we need to perform \textbf{online learning}, so we can update our estimates as each new data point arrives rather than waiting until “the end” (which may never occur). And even if we have a batch of data, we might want to treat it like a stream if it is too large to hold in main memory. Below we discuss learning methods for this kind of scenario.

TODO

\subsection{The perceptron algorithm}

\subsubsection{Representation}
\begin{equation}
\mathcal{H}:y=f(\vec{x})=\text{sign}(\vec{w}^T\vec{x}+b)
\end{equation}
where $\text{sign}(x)=\begin{cases}+1, & x \geq 0\\-1, & x<0\\\end{cases}$, see Fig. \ref{fig:perceptron}\footnote{\url{https://en.wikipedia.org/wiki/Perceptron}}.

\begin{figure}[hbtp]
\centering
    \includegraphics[scale=.50]{perceptron.png}
\caption{Perceptron}
\label{fig:perceptron}
\end{figure}

\subsubsection{Evaluation}
\begin{eqnarray}
L(\vec{w},b)&=&-y_i(\vec{w}^T\vec{x}_i+b)\\
R_{emp}(f)&=&-\sum\limits_i y_i(\vec{w}^T\vec{x}_i+b)\\
\end{eqnarray}

\subsubsection{Optimization}
\textbf{Primal form}
\begin{algorithm}[htbp]
\caption{Perceptron learning algorithm, primal form, using SGD}

    $\vec{w} \leftarrow 0;\; b \leftarrow 0;\; k \leftarrow 0$\;
    \While{no mistakes made within the for loop}{
        \For{$i\leftarrow 1$ \KwTo $N$}{
			\If{$y_i(\vec{w} \cdot \vec{x}_i+b) \leq 0$}{
				$\vec{w} \leftarrow \vec{w}+\eta y_i \vec{x}_i$\;
				$b \leftarrow b+\eta y_i$\;
				$k \leftarrow k+1$\;
			}
		}
    }
\end{algorithm}

\textbf{Convergency}
\begin{theorem}
(\textbf{Novikoff}) If traning data set $\mathcal{D}$ is linearly separable, then
\begin{enumerate}
\item There exists a hyperplane denoted as $\widehat{\vec{w}}_{opt} \cdot \vec{x}+b_{opt}=0$ which can correctly seperate all samples, and
\begin{equation}
\exists\gamma>0,\; \forall i, \; y_i(\vec{w}_{opt} \cdot \vec{x}_i+b_{opt}) \geq \gamma
\end{equation}
\item \begin{equation}k \leq \left(\dfrac{R}{\gamma}\right)^2,\text{ where } R=\max\limits_{1 \leq i \leq N} \abs{\abs{\widehat{\vec{x}}_i}}
\end{equation}
\end{enumerate}
\end{theorem}

\begin{proof}
(1) let $\gamma=\min\limits_{i} y_i(\vec{w}_{opt} \cdot \vec{x}_i+b_{opt})$, then we get $y_i(\vec{w}_{opt} \cdot \vec{x}_i+b_{opt}) \geq \gamma$.

(2) The algorithm start from $\widehat{\vec{x}}_0=0$, if a instance is misclassified, then update the weight. Let $\widehat{\vec{w}}_{k-1}$ denotes the extended weight before the k-th misclassified instance, then we can get
\begin{eqnarray}
y_i(\widehat{\vec{w}}_{k-1} \cdot \widehat{\vec{x}_i})&=&y_i(\vec{w}_{k-1} \cdot \vec{x}_i+b_{k-1}) \leq 0\\
\widehat{\vec{w}}_k&=&\widehat{\vec{w}}_{k-1}+\eta y_i \widehat{\vec{x}_i}
\end{eqnarray}

We could infer the following two equations, the proof procedure are omitted.
\begin{enumerate}
\item $\widehat{\vec{w}}_k \cdot \widehat{\vec{w}}_{opt} \geq k\eta\gamma$
\item $\abs{\abs{\widehat{\vec{w}}_k}}^2 \leq k\eta^2R^2$
\end{enumerate}

From above two equations we get
\begin{eqnarray}
\nonumber k\eta\gamma & \leq & \widehat{\vec{w}}_k \cdot \widehat{\vec{w}}_{opt} \leq \abs{\abs{\widehat{\vec{w}}_k}}\abs{\abs{\widehat{\vec{w}}_{opt}}} \leq \sqrt k \eta R \\
\nonumber k^2\gamma^2 & \leq & kR^2 \\
\nonumber \text{i.e. } k & \leq & \left(\dfrac{R}{\gamma}\right)^2
\end{eqnarray}
\end{proof}

\textbf{Dual form}
\begin{eqnarray}
\vec{w}&=&\sum\limits_{i=1}^{N} \alpha_iy_i\vec{x}_i \\
b&=&\sum\limits_{i=1}^{N} \alpha_iy_i \\
f(\vec{x})&=&\text{sign}\left(\sum\limits_{j=1}^{N} \alpha_jy_j\vec{x}_j \cdot \vec{x}+b\right)
\end{eqnarray}

\begin{algorithm}[htbp]
    %\SetAlgoLined
    \SetAlgoNoLine

    $\vec{\alpha} \leftarrow 0;\; b \leftarrow 0;\; k \leftarrow 0$\;
    \While{no mistakes made within the for loop}{
        \For{$i\leftarrow 1$ \KwTo $N$}{
			\If{$y_i\left(\sum\limits_{j=1}^{N} \alpha_jy_j\vec{x}_j \cdot \vec{x}_i+b\right) \leq 0$}{
				$\vec{\alpha} \leftarrow \vec{\alpha}+\eta$\;
				$b \leftarrow b+\eta y_i$\;
				$k \leftarrow k+1$\;
			}
		}
    }
\caption{Perceptron learning algorithm, dual form}
\end{algorithm}

\section{Generative vs discriminative classifiers}

\subsection{Pros and cons of each approach}
\begin{itemize}
\item{\textbf{Easy to fit?} As we have seen, it is usually very easy to fit generative classifiers. For example, in Sections \ref{sec:NBC-Optimization} and \ref{sec:MLE-for-discriminant-analysis}, we show that we can fit a naive Bayes model and an LDA model by simple counting and averaging. By contrast, logistic regression requires solving a convex optimization problem (see Section \ref{sec:binomial-LR-Optimization} for the details), which is much slower.}

\item{\textbf{Fit classes separately?} In a generative classifier, we estimate the parameters of each class conditional density independently, so we do not have to retrain the model when we add more classes. In contrast, in discriminative models, all the parameters interact, so the whole model must be retrained if we add a new class. (This is also the case if we train a generative model to maximize a discriminative objective Salojarvi et al. (2005).)}

\item{\textbf{Handle missing features easily?} Sometimes some of the inputs (components ofx) are not observed. In a generative classifier, there is a simple method for dealing with this, as we discuss in Section \ref{sec:Dealing-with-missing-data}. However, in a discriminative classifier, there is no principled solution to this problem, since the model assumes that $\vec{x}$is always available to be conditioned on (although see (Marlin 2008) for some heuristic approaches).}

\item{\textbf{Can handle unlabeled training data?} There is much interest in \textbf{semi-supervised learning}, which uses unlabeled data to help solve a supervised task. This is fairly easy to do using generative models (see e.g., (Lasserre et al. 2006; Liang et al. 2007)), but is much harder to do with discriminative models.}

\item{\textbf{Symmetric in inputs and outputs?} We can run a generative model “backwards”, and infer probable inputs given the output by computing $p(\vec{x}|y)$. This is not possible with a discriminative model. The reason is that a generative model defines a joint distribution on $\vec{x}$ and $y$, and hence treats both inputs and outputs symmetrically.}

\item{\textbf{Can handle feature preprocessing?} A big advantage of discriminative methods is that they allow us to preprocess the input in arbitrary ways, e.g., we can replace $\vec{x}$ with $\phi(\vec{x})$, which could be some basis function expansion, etc. It is often hard to define a generative model on such pre-processed data, since the new features are correlated in complex ways.}

\item{\textbf{Well-calibrated probabilities?} Some generative models, such as naive Bayes, make strong independence assumptions which are often not valid. This can result in very extreme posterior class probabilities (very near 0 or 1). Discriminative models, such as logistic regression, are usually better calibrated in terms of their probability estimates.}
\end{itemize}

See Table \ref{tab:List-of-various-models-for-classification} for a summary of the classification and regression techniques we cover in this book.

\begin{table*}
\centering
\begin{tabular}{lllll}
\hline\noalign{\smallskip}
\textbf{Model} & \textbf{Classif/regr} & \textbf{Gen/Discr} & \textbf{Param/Non} & \textbf{Section} \\
\noalign{\smallskip}\svhline\noalign{\smallskip}
Discriminant analysis & Classif & Gen & Param & Sec. \ref{sec:Linear-discriminant-analysis}, \ref{sec:MLE-for-discriminant-analysis} \\
Naive Bayes classifier & Classif & Gen & Param & Sec. \ref{sec:NBC}, \ref{sec:Bayesian-naive-Bayes} \\
Tree-augmented Naive Bayes classifier & Classif & Gen & Param & Sec. 10.2.1 \\
Linear regression & Regr & Discrim & Param & Sec. 1.4.5, 7.3, 7.6 \\
Logistic regression & Classif & Discrim & Param & Sec. 1.4.6, \ref{sec:IRLS}, \ref{sec:Gaussian-approximation-for-logistic-regression}, 21.8.1.1 \\
Sparse linear/ logistic regression & Both & Discrim & Param & Ch. 13 \\
Mixture of experts & Both & Discrim & Param & Sec. 11.2.4 \\
Multilayer perceptron (MLP)/ Neural network & Both & Discrim & Param & Ch. 16 \\
Conditional random field (CRF) & Classif & Discrim & Param & Sec. 19.6 \\
\noalign{\smallskip}\hline \\
$K$ nearest neighbor classifier & Classif & Gen & Non & Sec. TODO, TODO \\
(Infinite) Mixture Discriminant analysis & Classif & Gen & Non & Sec. 14.7.3 \\
Classification and regression trees (CART) & Both & Discrim & Non & Sec. 16.2 \\
Boosted model & Both & Discrim & Non & Sec. 16.4 \\
Sparse kernelized lin/logreg (SKLR) & Both & Discrim & Non & Sec. 14.3.2 \\
Relevance vector machine (RVM) & Both & Discrim & Non & Sec. 14.3.2 \\
Support vector machine (SVM) & Both & Discrim & Non & Sec. 14.5 \\
Gaussian processes (GP) & Both & Discrim & Non & Ch. 15 \\
Smoothing splines & Regr & Discrim & Non & Section 15.4.6 \\
\noalign{\smallskip}\hline
\end{tabular}
\caption{List of various models for classification and regression which we discuss in this book. Columns are as follows: Model name; is the model suitable for classification, regression, or both; is the model generative or discriminative; is the model parametric or non-parametric; list of sections in book which discuss the model. See also \url{http://pmtk3.googlecode.com/svn/trunk/docs/tutorial/html/tutSupervised.html} for the PMTK equivalents of these models. Any generative probabilistic model (e.g., HMMs, Boltzmann machines, Bayesian networks, etc.) can be turned into a classifier by using it as a class conditional density}
\label{tab:List-of-various-models-for-classification}
\end{table*}

\subsection{Dealing with missing data}
\label{sec:Dealing-with-missing-data}
Sometimes some of the inputs (components of $\vec{x}$) are not observed; this could be due to a sensor failure, or a failure to complete an entry in a survey, etc. This is called the \textbf{missing data problem} (Little. and Rubin 1987). The ability to handle missing data in a principled way is one of the biggest advantages of generative models.

To formalize our assumptions, we can associate a binary response variable $r_i \in \{0,1\}$ that specifies whether each value $\vec{x}_i$ is observed or not. The joint model has the form $p(\vec{x}_i,r_i|\vec{\theta},\vec{\phi})=p(r_i|\vec{x}_i,\vec{\phi})p(\vec{x}_i|\vec{\theta})$, where $\vec{\phi}$ are the parameters controlling whether the item is observed or not.
\begin{itemize}
\item{If we assume $p(r_i|\vec{x}_i,\vec{\phi})=p(r_i|\vec{\phi})$, we say the data is \textbf{missing completely at random} or \textbf{MCAR}.}
\item{If we assume $p(r_i|\vec{x}_i,\vec{\phi})=p(r_i|\vec{x}_i^o,\vec{\phi})$, where $\vec{x}_i^o$ is the observed part of $\vec{x}_i$, we say the data is \textbf{missing at random} or \textbf{MAR}.}
\item{If neither of these assumptions hold, we say the data is \textbf{not missing at random} or \textbf{NMAR}. In this case, we have to model the missing data mechanism, since the pattern of missingness is informative about the values of the missing data and the corresponding parameters. This is the case in most collaborative filtering problems, for example.}
\end{itemize}

See e.g., (Marlin 2008) for further discussion. We will henceforth assume the data is MAR.

When dealing with missing data, it is helpful to distinguish the cases when there is missingness only at test time (so the training data is \textbf{complete data}), from the harder case when there is missingness also at training time. We will discuss these two cases below. Note that the class label is always missing at test time, by definition; if the class label is also sometimes missing at training time, the problem is called semi-supervised learning.

\subsubsection{Missing data at test time}
In a generative classifier, we can handle features that are MAR by marginalizing them out. For example, if we are missing the value ofx1, we can compute
\begin{align}
p(y=c|\vec{x}_{2:D},\vec{\theta}) & \propto p(y=c|\vec{\theta})p(\vec{x}_{2:D}|y=c,\vec{\theta}) \\
      & = \propto p(y=c|\vec{\theta})\sum\limits_{x_1}p(x_1, \vec{x}_{2:D}|y=c,\vec{\theta})
\end{align}

Similarly, in discriminant analysis, no matter what regularization method was used to estimate the parameters, we can always analytically marginalize out the missing variables (see Section \ref{sec:Inference-in-jointly-Gaussian-distributions}):
\begin{equation}
p(\vec{x}_{2:D}|y=c,\vec{\theta})=\mathcal{N}(\vec{x}_{2:D}|\vec{\mu}_{c,2:D},\vec{\Sigma}_{c,2:D})
\end{equation}

\subsubsection{Missing data at training time}
Missing data at training time is harder to deal with. In particular, computing the MLE or MAP estimate is no longer a simple optimization problem, for reasons discussed in Section TODO. However, soon we will study are a variety of more sophisticated algorithms (such as EM algorithm, in Section 11.4) for finding approximate ML or MAP estimates in such cases.

\subsection{Fisher’s linear discriminant analysis (FLDA) *}
TODO


\section{Bilinar Forms}
\begin{definition}
    Let $U,V$ be vector spaces over $F$, then $\phi:U\times V\to F$ is a bilinear form if $\phi(u,\cdot)\in V^\ast$ and $\phi(\cdot,v)\in U^\ast$ for any $u\in U,v\in V$.
\end{definition}
We write $\phi_L\in L(U,V^\ast)$ to be the map $u\mapsto\phi(u,\cdot)$ and $\phi_R\in L(V,U^\ast)$ as $v\mapsto\phi(\cdot,v)$.
In particular, $\phi_L(u)(v)=\phi(u,v)=\phi_R(v)(u)$.
\begin{example}
    1. The map $V\times V^\ast\to F$ via $(v,\theta)\mapsto \theta$ is a bilinear form.\\
    2. The scalar product on $F^n$, that is
    $$\left( \begin{pmatrix}
        x_1\\
        \vdots\\
        x_n
    \end{pmatrix}, \begin{pmatrix}
        y_1\\
        \vdots\\
        y_n
    \end{pmatrix}\right)\mapsto \sum_{i=1}^nx_iy_i$$
    is a bilinear form.\\
    3. Take $U=V=C([0,1],\mathbb R)$, then
    $$(f,g)\mapsto\int_0^1f(t)g(t)\,\mathrm dt$$
    is a bilinear form.
\end{example}
\begin{definition}
    Take a basis $B=\{e_1,\ldots,e_m\}$ of $U$ and $C=\{f_1,\ldots,f_n\}$ basis of $V$ and $\phi:U\times V\to F$ a bilinear form, then the matrix of $\phi$ with respect to $B,C$ is
    $$[\phi]_{B,C}=(\phi(e_i,f_j))_{1\le i\le m,1\le j\le n}$$
\end{definition}
\begin{lemma}
    We have $\phi(u,v)=[u]_B^\top[\phi]_{B,C}[v]_C$ for any $u\in U,v\in V$.
\end{lemma}
\begin{proof}
    If $u=\sum_i\lambda_ie_i$ and $v=\sum_j\mu_jf_j$, then by linearity,
    $$\phi(u,v)=\sum_{i=1}^n\sum_{j=1}^n\lambda_i\mu_j\phi(e_i,f_j)=[u]_B^\top[\phi]_{B,C}[v]_C$$
    by simple expansion.
\end{proof}
\begin{remark}
    The matrix $[\phi]_{B,C}$ is the unique matrix such that the previous lemma holds.
\end{remark}
\begin{lemma}
    Take a basis $B=\{e_1,\ldots,e_m\}$ of $U$ and the dual basis $B^\ast=\{\epsilon_1,\ldots,\epsilon_m\}$ of $U^\ast$.
    Similarly take a basis $C=\{f_1,\ldots,f_n\}$ of $v$ and the dual basis $\{\eta_1,\ldots,\eta_n\}$ of $V^\ast$.
    If $A=[\phi]_{B,C}$ where $\phi:U\times V\to F$ is a bilinear form, then $[\phi_R]_{C,B^\ast}=A$ and $[\phi_L]_{B,C^\ast}=A^\top$.
\end{lemma}
\begin{proof}
    Completely trivial.
\end{proof}
\begin{definition}
    $\ker\phi_L$ is called the left kernel of $\phi$ and $\ker\phi_R$ is called the right kernel of $\phi$.\\
    $\phi$ is nondegenerate if both kernels are $\{0\}$.
    Otherwise, we say $\phi$ is degenerate.
\end{definition}
\begin{lemma}
    Let $B,C$ be bases of $U,V$ respectively and $\phi:U\times V\to F$ be bilinear.
    Let $A=[\phi]_{B,C}$, then $\phi$ is nondegenerate iff $A$ is invertible.
\end{lemma}
\begin{proof}
    Immediate from the preceding lemma.
\end{proof}
\begin{corollary}
    If $\phi$ is nondegenerate, then $\dim U=\dim V$.
\end{corollary}
\begin{proof}
    All invertible matrices are square.
\end{proof}
\begin{example}
    So the dot product on $\mathbb R^n$ is nondegenerate.
\end{example}
\begin{corollary}
    If $U,V$ are finite dimensional vector spaces over $F$, then choosing a nondegenerate bilinear form $U\times V\to F$ is just choosing an isomorphism $\phi_L:U\to V^\ast$.
\end{corollary}
\begin{proof}
    Obvious.
\end{proof}
\begin{definition}
    For $T\subset U$, we define $T^\perp=\{v\in V:\forall t\in T,\phi(t,v)=0\}$ and for $S\subset V$ we define ${}^\perp S=\{u\in U:\forall s\in S,\phi(u,s)=0\}$
\end{definition}
Of course we want to change the basis.
\begin{proposition}\label{bilinear_change_of_basis}
    Let $B,B'$ be bases of $U$ and $P=[\operatorname{id}]_{B',B}$ and $C,C'$ be basis of $V$ and $Q=[\operatorname{id}]_{C',C}$ and let $\phi:U\times V\to F$ be a bilinear form.
    Then $[\phi]_{B',C'}=P^\top[\phi]_{B,C}Q$.
\end{proposition}
\begin{proof}
    We have
    $$\phi(u,v)=[u]_B^\top[\phi]_{B,C}[v]_C=(P[u]_{B'})^\top [\phi]_{B,C}(Q[v]_{C'})=[u]_{B'}^\top (P^\top[\phi]_{B,C}Q)[v]_{C'}$$
    So necessarily $[\phi]_{B',C'}=P^\top[\phi]_{B,C}Q$.
\end{proof}
\begin{lemma}
    The rank of the matrix of $\phi$ in any basis is fixed.
\end{lemma}
\begin{proof}
    Immediate.
\end{proof}
\begin{definition}
    The rank $r(\phi)$ of $\phi$ is the rank of its matrix in any basis.
\end{definition}
\begin{remark}
    We have $r(\phi)=r(\phi_R)=r(\phi_L)$.
\end{remark}


\section{Generation of Random Variables}
\begin{example}
    Suppose $U\sim\operatorname{Unif}(0,1)$ and set $Y=-\log U$, so $\mathbb P(Y\le x)=\mathbb P(U\ge e^{-x})=1-e^{-x}$ which is $\operatorname{Exp}(1)$.
\end{example}
\begin{theorem}
    Let $X$ be a continuous random variable with distribution $F$ and $U\sim\operatorname{Unif}(0,1)$, then $F^{-1}(U)$ has the same distribution as $X$.
\end{theorem}
\begin{proof}
    Obvious.
\end{proof}
Another way to generate is called the rejection sampling.
Suppose $A\subset [0,1]^d$ (measurable) and $f(x)=1_{A}(x)/|A|$ where $|A|$ is the volume of $A$.
We want a random variables $X$ to have density $f$.
Let $U_n$ be an i.i.d. sequence of $d$-dimensional uniforms, i.e. $U_n=(U_{k,n}:k=1,\ldots,d)$ where $U_{k,n}$ are i.i.d. $\operatorname{Unif}[0,1]$.
Let $N=\min\{n\ge 1:U_n\in A\}$ and set $X=U_N$.
\begin{proposition}
    Let $B\in[0,1]^d$, then $\mathbb P(X\in B)=|B\cap A|/|A|$.
\end{proposition}
\begin{proof}
    \begin{align*}
        \mathbb P(X\in B)&=\sum_{n=1}^\infty\mathbb P(X\in B,N=n)\\
        &=\sum_{n=1}^\infty\mathbb P(U_n\in A\cap B,U_{n-1}\notin A,\ldots,U_1\notin A)\\
        &=\sum_{n=1}^\infty |A\cap B|(1-|A|)^{n-1}\\
        &=\frac{|A\cap B|}{|A|}\\
        &\left(=\int_Bf(x)\,\mathrm dx\right)
    \end{align*}
    As desired.
\end{proof}
Let $f$ be a bounded density on $[0,1]^{d-1}$, that is $\exists\lambda,\sup f\le\lambda$ and we want to find $X\sim f$.
Consider $A=\{(x_1,\ldots,x_d)\in [0,1]^d:x_d\in f(x_1,\ldots,x_{n-1})/\lambda\}$.
Let $Y=(X_1,\ldots,X_d)$ be uniform on $A$ generated as above and set $X=(X_1,\ldots,X_{d-1})$.
\begin{proposition}
    $X\sim f$.
\end{proposition}
\begin{proof}
    Let $B\subset [0,1]^{d-1}$, then
    \begin{align*}
        \mathbb P(X\in B)&=\mathbb P((X_1,\ldots,X_d)\in (B\times [0,1])\cap A)\\
        &=\frac{(B\times [0,1]\cap A)}{|A|}\\
        &=\frac{1}{|A|}\int\cdots\int_{[0,1]^d}1_{(B\times [0,1])\cap A}((x_1\cdots x_d))\,\mathrm dx_1\cdots\mathrm dx_n\\
        &=\frac{1}{|A|}\int\cdots\int_{[0,1]^d}1_{B}((x_1,\ldots ,x_{d-1}))\frac{f(x_1,\ldots,x_{d-1})}{\lambda}\,\mathrm dx_1\cdots\mathrm dx_n\\
        &=\int_Bf(x)\,\mathrm dx
    \end{align*}
    which is what we wanted.
\end{proof}


If you haven't downloaded and unzipped \href{https://libaoj.in/courses/2021f/MATH3341/zip/Math.3341.zip}{\texttt{Math.3341.zip}}. Download and unzip it under \verb|H:| (H Drive if you are working on the Remote Lab). Change the current working directory by typing \verb|cd H:\Math.3341\Math.3341.Lab.09| in the Command Window, and type \verb|edit lab_09_script| in the Command Window to edit \verb|lab_09_script.m|.

%---------------------------------------------
\section{Ill-Conditioned Systems}
%---------------------------------------------
\begin{enumerate}[(a)]
    \item Open the function file \verb|lab_09_function.m|.
    \item Generate an $n \times n$ Hilbert matrix \verb`A`.
    \item Create an $n \times 1$ all-one vector \verb`b`.
    \item Now solve the ill-conditioned linear system $A x = b$ for $x$ by means of $x = A^{-1} b$:
        \begin{itemize}
            \item Use \verb|invhilb| to find $A^{-1}$, and store the exact solution $x$ to \verb|x_exact|.
            \item Use \verb|\| to solve the linear system and store the approximate solution $\hat{x}$ to \verb|x_backslash|.
            \item Use \verb|inv| to find $A^{-1}$, and store the approximate solution $\hat{x}$ to \verb|x_inv|.
        \end{itemize}
    \item Calculate the infinity norm of relative error of \verb|x_backslash| and \verb|x_inv| using the formula below, and store each relative error to \verb|error_backslash| and \verb|error_inv|, respectively.
        $$
        \mathrm{error}_{\mathrm{relative}} = \frac{\|\hat{x} - x\|_{\infty}}{\|x\|_{\infty}}.
        $$
    \item Calculate the 1-norm condition number of matrix \verb`A` using \verb|cond| and store the result to \verb`cond_A`.
    \item Next, go to \verb|lab_09_script.m|, create a vector named \verb|n|: \verb|n = [9, 11, 13, 15]|. Using a for-loop to call \verb|lab_09_function| by passing each entry of \verb|n| as the input argument. What do you notice about the results?
\end{enumerate}
%---------------------------------------------
\section{Finite Precision Arithmetic}
%---------------------------------------------
\begin{enumerate}[(a)]
    \item Use either a for-loop or \verb|sum| to calculate
            $$
            s = \sum_{i=1}^{10} 0.1 = 0.1 + 0.1 + \cdots + 0.1.
            $$
            Then calculate the error \verb|error_1 = abs(s - 1)|.
    \item Calculate
            $$
            b = 2 - 3 \left(\frac{4}{3} - 1\right).
            $$
            Then calculate the error \verb|error_2 = abs(b - 1)|.
    \item Calculate each side of the following equation in the variables \verb|lhs| and \verb|rhs|:
            $$
            1 + a + a^2 + a^3 + a^4 + a^5 = \frac{1 - a^6}{1 - a},
            $$
            for the value $a = 0.3$. Then calculate the error \verb|error_3 = abs(lhs - rhs)|. Hint: you can use either a for-loop or \verb`polyval` to calculate \verb|lhs|.
    \item Calculate the following and compare their output.
            % \begin{verbatim}
            \begin{lstlisting}{style=MATLAB}
x = 1e16 + 1 - 1e16;
y = 1e16 - 1e16 + 1;
z = 1e16 - (1e16 - 1);
            \end{lstlisting}
            % \end{verbatim}
    \item Calculate the following and compare their output.
            % \begin{verbatim}
            \begin{lstlisting}{style=MATLAB}
u = 1 + 0.1 - 1;
v = 1 - 1 + 0.1;
w = 1 - (1 - 0.1);
            \end{lstlisting}
            % \end{verbatim}
\end{enumerate}

Once you finish, call \verb|diary('lab_09_output.txt')|, then run the script file \verb|lab_09_script.m|, and call \verb|diary off| to save the resulting output. Then you need to upload files \verb|lab_09_script.m|, \verb|lab_09_function.m|, and \verb|lab_09_output.txt| to Overleaf, and answer the corresponding questions in \verb|body.tex|.


\section*{Exercises}

\begin{ex}
  Consider the following collection of points in $\R^2$:
  \begin{equation*}
    \set{
      \begin{mymatrix}{r} 1 \\ -1 \end{mymatrix},
      \begin{mymatrix}{r} -2 \\ 1 \end{mymatrix},
      \begin{mymatrix}{r} 3 \\ -2 \end{mymatrix},
      \begin{mymatrix}{r} 2 \\ -2 \end{mymatrix},
      \begin{mymatrix}{r} -2 \\ 1 \end{mymatrix},
      \begin{mymatrix}{r} 3 \\ -1 \end{mymatrix},
      \begin{mymatrix}{r} -3 \\ 0 \end{mymatrix},
      \begin{mymatrix}{r} 0 \\ 3 \end{mymatrix},
      \begin{mymatrix}{r} 0 \\ 0 \end{mymatrix},
      \begin{mymatrix}{r} 0 \\ -2 \end{mymatrix}
    }.
  \end{equation*}
  Find the 1-dimensional subspace that best approximates this
  collection of points. What is the total squared distance of the
  points to the subspace? Sketch the subspace and the points.
\end{ex}

\begin{ex}
  Consider the following collection of points in $\R^3$:
  \begin{equation*}
    \set{
      \begin{mymatrix}{r} 0 \\ 0 \\ 9 \end{mymatrix},
      \begin{mymatrix}{r} 1 \\ 0 \\ -3 \end{mymatrix},
      \begin{mymatrix}{r} 0 \\ 0 \\ 1 \end{mymatrix}
      \begin{mymatrix}{r} 0 \\ 5 \\ -4 \end{mymatrix},
      \begin{mymatrix}{r} 6 \\ -2 \\ 13 \end{mymatrix},
      \begin{mymatrix}{r} 1 \\ 0 \\ 0 \end{mymatrix},
      \begin{mymatrix}{r} 0 \\ 2 \\ 0 \end{mymatrix},
      \begin{mymatrix}{r} 0 \\ 1 \\ 1 \end{mymatrix},
      \begin{mymatrix}{r} 2 \\ 0 \\ 0 \end{mymatrix},
      \begin{mymatrix}{r} 4 \\ 0 \\ 0 \end{mymatrix},
      \begin{mymatrix}{r} 1 \\ -3 \\ 0 \end{mymatrix},
      \begin{mymatrix}{r} 0 \\ 0 \\ 4 \end{mymatrix}
    }.
  \end{equation*}
  \begin{enumerate}
  \item Find the 1-dimensional subspace that best approximates this
    collection of points.
  \item Find the 2-dimensional subspace that best approximates this
    collection of points.
  \item What is the 3-dimensional subspace that best approximates this
    collection of points?
  \end{enumerate}
  In each case, what is the total squared distance of the points to
  the subspace?
\end{ex}

\begin{ex}
  Find the principal components of the matrix
  \begin{equation*}
    A = \begin{mymatrix}{ccccccccc}
      0 & 0 & 0 & 0 & -1 & 0 & 3 & 0 & 1 \\
      0 & 1 & 0 & 0 & 2 & 3 & 0 & 2 & -1 \\
      1 & -1 & -3 & 3 & -1 & 0 & 0 & -2 & 2 \\
    \end{mymatrix}.
  \end{equation*}
\end{ex}

\begin{ex}
  Compute the centroid of the following collection of points:
  \begin{equation*}
    \set{
      \begin{mymatrix}{r} 2 \\ -1 \end{mymatrix},
      \begin{mymatrix}{r} 0 \\ 1 \end{mymatrix},
      \begin{mymatrix}{r} 4 \\ -3 \end{mymatrix},
      \begin{mymatrix}{r} -4 \\ -4 \end{mymatrix},
      \begin{mymatrix}{r} 0 \\ -2 \end{mymatrix},
      \begin{mymatrix}{r} -2 \\ -1 \end{mymatrix},
      \begin{mymatrix}{r} 0 \\ -2 \end{mymatrix},
      \begin{mymatrix}{r} -2 \\ -4 \end{mymatrix},
      \begin{mymatrix}{r} 3 \\ -1 \end{mymatrix},
      \begin{mymatrix}{r} -5 \\ -5 \end{mymatrix}
    }.
  \end{equation*}
\end{ex}

\begin{ex}
  Consider the following collection of points in $\R^2$:
  \begin{equation*}
    \set{
      \begin{mymatrix}{r} 4 \\ -3 \end{mymatrix},
      \begin{mymatrix}{r} 10 \\ -9 \end{mymatrix},
      \begin{mymatrix}{r} 4 \\ -7 \end{mymatrix},
      \begin{mymatrix}{r} -2 \\ 3 \end{mymatrix},
      \begin{mymatrix}{r} 10 \\ -3 \end{mymatrix},
      \begin{mymatrix}{r} 4 \\ -3 \end{mymatrix},
      \begin{mymatrix}{r} 5 \\ -3 \end{mymatrix},
      \begin{mymatrix}{r} 4 \\ 1 \end{mymatrix},
      \begin{mymatrix}{r} -2 \\ -3 \end{mymatrix},
      \begin{mymatrix}{r} 3 \\ -3 \end{mymatrix}
    }.
  \end{equation*}
  Compute the centroid, and then find the 1-dimensional affine
  subspace that best approximates this collection of points. What is
  the total squared distance of the points to the subspace?
\end{ex}

\begin{ex}
  Consider the following collection of points in $\R^3$:
  \begin{equation*}\scriptstyle
    \set{
      \begin{mymatrix}{r} 0 \\ -2 \\ 2 \end{mymatrix},
      \begin{mymatrix}{r} 6 \\ 2 \\ 2 \end{mymatrix},
      \begin{mymatrix}{r} 2 \\ 2 \\ 6 \end{mymatrix},
      \begin{mymatrix}{r} 3 \\ 2 \\ 2 \end{mymatrix},
      \begin{mymatrix}{r} 3 \\ 0 \\ 5 \end{mymatrix},
      \begin{mymatrix}{r} 5 \\ 0 \\ 2 \end{mymatrix},
      \begin{mymatrix}{r} 4 \\ -2 \\ -2 \end{mymatrix},
      \begin{mymatrix}{r} 5 \\ 0 \\ 2 \end{mymatrix},
      \begin{mymatrix}{r} 1 \\ 0 \\ 2 \end{mymatrix},
      \begin{mymatrix}{r} 3 \\ 0 \\ -1 \end{mymatrix},
      \begin{mymatrix}{r} 3 \\ 0 \\ 1 \end{mymatrix},
      \begin{mymatrix}{r} 3 \\ 0 \\ 3 \end{mymatrix},
      \begin{mymatrix}{r} 1 \\ 0 \\ 2 \end{mymatrix},
      \begin{mymatrix}{r} 3 \\ -2 \\ 2 \end{mymatrix}
    }.
  \end{equation*}
  Find the 1- and 2-dimensional affine subspaces that best approximate
  this collection of points. What is the total squared
  distance of the points to each subspace?
\end{ex}


\section{Bifurcation Diagrams as Tangent Fields}
\label{appendix:tangent-fields}

Let each component of the vector function $\rates$ in the model \eqref{eq:model} implicitly define a surface embedded in $\Reals^{N+1}$. Let's assume that the intersection of these $N$ surfaces exists and is not null or degenerate, then the steady states of \eqref{eq:model} must be a set of one dimensional space curves in $z\in\Reals^{N+1}$ defined by
\begin{align}
    \rates(z) = 0
\end{align}
\begin{figure}[H]
\centering
\includegraphics[width=5cm]{implicit-surfaces}
\caption{Two implicit surfaces $f_{\theta}(z)=0$ and $g_{\theta}(z)=0$ in $\mathbb{R}^3$ intersecting to form a space curve which is tangent to field $\tangent(z)$ and perpendicular to gradients $\partial_{z}f_{\theta}$ and $\partial_{z}g_{\theta}$}
\label{fig:implicit-surfaces}
\end{figure}
An expression for the field $\tangent(z)$ tangent to the set of curves would allow us to take derivatives and integrals along the bifurcation curve. This is exactly what we need to do to evaluate our cost function \ref{eq:loss}. Fortunately the tangent field can be constructed by ensuring it is perpendicular to the gradient $\partial_z$ of each component of $\rates$ as illustrated by an example two component system in Figure \ref{fig:implicit-surfaces}. The tangent field $\tangent(z)$ can be constructed perpendicular to all gradient vectors using the properties of the determinant \cite{Goldman2005CurvatureSurfaces}
\begin{align}
    \tangent(z):=
    \label{eq:tangent-field}
    \left|\begin{matrix}
        \hat{z} \\
        \,\partial_{z}\rates\,
    \end{matrix}\right|
    \qquad\tangent : \Reals^{N+1}\rightarrow\Reals^{N+1}\\
    =\sum_{i=1}^{N+1}\hat{z}_{i}(-1)^{i+1} \left|\frac{\partial \rates}{\partial(z\setminus z_{i}) }\right|
\end{align}
where $\hat{z}$ is a collection of unit basis vectors in the $\Reals^{N+1}$ space and $\partial_{z}\rates$ is an $N\times(N+1)$ rectangular Jacobian matrix of partial derivatives and $z\setminus z_{i}$ denotes the $N$ dimensional vector $z$ with component $z_{i}$ removed. This construction ensures perpendicularity to any gradients of $\rates$
\begin{align}
    \tangent(z)\cdot\partial_z f_{\theta} =
    \left|\begin{matrix}
        \partial_z f_{\theta} \\
        \,\partial_{z}\rates\,
    \end{matrix}\right|
    \quad =0 \quad\forall f_{\theta}\in \rates
\end{align}
since the determinant of any matrix with two identical rows or columns is zero. Note that the tangent field $\tangent(z)$ is actually defined for all values of $z$ where adjacent field lines trace out other level sets where $\rates(z)\neq0$. Furthermore deformations with respect to $\theta$ are always orthogonal to the tangent
\begin{align} % \todo{numerically true. analytic proof?}
    \tangent(z)\cdot\frac{d\tangent}{d\theta}=0
\end{align}
\begin{figure}
\centering
\includegraphics[width=13cm]{determinant-field}
\caption{Left/Right : Determinant $\Det$ and tangent field $\tangent(z)$ for the saddle-node/pitchfork models for some set values of $\theta$ revealing that $\Det=0$ defines bifurcations}
\label{fig:determinant-field}
\end{figure}
Figure \ref{fig:determinant-field} shows how the bifurcation curve defined by $\rates(z)=0$ picks out one of many level sets or traces in tangent field $\tangent(z)$ for the saddle and pitchfork. The tangent field $\tangent(z)$ can always be analytically evaluated by taking the determinant in \eqref{eq:tangent-field}. We will proceed with calculations on $\tangent(z)$ in the whole space $z$ and pick out a single trace by solving $\rates(z)=0$ later. For our two models
\begin{align}
    \underset{\mathrm{saddle-node\,\,model}}{
    \tangent(z)=\hat{u}-(\,3\theta_2 u^2+\theta_1\,)\,\hat{p}}
    \qquad\qquad
    \underset{\mathrm{pitchfork\,\,model}}{
    \tangent(z)=u\hat{u}-(\,3\theta_2 u^2+p\,)\,\hat{p}}
    \label{eq:tangent-field-examples}
\end{align}
Figure \ref{fig:determinant-field} reveals that $\Det=0$ is also a level set and that the intersection with level set $\rates(z)=0$ defines the bifurcations at specific parameter $\theta$. In this particular setting we can see that the tangent field $\tangent(z)$ only folds when $\Det=0$. Plotting the value of the determinant along $\rates(z)=0$ from Figure \ref{fig:determinant-field} would give rise to Figures \ref{fig:minimal-models}. The directional derivative of the determinant $\Det$ along the tangent field $\tangent(z)$ is defined as
\begin{align}
    \frac{d}{ds}\Det := \hat{\tangent}(z) \cdot \frac{\partial}{\partial z}\Det
\end{align}
where $\hat{\tangent}(z)$ is the unit tangent field.

\section{Bifurcation Measure Properties}
\label{appendix:conditions}
Consider a vector $v(s)\in\Reals^N$ parametrised by $s\in\Reals$ that is tangent to an equilibrium manifold defined by $\rates(u)=0$. The conditions for a non-degenerate static bifurcation at $s^*$ along such a tangent can be expressed in terms of an eigenvalue $\lambda(s)$ of the state-space Jacobian crossing zero with a finite slope. A bifurcation exists at $s^*$ if
\begin{align}
    \frac{\partial\rates}{\partial u}v(s)=\lambda(s) \,v(s)
    \quad\exists\lambda:\quad
    \left.\lambda(s)\right|_{s=s^*}=0
    \qquad
    \left.\frac{d\lambda}{ds}\right|_{s=s^*}\neq 0
    \label{eq:conditions}
\end{align}
These conditions are necessary and sufficient for a non-degenerate static local breakdown of stability. For now we do not consider dynamic bifurcations involving limit cycles or imaginary parts of eigenvalues and restrict $\lambda\in\Reals$. Cases where both $\left.\lambda(s)\right|_{s=s^*}=0$ and $\left.\frac{d\lambda}{ds}\right|_{s=s^*}=0$ require investigation into higher order derivatives $\frac{d^n\lambda}{ds^n}$. These are the cases we refer to as \emph{degenerate} and are not considered here.

Instead of considering conditions on each eigenvalue individually it is possible to use the determinant of the state-space Jacobian to detect whether the conditions \eqref{eq:conditions} are satisfied. The determinant can be expressed as the product of eigenvalues
\begin{equation}
    \Det=\prod_{n=1}^N\lambda_n(s)
    \label{eq:determinant}
\end{equation}
Applying the product rule when differentiating yields
\begin{align}
    \frac{d}{ds}\Det&=
    \sum_{n=1}^N\frac{d\lambda_n}{ds}\prod_{n'\neq n}\lambda_{n'}(s)\\
    &=\Det\sum_{n=1}^N\frac{d\lambda_n}{ds}\lambda_{n}(s)^{-1}
\end{align}
Substituting this expression into measure \eqref{eq:measure}
\begin{equation}
    \measure(s)=
    \left(1+\left|\sum_{n=1}^N\frac{d\lambda_n}{ds}\lambda_{n}(s)^{-1}\right|^{-1}\right)^{-1}
\end{equation}
Which implies the following
\begin{equation}
    \exists\lambda:\quad
    \begin{cases}
        \,\lambda(s)=0 \quad\frac{d\lambda}{ds}\neq 0\\
        \,\lambda(s)\neq0 \quad\frac{d\lambda}{ds}\rightarrow\pm\infty
    \end{cases}
    \implies
    \measure(s)=1
    \label{eq:measure-conditions}
\end{equation}
If there exists an eigenvalue that satisfies conditions \eqref{eq:conditions} then the measure is equal to one. The measure also approaches one in cases where the rate of change of an eigenvalue with respect to a manifold $s$ location  diverges while not crossing zero. This gives rise to finite gradients in the eigenvalue term in regimes far away from any bifurcation.

\section{Leibniz Rule for Space Curves}
\label{appendix:leibniz-rule}

Suppose there exists a one dimensional space curve $\mathcal{C(\theta)}$ embedded in $z\in\Reals^{N+1}$ whose geometry changes depending on input parameters $\theta\in\Reals^M$. This curve could be open or closed and changes in $\theta$ could change the curve topology as well. Let the function $\gamma_{\theta}:\Reals\rightarrow\Reals^{N+1}$ be a parametrisation of the position vector along the curve within a fixed domain $s\in\mathcal{S}$. Note that the choice of parametrisation is arbitrary and our results should not depend on this choice. Furthermore, if we parametrise the curve $\mathcal{C}(\theta)$ with respect to a fixed domain $\mathcal{S}$ the dependence on $\theta$ is picked up by the parametrisation $\gamma_{\theta}(s)$. We can write a line integral of any scalar function $L_{\theta}:\Reals^{N+1}\rightarrow\Reals$ on the curve as
\begin{align}
    L(\theta):=
    \int_\mathcal{C(\theta)}\! L_{\theta}(z)\,\mathrm{d}z
    =\int_\mathcal{S}\! L_{\theta}(z)\left|\frac{d\gamma_{\theta}}{ds}\right|\mathrm{d}s_{\,\,z=\gamma_{\theta}(s)}
\end{align}
where $\left|\frac{d\gamma_{\theta}}{ds}\right|$ is the magnitude of tangent vectors to the space curve and we remind ourselves that the integrand is evaluated at $z=\gamma_{\theta}(s)$. We would like to track how this integral changes with respect to $\theta$. The total derivative with respect to $\theta$ can be propagated into the integrand \cite{Flanders1973DifferentiationSign} as long as we keep track of implicit dependencies
\begin{align}
    \frac{dL}{d\theta} &=\int_\mathcal{S}
    \left|\frac{d\gamma_{\theta}}{ds}\right|
    \left(
        \frac{\partial L}{\partial\theta}+
        \frac{\partial L}{\partial z}\cdot
        \frac{dz}{d\theta}
    \right)
    +L_{\theta}(z)\frac{d}{d\theta}\left|\frac{d\gamma_{\theta}}{ds}\right|
    \mathrm{d}s_{\,\,z=\gamma_{\theta}(s)}
\end{align}
Here we applied the total derivative rule in the first term due to the implicit dependence of $z$ on $\theta$ through $z=\gamma_{\theta}(s)$. Applying the chain rule to the second term
\begin{align}
    \frac{d}{d\theta}\left|\frac{d\gamma_{\theta}}{ds}\right|=
    \left|\frac{d\gamma_{\theta}}{ds}\right|^{-1}
    \frac{d\gamma_{\theta}}{ds}\cdot\frac{d}{d\theta}
    \left(\frac{d\gamma_{\theta}}{ds}\right)
\end{align}
By choosing an $s$ that has no implicit $\theta$ dependence we can commute derivatives
\begin{align}
    \frac{d}{d\theta}\left(\frac{d\gamma_{\theta}}{ds}\right)
    = \frac{d}{ds}\left(\frac{d\gamma_{\theta}}{d\theta}\right)
    \quad\Rightarrow\quad
    \frac{d}{d\theta}\left|\frac{d\gamma_{\theta}}{ds}\right|=
    \left|\frac{d\gamma_{\theta}}{ds}\right|^{-1}
    \frac{d\gamma_{\theta}}{ds}\cdot\frac{d}{d s}
    \left(\frac{d\gamma_{\theta}}{d\theta}\right)
\end{align}
To proceed we note that the unit tangent vector can be written as an evaluation of a tangent field $\hat{T}_{\theta}(z)$ defined in the whole domain $z\in\Reals^{N+1}$ along the parametric curve $z=\gamma_{\theta}(s)$. The unit tangent field may disagree with the tangent given by $\frac{d\gamma_{\theta}}{ds}$ up to a sign
\begin{align}
    \left.\hat{\tangent}(z)\right|_{z=\gamma_{\theta}(s)}=
    \pm\left|\frac{d\gamma_{\theta}}{ds}\right|^{-1}\frac{d\gamma_{\theta}}{d s}
\end{align}
this leads to
\begin{align}
    \frac{d}{d\theta}\left|\frac{d\gamma_{\theta}}{ds}\right|=
    \left|\frac{d\gamma_{\theta}}{ds}\right|\left(
   \hat{\tangent}(z)\cdot\frac{\partial }{\partial z}\left(\frac{d\Gamma_{\theta}}{d \theta}\right)\cdot
   \hat{\tangent}(z)
    \right)_{z=\gamma_\theta(s)}
    \label{eq:divergence-term}
\end{align}
It is possible to find the normal deformation of the implicit space curves due to changes in $\theta$. This can be done by taking the total derivative of the implicit equation defining the level set
\begin{align}
    \frac{d\rates(z)}{d\theta}=\frac{\partial F}{\partial\theta}+
    \frac{\partial F}{\partial z}\cdot\frac{d z}{d \theta}
\end{align}
We can rearrange for $\frac{d z}{d \theta}$ using the Moore-Penrose inverse of the rectangular Jacobian matrix $\frac{\partial F}{\partial z}$ which appeared in equation \eqref{eq:tangent-field}. Since the level set is defined by $\rates(z)=0$ the total derivative along the level set $d\rates(z)=0$ and we arrive at an expression for the deformation field \cite{Jos2011OnSurface}
\begin{align}
    \frac{d z}{d \theta} = - \frac{\partial F}{\partial z}^\top
    \left(\,
        \frac{\partial F}{\partial z}\,\frac{\partial F}{\partial z}^\top
    \right)^{-1}
    \frac{\partial F}{\partial\theta}
\end{align}
The tangential component of the deformation field is not uniquely determined because there is no unique way of parametrising a surface. This is the subject of many computer graphics papers \cite{Jos2011OnSurface,Tao2016Near-IsometricTracking,Fujisawa2013CalculationInvariance}. We are however not interested in the continuous propagation of a mesh - as is the subject of those papers. In fact we are looking for a deformation field that is orthogonal to the tangent vector $\hat{\tangent}(z) \cdot\frac{d z}{d\theta} =0$ for the space curve, and therefore letting the tangential component of the deformation equal zero is a valid choice and we can it instead of the parametrised deformation
\begin{align}
    \frac{d \gamma_{\theta}}{d\theta} \rightarrow \frac{d z}{d\theta}
\end{align}
To summarise we now have the gradient of our line integral only in terms of the implicit function defining the integration region.
\begin{align}
    \frac{d L}{d\theta} =\int_{\rates(z)=0}
        \frac{\partial L}{\partial\theta}+
        \frac{\partial L}{\partial z}\cdot
        \varphi_{\theta}(z)
    +L_{\theta}(z)\,\,
    \hat{\tangent}(z)\cdot\frac{\partial \varphi}{\partial z}\cdot\hat{\tangent}(z)
    \,\mathrm{d}z\qquad\qquad\\
    \mathrm{where}\quad
    \hat{\tangent}(z):= \frac{\tangent(z)}{|\tangent(z)|}
    \qquad
    \tangent(z):=
    \left|\begin{matrix}
        \hat{z} \\
        \,\partial_{z}\rates\,
    \end{matrix}\right|
    \qquad
    \varphi_{\theta}(z) :=
- \frac{\partial F}{\partial z}^\top
    \left(\,
        \frac{\partial F}{\partial z}\,\frac{\partial F}{\partial z}^\top
    \right)^{-1}
    \frac{\partial F}{\partial\theta}
\end{align}
We have settled on choosing normal deformations which we will call $\varphi_\theta(z)$. The above result can be seen a the generalised Leibniz rule \cite{Flanders1973DifferentiationSign} for the case of line integration regions. The last integrand term can be seen as the divergence the vector field $\varphi_\theta(z)$ projected onto the one dimensional space curve.

\clearpage

\section{Application of Bifurcation Inference to a Complex Model}
\label{appendix:double-exclusive}

To demonstrate the wider reaching applicability of our method we optimise the \emph{double exclusive reporter} \cite{Grant2020InterpretationCircuit}, a synthetic gene circuit in \emph{E. coli} that was designed to exhibit a cusp bifurcation. The circuit behaviour is observed by measuring a fluorescent protein whose expression is controlled by transcription factors (regulatory proteins) LacI $(L)$ and TetR $(T)$, whose expression is in turn controlled by externally controllable \emph{input} signals $c_{6}$ and $c_{12}$. To apply the method, we consider one of the input signals be the control condition $c_{6}=p$, with the other packed together with the remaining 20 parameters into vector $\theta$. Once the optima $\theta^*$ have been obtained, we perform dimensionality reduction using \texttt{GigaSOM.jl} \cite{Kratochvil2020GigaSOM.jl:Datasets} so that the results can be visualised in a two dimensional embedding (Figure \ref{fig:double-exclusive-optima:parameters}).

The embedding reveals four optimal parameter regions. We find that, as with the two-state model in the main text \eqref{eq:two-state}, there are two qualitatively distinct regimes: mutual activation (region 1) and inhibition (regions 2-4). The mutual inhibition region can be further subdivided into three regions that are geometrically equivalent, but kinetically distinct: region 3 has swapped kinetic roles for regulatory proteins LacI and TetR compared to region 2, and region 4 has additional damped oscillations in the dynamics across the whole range of \emph{input} $c_{6}$ (Figure \ref{fig:double-exclusive-optima:models}). The two dimensional embedding of sampled optima $\theta^*$ enables navigation the space of qualitative behaviours of the \emph{double exclusive reporter} and organisation in terms of geometric and kinetic equivalence.

\begin{figure}[ht]
\centering
\setlength\unitlength{1cm}
{\phantomsubcaption\label{fig:double-exclusive-optima:parameters}}
{\phantomsubcaption\label{fig:double-exclusive-optima:models}}
\includegraphics[width=13cm]{double-exclusive-optima}
\begin{picture}(0,0) \put(-13,7){\subref{fig:double-exclusive-optima:parameters}} \end{picture}
\begin{picture}(0,0) \put(-7,7){\subref{fig:double-exclusive-optima:models}}
\end{picture}
\caption{Bifurcation inference for the \emph{double exclusive reporter}. \subref{fig:double-exclusive-optima:parameters}. Optimal parameter estimates $\theta^*$ for the targets $\targets=\{1,2\}$ (indicated by yellow lines in panel B) reveal four regions  with two geometrically different regimes: mutual activation (region 1) and mutual inhibition (regions 2-4). \subref{fig:double-exclusive-optima:models}. Example bifurcation diagrams indicate that region 2 has swapped kinetics between $L$ and $T$ to region 3. Region 4 has models with non-zero imaginary parts to eigenvalues indicating damped oscillations (shown in light green).}
\label{fig:double-exclusive-optima}
\end{figure}

These results were obtained with a modification of the bifurcation measure \eqref{eq:measure} to improve convergence rates. In parameter regimes where bifurcations are not present, according to conditions \eqref{eq:measure-conditions}, maximising the measure $\measure(s)$ can lead to a divergence in directional derivative $\frac{d\lambda}{ds}\rightarrow\pm\infty$ rather than a creation of a bifurcation. To discourage this from happening we can flatten out the gradients in that regime by applying the $\tanh$ non-linearity to the determinant. This leads to
\begin{align}
    \measure(s):=
    \left(1+\left|\frac{\tanh\Det}{\frac{d}{ds}\tanh\Det}\right|\right)^{-1}
    \label{eq:measure-tanh}
\end{align}

\clearpage
\section{Extension for Hopf Bifurcations}
\label{appendix:hopf}

In order to detect bifurcations involving limit cycles, the measure must be extended to detect changes in the real part $\Re\mathrm{e}[\lambda(s)]$ for any eigenvalue of the Jacobian. These conditions can no longer be compactly written in terms of the determinant. Instead, the measure can be defined as the sum of eigenvalue terms
\begin{align}
    \measure(s):=\sum_{\lambda(s)\in\frac{\partial\rates}{\partial u}}
    \left(\left|\frac{d}{ds}\log\Re\mathrm{e}[\lambda(s)]\right|^{-1}+1\right)^{-1}
    \label{eq:hopf-measure}
\end{align}

The directional derivative of the logarithm diverges under two conditions: when eigenvalues vanish $\lambda(s)=0$ and when the directional derivative $\frac{d}{ds}\Re\mathrm{e}[\lambda(s)]$ diverges. These properties are sufficient for detecting the onset of damped oscillations and emergence of limit cycles via Hopf bifurcation as shown in Figure \ref{fig:hopf-measure}. Eigenvalues with negative real part which gain a finite imaginary part give rise to damped oscillations. At this onset we observe a discontinuity in the derivative $\frac{d}{ds}\Re\mathrm{e}[\lambda(s)]$ which is detected by equation \eqref{eq:hopf-measure}. Once damped oscillations exist, flipping the stability of the stable fixed point gives rise to a limit cycle, which can be detected by inspecting $\Re\mathrm{e}[\lambda(s)]$.

\begin{figure}[H]
\centering
    \includegraphics[width=60mm]{hopf-measure}
    \caption{Bifurcation measure $\measure(s)$ and eigenvalues $\lambda(s)$ along the arclength $s$ for two different bifurcation curves demonstrating how the measure detects non-zero imaginary parts $\Im\mathrm{m}[\lambda]$ (onset of damped oscillations marked by circle) and sign changes in real parts $\Re\mathrm{e}[\lambda]$ (Hopf bifurcations marked by stars)}
    \label{fig:hopf-measure}
\end{figure}

In principle it is possible to construct measures to detect a variety of bifurcations as long as the conditions can be expressed in terms of derivatives with respect to fixed-point manifold direction $s$. Measures can be used sequentially or in parallel to encourage optimisers to run through a sequence of bifurcations or place specific bifurcation types next to each other.


\section{Problem Statement}

\subsection{Input}

\begin{enumerate}
    \item a directed acyclic graph $\graph = \tuple{\vertices,\edges}$;
    \item a multi-dimensional weight function $\function{\weight}{\vertices}{\weightCodomain}$, where $\wn \in \natural$;
        \subitem We will usually write $\weightE = \apply{\weight}{\solutionE}$
    \item a maximum capacity of the knapsack $\maximumWeight \in \weightCodomain$;
\end{enumerate}

Besides that, one requires the input to satisfy the constraints below \cite{bib:constrained-knapsack}, otherwise the problem would be trivial:

\begin{enumerate}
    \item $\weightE \leqslant \maximumWeight$: the weight of each vertex must be smaller than the knapsack capacity;
    \item $\Sum{\solutionE \in \vertices}{}{\weightE} \geqslant \maximumWeight$: the weight of all vertices combined must be greater than the knapsack capacity;
\end{enumerate}

\subsubsection{Partial Order}

\begin{defn}[Partial Order on Directed Acyclics Graph]
    Given a directed acyclic graph $\graph = \tuple{\vertices,\edges}$, we define the set:
    \begin{equation}
        \partialLower
        \ =
        \SetOf
            {\tuple{\solutionE, \solutionE'}}
            {\mbox{there is a path from the first to the second}}
    \end{equation}
    and so $\partialLower$ is a partial order over the set $\vertices$.
\end{defn}

\begin{figure}[ht!]
    \centering
    \includegraphics[width=0.5\textwidth]{images/directed acyclic graph.png}
    \caption{Example of a directed acyclic graph. The black dot indicates the root vertices. For this case, the induced partial order satisfy: $v5 \partialLower v2$, $v7 \partialLower v1$, $v14 \partialLower v10$.}
\end{figure}

\subsection{Output}

A subset $\solution \subseteq \vertices$ of the vertices which satisfy:

\begin{equation}
    \label{eq:capacity-constraint}
    \Sum
        {\solutionE \in \solution}
        {}
        {\weightE}
    \leqslant
    \maximumWeight
\end{equation}

\begin{equation}
    \label{eq:precedence-constraint}
    \forAll
        {\solutionE}
        {\ifThen
            {\solutionE \in \solution}
            {\forAll
                {\solutionEp}
                {\ifThen
                    {\solutionEp \partialLower \solutionE}
                    {\solutionEp \in \solution}
                }
            }
        }
\end{equation}

\eqref{eq:capacity-constraint} is the maximum weight constraint, the total weight of all vertices in the solution set $\solution$ must not be greater than the weight limit $\maximumWeight$. It is called Capacity-Constraint.

\eqref{eq:precedence-constraint}\footnote{It is a First-order logic expression \cite{bib:logic}.} says that, if a $\solutionE$ is included in the solution, then all the $\solutionEp$ lower than it (in the sense of the partial order $\partialLower$) must also be included. It is called Precedence-Constraint.

\subsection{Objective}

Find $\solution$ that maximizes $\abs{\solution}$. In other words: find the solution with the maximum number of vertices.