\section{Question 1}
\begin{enumerate}[label=(\alph*)]
    \item Write down the definition of \textbf{norm} for a vector space.
        \begin{solution}{}{}
            Let $\mX$ be a real vector space. A nonnegative-valued function $\norm{\cdot}: \mX\to\RR$ is called a norm if $\forall\vx,\vy\in\mX, \lambda\in\RR$
            \begin{itemize}
                \item $\norm{\lambda\vx}=\abs{\lambda}\norm{\vx}$ (absolutely homogeneous);
                \item $\norm{\vx+\vy}\leq\norm{\vx}+\norm{\vy}$ (triangle inequality);
                \item $\norm{\vx}=0\Rightarrow\vx =\va{0}$ (positive definite).
            \end{itemize}
            Moreover, the vector norm $\norm{\cdot}_p$ for $p=1,2,\ldots$ is defined as
            \[
                \norm{\vx}_p = \qty(\sum_{1\leq i\leq n}\abs{x_i}^p)^{1/p}.
            \]
            And the pair $(\mX, \norm{\cdot})$ is called a normed space.
        \end{solution}
    \item Given $\vx\in\RR^n$, show that followings are norm on $\RR^n$.
        \begin{enumerate}[label=\roman*.]
            \item $\norm{\vx}_\infty=\max_{1\leq i\leq n}\abs{x_i}$;
                \begin{solution}{}{}
                    \begin{itemize}
                        \item $\norm{\lambda\vx}_\infty=\max_{1\leq i\leq n}\abs{\lambda x_i}=\abs{\lambda}\max_{1\leq i\leq n}\abs{x_i}=\abs{\lambda}\norm{\vx}_\infty$
                        \item $\norm{\vx+\vy}_\infty=\max_{1\leq i\leq n}\abs{x_i+y_i}\leq\max_{1\leq i\leq n}\abs{x_i}+\max_{1\leq i\leq n}\abs{y_i}=\norm{\vx}_\infty+\norm{\vy}_\infty$
                        \item $\norm{\vx}_\infty=\max_{1\leq i\leq n}\abs{x_i}=0\Rightarrow\abs{x_i}=0\text{ where $1\leq i\leq n$}\Rightarrow\vx =\va{0}$
                    \end{itemize}
                \end{solution}
            \item $\norm{\vx}_1=\sum_{1\leq i\leq n}\abs{x_i}$;
                \begin{solution}{}{}
                    \begin{itemize}
                        \item $\norm{\lambda\vx}_1=\sum_{1\leq i\leq n}\abs{\lambda x_i}=\abs{\lambda}\sum_{1\leq i\leq n}\abs{x_i}=\abs{\lambda}\norm{\vx}_1$
                        \item $\norm{\vx+\vy}_1=\sum_{1\leq i\leq n}\abs{x_i+y_i}\leq\sum_{1\leq i\leq n}\abs{x_i}+\sum_{1\leq i\leq n}\abs{y_i}=\norm{\vx}_1+\norm{\vy}_1$
                        \item $\norm{\vx}_1=\sum_{1\leq i\leq n}\abs{x_i}=0\Rightarrow\abs{x_i}=0\text{ where $1\leq i\leq n$}\Rightarrow\vx =\va{0}$
                    \end{itemize}
                \end{solution}
            \item $\norm{\vx}_2=\qty(\sum_{1\leq i\leq n}\abs{x_i}^2)^{1/2}$.
                \begin{solution}{}{}
                    \begin{itemize}
                        \item $\norm{\lambda\vx}_2=\qty(\sum_{1\leq i\leq n}\abs{\lambda x_i}^2)^{1/2}=\abs{\lambda}\qty(\sum_{1\leq i\leq n}\abs{x_i}^2)^{1/2}=\abs{\lambda}\norm{\vx}_2$
                        \item $\norm{\vx+\vy}_2=\qty(\sum_{1\leq i\leq n}\abs{x_i+y_i}^2)^{1/2}\leq\qty(\sum_{1\leq i\leq n}\abs{x_i}^2)^{1/2}+\qty(\sum_{1\leq i\leq n}\abs{y_i}^2)^{1/2} = \norm{\vx}_2+\norm{\vy}_2$
                        \item $\norm{\vx}_2=\qty(\sum_{1\leq i\leq n}\abs{x_i}^2)^{1/2}=0\Rightarrow\sum_{1\leq i\leq n}\abs{x_i}^2\Rightarrow\vx =\va{0}$
                    \end{itemize}
                \end{solution}
        \end{enumerate}
        Plot the regions for $\norm{\vx}_p\leq 1$ in $\RR^2$ for $p=\infty,1,2$ respectively.
        \begin{figure}[H]
            \centering
            \includegraphics[width=.9\textwidth]{figures/1-1}
            \caption{Please refer to Appendix~\ref{S:appendix-1}}
        \end{figure}
    \item Given the vector norm $\norm{\cdot}_p$ for $\RR^n$, the induced norm for matrices $\mA\in\RR^{n\times n}$ is defined as
        \[
            \norm{\mA}_p = \max_{\vx\neq 0}\frac{\norm{\mA\vx}_p}{\norm{\vx}_p} = \max_{\norm{\vx}_p=1}\norm{\mA\vx}_p.
        \]
        Show that
        \begin{enumerate}[label=\roman*.]
            \item $\norm{\mA}_\infty=\max_{1\leq i\leq n}\sum_{j=1}^n\abs{a_{ij}}$;
                \begin{solution}{}{}
                    \begin{align*}
                        \norm{\mA}_\infty &= \max_{\norm{\vx}_\infty=1}\norm{\mA\vx}_\infty \\
                        &= \max_{\max_{1\leq j\leq n}\abs{x_j}=1}\max_{1\leq i\leq n}\sum_{j=1}^n\abs{a_{ij}x_j} \\
                        &\leq \max_{1\leq i\leq n}\sum_{j=1}^n\abs{a_{ij}} \\
                    \end{align*}
                    The equality holds when $x_j=\pm 1$ for $j=1,2,\ldots,n$.
                \end{solution}
            \item $\norm{\mA}_1=\max_{1\leq j\leq n}\sum_{i=1}^n\abs{a_{ij}}$;
                \begin{solution}{}{}
                    \begin{align*}
                        \norm{\mA}_1 &= \max_{\norm{\vx}_1=1}\norm{\mA\vx}_1 \\
                        &= \max_{\sum_{j=1}^n\abs{x_j}=1}\sum_{i=1}^n\sum_{j=1}^n\abs{a_{ij}x_j} \\
                        &\leq \max_{\sum_{j=1}^n\abs{x_j}=1}\sum_{j=1}^n\sum_{i=1}^n\abs{a_{ij}}\abs{x_j} = \max_{\sum_{j=1}^n\abs{x_j}=1}\sum_{j=1}^n\abs{x_j}\sum_{i=1}^n\abs{a_{ij}} \\
                        &\leq \max_{\sum_{j=1}^n\abs{x_j}=1}\sum_{j=1}^n\abs{x_j}\max_{1\leq j\leq n}\sum_{i=1}^n\abs{a_{ij}} = \max_{1\leq j\leq n}\sum_{i=1}^n\abs{a_{ij}}
                    \end{align*}
                    The equality holds when $j=\argmax_{1\leq j\leq n}\sum_{i=1}^n\abs{a_{ij}}$.
                \end{solution}
            \item $\norm{\mA}_2=\sqrt{\lambda_{\max}\qty(\mA^T\mA)}$.
                \begin{solution}{}{}
                    $\mA$ can be decomposed as $\vb{U}\vb*{\Sigma}\vb{V}^T$ by singular value decomposition, and the singular values on the diagonal are decreasing from top left to bottom right.
                    \begin{align*}
                        \norm{\mA}_2^2 &= \max_{\norm{\vx}_2=1} \norm{\mA\vx}_2^2 \\
                        &= \max_{\norm{\vx}_2=1} \qty(\mA\vx)^T\mA\vx = \max_{\norm{\vx}_2=1}\vx^T\vb{V}\vb*{\Sigma}^2\vb{V}^T\vx \\
                        &= \max_{\norm{\vx}_2=1}\vy^T\vb*{\Sigma}\vy = \max_{\norm{\vx}_2=1}\sum_{i=1}^n\sigma_iy_i^2 \\
                        &\leq \max_{\norm{\vx}_2=1}\max_{1\leq i\leq n}\sigma_i \vy^T\vy = \max_{\norm{\vx}_2=1}\max_{1\leq i\leq n}\sigma_i \vx^T\vx\\
                        &= \max_{1\leq i\leq n}\sigma_i = \lambda_{\max}\qty(\mA^T\mA).
                    \end{align*}
                    The equality holds when $\vx=\vb{V}\vb{e}_1$, where $\vb{e}_1=[1,0,\ldots,0]^T$.
                \end{solution}
        \end{enumerate}
\end{enumerate}

\section{Question 2}
\begin{enumerate}[label=(\alph*)]
    \item Use the method of undetermined coefficients to design third order accurate approximation to $u'(\bar{x})$ by using the discrete points $\bar{x}+h$, $\bar{x}$, $\bar{x}-h$, $\bar{x}-2h$.
        \begin{solution}{}{}
            According to Appendix~\ref{S:appendix-1}, we have
            \[
                u'(\bar{x}) = \frac{1}{h}\qty(\frac{u(\bar{x}-2h)}{6}-u(\bar{x}-h)+\frac{u(\bar{x})}{2}+\frac{u(\bar{x}+h)}{3}).
            \]
        \end{solution}
    \item Assuming $u(x)$ is smooth enough, compute the truncation error (leading term) for the finite difference formula above.
        \begin{solution}{}{}
            The leading term of the truncation error is of $\order{h^3}$, that is
            \[
                \frac{h^3}{12}u^{(4)}(\bar{x}).
            \]
        \end{solution}
    \item What is the truncation error if $u(x) = 4x^4 + 12x^3 + 6x^2 + x/2 + \pi$.
        \begin{solution}{}{}
            \begin{align*}
                u'(\bar{x}) &= \frac{1}{h}\qty(\frac{u(\bar{x}-2h)}{6}-u(\bar{x}-h)+\frac{u(\bar{x})}{2}+\frac{u(\bar{x}+h)}{3}) \\
                &= 16\bar{x}^3 + 36\bar{x}^2 + 12\bar{x} + \frac{1}{2}
            \end{align*}
            We have $u'(x)=16x^3 + 36x^2 + 12x + 1/2$, that is, the truncation error of this given function is $0$.
        \end{solution}
\end{enumerate}

\section{Question 3}
\begin{enumerate}[label=(\alph*)]
    \item For the following $N\times N$ matrix, show that all eigenvalues are given by $\lambda_p = a+2b\cos\tfrac{\pi p}{N+1}$ for $p=1,2,\ldots,N$.
        \[
            \qty[\begin{NiceArray}{CCCC}
                a & b      &        &   \\
                b & \Ddots & \Ddots &   \\
                  & \Ddots & \Ddots & b \\
                  &        & b      & a \\
            \end{NiceArray}]_{N\times N}
        \]
        \begin{solution}{}{}
            Please refer to Section~\ref{Q:4-1}.
        \end{solution}
\end{enumerate}

\section{Question 4}\label{Q:4-1}
\begin{enumerate}[label=(\alph*)]
    \item For the following $N\times N$ matrix, show that all eigenvalues are given by $\lambda_p = a+2\sqrt{bc}\cos\tfrac{\pi p}{N+1}$ for $p=1,2,\ldots,N$ and $bc>1$.
        \[
            \mA = \qty[\begin{NiceArray}{CCCC}
                a & c      &        &   \\
                b & \Ddots & \Ddots &   \\
                  & \Ddots & \Ddots & c \\
                  &        & b      & a \\
            \end{NiceArray}]_{N\times N}
        \]
        \begin{solution}{}{}
            The eigenvalues are $\lambda_p = a+2\sqrt{bc}\cos\tfrac{\pi p}{N+1}$, and the corresponding eigenvector $\vx^p$ has $j$th component
            \[
                \vx^p_j = \qty(\sqrt{\frac{b}{c}})^j \sin\qty(\frac{p\pi j}{N+1}).
            \]
            And $\vx^p_0=\vx^p_{N+1}=0$, if we want to show that $\mA\vx^p=\lambda_p\vx_p$, then we just need to show that $b\vx^p_{j-1}+a\vx^p_j+c\vx^p_{j+1}=\lambda_px^p_j$ for $j=1,2,\ldots,N$. We use the method of subtracting two sides to determine whether the two sides are equal by judging whether the result is zero. That is,
            \begin{align*}
                & b\vx^p_{j-1}+a\vx^p_j+c\vx^p_{j+1}-\lambda_px^p_j \\
                =& b\vx^p_{j-1}-2\sqrt{bc}\cos\frac{\pi p}{N+1}\vx^p_j+c\vx^p_{j+1} \\
                =& \qty(\frac{b}{c})^{(j-1)/2}\qty(b\sin\qty(\frac{\pi p(j-1)}{N+1})+b\sin\qty(\frac{\pi p(j+1)}{N+1})-2b\sin\qty(\frac{\pi pj}{N+1})\cos\qty(\frac{\pi p}{N+1})) \\
                =& \qty(\frac{b}{c})^{(j-1)/2}b\qty(\sin\qty(\frac{\pi p(j-1)}{N+1})+\sin\qty(\frac{\pi p(j+1)}{N+1})-2\sin\qty(\frac{\pi pj}{N+1})\cos\qty(\frac{\pi p}{N+1})) \\
                =& 0.
            \end{align*}
            The last step relies on $\sin(x+y)=\sin(x)\cos(y)+\cos(x)\sin(y)$. Since $N\times N$ matrix has at most $N$ eigenvalues, then $\lambda_p = a+2\sqrt{bc}\cos\tfrac{\pi p}{N+1}$ is all the eigenvalues the matrix has.
        \end{solution}
\end{enumerate}

\section{Question 5}
For the 2-point BVP with Neumann boundary conditions:
\[
    \begin{cases}
        u''(x) = f(x), & x\in(0, 1) \\
        u'(0) = u'(1) = 0 & \\
    \end{cases}
\]
\begin{enumerate}[label=(\alph*)]
    \item Set the grid points as $x_j=jh$ for $j=0,1,\ldots,N$ and $h=1/N$. Write down the central FD scheme for the main equation with using $u_j$ to approximate $u(x_j)$.
        \begin{solution}{}{}
            According to the central finite difference scheme, for $j=1,2,\ldots,N-1$ we have
            \[
                u''(x_j) = \frac{u(x_{j+1})-2u(x_j)+u(x_{j-1})}{h^2} = f(u_j)
            \]
            If we use $u'(\bar{x})=(u(\bar{x}+h)-u(\bar{x}))/h$ or $u'(\bar{x})=(u(\bar{x})-u(\bar{x})-h)/h$, we can approximate $u(x_0)$ by $u(x_1)$ and $u(x_N)$ by $u(x_{N-1})$. That is
            \[
                \frac{1}{h^2}
                \qty[\begin{NiceArray}{RRRRR}
                    -1 & 1      &        &        &    \\
                    1  & -2     & \Ddots &        &    \\
                       & \Ddots & \Ddots & \Ddots &    \\
                       &        & \Ddots & -2     & 1  \\
                       &        &        & 1      & -1 \\
                \end{NiceArray}]_{N-1\times N-1}
                \begin{bNiceMatrix}
                    u_1 \\ u_2 \\ \Vdots \\ \Vdots \\ u_{-1})
                \end{bNiceMatrix}
                =
                \begin{bNiceMatrix}
                    f(x_1) \\ f(x_2) \\ \Vdots \\ \Vdots \\ f(x_{N-1})
                \end{bNiceMatrix}.
            \]
        \end{solution}
    \item\label{tmp-1} Add two ``ghost'' points as $x_{-1}=-h$ and $x_{N+1}=1+h$, and two more variables $u_{-1}$ and $u_{N+1}$. Treat the boundary condition as $(u_1-u_{-1})/2h=0$, $(u_{N+1}-u_{N-1})/2h=0$. Assemble all the equations as $\mA\vb{U}=\vb{F}$ where $\vb{U}=[u_0, u_1, \ldots, u_N]^T$.
        \begin{solution}{}{}
            If we treat the boundary condition as the given way, we can approximate $u(x_{-1})$ by $u(x_1)$ and $u(x_{N+1})$ by $u(x_{N-1})$. Then all the equations can be assemble as $\mA\vb{U}=\vb{F}$ where $\vb{U}=[u_0,u_1,\ldots,u_N]^T$ and $\vb{F}=[f(x_0),f(x_1),\ldots,f(x_N)]^T$. Also, we have
            \[
                \mA = \frac{1}{h^2}
                \qty[\begin{NiceArray}{RRRRR}
                    -2 & 2      &        &        &    \\
                    1  & \Ddots & 1      &        &    \\
                       & \Ddots & \Ddots & \Ddots &    \\
                       &        & 1      & \Ddots & 1  \\
                       &        &        & 2      & -2 \\
                \end{NiceArray}]_{N+1\times N+1}
            \]
        \end{solution}
    \item Show that $\mA$ is singular, and find out when the system $\mA\vb{U}=\vb{F}$ has solutions.
        \begin{solution}{}{}
            We can first check the validity of the conclusion by MATLAB.
            \begin{matlabcode}{}
A = @(N) -2*eye(N+1) + diag([2, ones(1, N-1)], 1) + diag([ones(1, N-1), 2], -1);
for N = 1 : 99
    if det(A(N)) ~= 0
        disp(N);
    end
end
            \end{matlabcode}
            Then, we notice that the row sum of $\mA$ is 0, which means $\rank\mA\neq N$, this leads to the conclusion that $\mA$ is singular. As for when the system $\mA\vb{U}=\vb{F}$ has solutions, through the knowledge of linear algebra, we know that $\vb{F}$ must in the range space\footnote{In linear algebra, range space refers to the column space of a matrix, which is the set of all possible linear combinations of its column vectors.} of $\mA$.
        \end{solution}
    \item Find the kernel space of $\mA$.
        \begin{solution}{}{}
            The kernel space\footnote{The definition of kernel space is also appears in the computer field: System memory in Linux can be divided into two distinct regions: kernel space and user space. Kernel space is where the kernel (i.e., the core of the operating system) executes (i.e., runs) and provides its services.} is also called the null space, which is defined as
            \[
                \nullspace{\mA_{n\times n}} = \qty{\vx\in\RR^n \mid \mA\vx=\va{0}}.
            \]
            Therefor, the kernel space of $\mA$ is
            \[
                \spanof\qty{
                    \qty[\begin{NiceArray}{C}
                        1 \\ 1 \\ \Vdots \\ 1
                    \end{NiceArray}]_{N+1\times 1}
                }.
            \]
        \end{solution}
    \item Find out all eigenvalues and eigenvectors for the matrix $\mA$ given in \ref{tmp-1}.\todo[inline,color=green!40]{todo: re-derive it later.}
        \begin{solution}{}{}
            The non-zero eigenvalues of the matrix $\mA_{N+1\times N+1}$ is
            \[
                \lambda_p = -2 + 2\cos\frac{\pi p}{N},
            \]
            and the corresponding eigenvector $\vx^p$ has $j$th component
            \[
                \vx^p_j = \cos\qty(\frac{p\pi(j-1)}{N})
            \]
            for $p=0,1,\ldots,N$. We can then check the correctness of the result by MATLAB.
            \begin{matlabcode}{}
A = @(N) -2*eye(N+1) + diag([2, ones(1, N-1)], 1) + diag([ones(1, N-1), 2], -1);
lambda = @(N, p) -2 + 2*cos(pi*p/N);
x = @(N, p) cos(pi*p*(0:N)/N)';
error = @(N) N^2 * eps;
for N = 1 : 99
    for p = 0 : N
        delta = A(N)*x(N, p) - lambda(N, p)*x(N, p);
        if norm(delta, 1) > error(N)
            disp(['N=', num2str(N), '; p=', num2str(p), '; error=', num2str(norm(delta, 1))]);
        end
    end
end
            \end{matlabcode}
        \end{solution}
\end{enumerate}

\clearpage
\section{Appendix}\label{S:appendix-1}
\begin{appendices}
    \pythonfile{code/hw1.py}{\texttt{hw1.py}}
    \pythonfile{code/1_2.py}{\texttt{1\_2.py}}
    \pythonfile{code/1_2_sympy.py}{\texttt{1\_2\_sympy.py}}
\end{appendices}


\documentclass{article}
\usepackage{amsmath}
\usepackage[margin=1.0in]{geometry}
\usepackage{xcolor}

\begin{document}

\noindent
Does $\displaystyle \sum_{n=1}^\infty \frac{n!}{(2n)!}$
diverge, converge absolutely, or converge conditionally?

\subsection*{Solution}

\begin{align*}
L
&=\lim_{n \to \infty} \left|\frac{a_{n+1}}{a_n}\right|\\
&=\lim_{n \to \infty} \left|\frac{(n+1)!}{(2(n+1))!} \cdot \frac{(2n)!}{n!}\right|\\
&=\lim_{n \to \infty} \left|\frac{(n+1)!}{(2n+2)!} \cdot \frac{(2n)!}{n!}\right|\\
&=\lim_{n \to \infty} \left|\frac{(n+1) \cdot n!}{(2n+2) \cdot (2n+1) \cdot (2n)!} \cdot \frac{(2n)!}{n!}\right|\\
&=\lim_{n \to \infty} \left|\frac{(n+1)}{(2n+2)(2n+1)}\right|\\
&=\lim_{n \to \infty} \frac{n+1}{4n^2+6n+2}\\
&=\lim_{n \to \infty} \frac{1}{8n+6} \text{ by L'hopital}\\
&= 0
\end{align*}

Since $L< 1$, the series $\displaystyle \sum_{n=1}^\infty \frac{n!}{(2n)!}$ converges absolutely by the Ratio Test.

\end{document}%%%%%%%%%%%%%%%%%

\begin{align*}
L&=\lim_{n \to \infty} \sqrt[n]{|a_n|}\\
&= \lim_{n \to \infty} \sqrt[n]{\left| \right|}\\
\end{align*}

\begin{align*}
L&=\lim_{n \to \infty} \left|\frac{a_{n+1}}{a_n}\right|\\
&= \lim_{n \to \infty} \left| \right|\\
\end{align*}

\begin{align*}
\lim_{n \to \infty} a_n
&= \lim_{n \to \infty} \\
\end{align*}

Since $\sum |a_n| = \sum a_n$, the series $\displaystyle \sum_{n=1}^\infty AAAAAAAAAAAAAA$ converges absolutely.

Since $|r| < 1$, the series ...  converges by the Geometric Series Test.

Since $|r| \geq 1$, the series ...  diverges by the Geometric Series Test.

The function $f(x)=\frac{}{}$ is continuous, positive, and decreasing on $[1,\infty)$.

\subsection*{Solution}


\documentclass{article}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{mathtools}
\usepackage{tikz}
\usepackage[a4paper, total={8in, 11in}]{geometry}

\newtheorem{theorem}{Theorem}
\newtheorem{lemma}{Lemma}
\newtheorem{definition}{Definition}
\DeclareMathOperator*{\argmin}{\arg\!\min}
\DeclareMathOperator*{\argmax}{\arg\!\max}
\DeclareMathOperator{\E}{\mathbb{E}}
\DeclareMathOperator{\p}{\mathbb{P}}
\DeclareMathOperator{\pr}{\mathbb{P}}
\newlength{\dhatheight}
\newcommand{\doublehat}[1]{%
    \settoheight{\dhatheight}{\ensuremath{\hat{#1}}}%
    \addtolength{\dhatheight}{-0.15ex}%
    \hat{\vphantom{\rule{1pt}{\dhatheight}}%
    \smash{\hat{#1}}}}

\begin{document}

After my talk on Thursday, some people asked how to improve Hoeffding's inequality, and how it could be made to give confidence-intervals that do not exceed the data's bounds.
Hoeffding's inequality is easy to use and can be essential in some formal proofs. But my research has led me to know that there are better bounds. And if my learning is usefull for someone else, then I regard that as a best kind of outcome!\\

The easiest way to show how Hoeffding's inequality can be improved is to show how a better bound can be created by omitting an approximation in its derivation - ie. creating a universally more powerfull bound.\\
Hoeffding's inequality is an example of a Chernoff bound, which uses Markov's inequality:\\

\begin{lemma}[Markov's Inequality] for any non-negative random variable $X$ and any $a>0$ that: $\p(X\ge a)\le \E[X]/a$
\end{lemma}

\begin{lemma}[Chernoff Bound]\label{chernoff1}
If $\hat{\mu} = \frac{1}{n}\sum_{i=1}^nx_i$ is sample mean of $n$ independent and identically distributed samples of random variable $X$ ($x_i\sim X$), then for any $s,t>0$: $\p(\hat{\mu}\ge t)\le\E\left[\exp(sX)\right]^n\exp(-snt)$
\end{lemma}
\begin{proof}
$\p(\hat{\mu}\ge t) =  \p\left(\exp\left(s\sum_{i=1}^nx_i\right)\ge \exp(snt)\right)$ hence by Markov's inequality $\p(\hat{\mu}\ge t)\le \E\left[\exp\left(s\sum_{i=1}^nx_i\right)\right]\exp(-snt)$\\
The result follows as we assume that our samples are independant (for any independant variables $A,B$ that $\E[AB]=\E[A]\E[B]$).
\end{proof}
%Many phenominally powerfull probability bounds follow by finding upper bounds for $\E\left[\exp(sX)\right]$ - which is also called the moment generating function; including Hoeffing's bound:

\begin{theorem}[Hoeffding's inequality for mean zero]\label{hoeffdings_inequality}
Let $X$ be a random variable that is bounded $a\le X\le b$, with a mean $\mu=0$.  Then letting $D=b-a$, then for any $t>0$, the mean $\hat{\mu}$ of $n$ independent samples of $X$ is bounded:
$\p(\hat{\mu}\ge t)\le \exp\left(\frac{-2nt^2}{D^2}\right)$
\end{theorem}
\begin{proof}
To prove Hoeffding's inequality we develop an upper bound for $\E[\exp(sX)]$, if we assume variable $X$ has a probability density function $f(x)$, then we can fit a line over $\exp(sx)$ as:
$\label{Hoeffdings_line_fitting}\E[\exp(sX)] = \int_a^bf(x)\exp(sx)dx \le \int_a^bf(x)(\frac{x-a}{b-a}e^{sb} + \frac{b-x}{b-a}e^{sa})dx$\\
Using the fact that the mean $\mu = \int_a^bf(x)xdx = 0$ thus:
$\E[\exp(sX)] \le \frac{1}{sb-sa}\left(sb\exp(sa) - sa\exp(sb) \right)$\\
Given the fact that for any $\kappa,\gamma$:
$\frac{1}{\kappa-\gamma}(\kappa\exp(\gamma)-\gamma\exp(\kappa))\le \exp\left(\frac{1}{8}(\kappa-\gamma)^2\right)~~~~~\refstepcounter{equation}(\theequation)\label{Hoeffdings_lemma} $\\
Thus $\label{hoeffdings_lemma_eq}\E[\exp(sX)] \le \exp\left(\frac{1}{8}s^2(b-a)^2 \right)$
and by our Chernoff bound (lemma \ref{chernoff1}) we get:
$ \p(\hat{\mu}\ge t) \le \exp\left(\frac{1}{8}s^2(b-a)^2 n-snt\right) $
And minimising with respect to $s$ gives the result.
\end{proof}
\-\hspace{1cm}\\
At a first glance, the most limiting feature of this derivation is the requirement that the mean is zero, however this is ultimately immaterial and simplifies the derivation. We just consider our data as if it were shifted to have a mean of zero, leaving $D$ unchanged. hence we get the equation we know and love:\\

\begin{theorem}[Hoeffding's inequality]\label{Hoeffdings_inequality_proper}
Let $X$ be a real-valued random variable that is bounded $a\le X\le b$.  Then for $D=b-a$ and any $t>0$, the mean $\hat{\mu}$ of $n$ independent samples of $X$ is probability bounded by:\\
\begin{equation}\p(\hat{\mu}-\mu\ge t)\le \exp\left(-2nt^2/D^2\right)
\quad\text{Or by rearranging:}\quad
\p\left(\hat{\mu}-\mu\ge \sqrt{D^2\log(1/t)/(2n)}\right)\le t\end{equation}
\end{theorem}
\-\hspace{1cm}\\
However we can easily do better.\\

\begin{theorem}\label{hoeffdings_inequality22}
Let $X$ be a real-valued random variable that is bounded $a\le X\le b$, with a mean $\mu$ of zero.  Then for $t>0$, the mean $\hat{\mu}$ of $n$ independent samples of $X$ is probability bounded by:
\begin{equation}\p(\hat{\mu}\ge t)\le \left( \frac{b}{b-a}\left(\frac{b(a-t)}{a(b-t)}\right)^{\frac{a-t}{b-a}} -\frac{a}{b-a}\left(\frac{b(a-t)}{a(b-t)}\right)^{\frac{b-t}{b-a}}  \right)^n
\end{equation}
\end{theorem}
\begin{proof}
Exactly the same proof as Theorem \ref{hoeffdings_inequality} except do not use Equation \ref{Hoeffdings_lemma}, leading to:\\
$ \p(\hat{\mu}\ge t) \le (b\exp(sa) - a\exp(sb))^n\exp(-snt)(b-a)^{-n} $ and minimising with respect to $s$ gives the result.
%$$ s = \frac{1}{b-a}\log\left(\frac{b(a - t)}{a(b - t)}\right) $$
\end{proof}
\-\hspace{1cm}\\
Now, this concentration inequality is more powerful but more difficult to manipulate, we also dont usually know $a$ or $b$ (since that would correspond to knowing the mean itself). But we usually know that our data is bounded, for instance if we are dealing with binary data then we know that, our variable $X$ (not shifted to have a zero mean) is bounded $0<X<1$, and we directly get:\\

\begin{theorem}[Also called Hoeffding's inequality]\label{hoeffdings_inequality23}
Let $X$ be a real-valued random variable that is bounded $0\le X\le 1$, with mean $\mu$. Then for $t>0$, the mean $\hat{\mu}$ of $n$ independent samples of $X$ is probability bounded by:
\begin{equation}\p(\hat{\mu}-\mu\ge t)\le \left[\left(\frac{1-\mu}{1-t-\mu}\right)^{1-t-\mu}  \left(\frac{\mu}{t+\mu}\right)^{t+\mu}\right]^n
\end{equation}
\end{theorem}
\begin{proof}Follows directly from Theorem \ref{hoeffdings_inequality22} with the substitution $a=-\mu$ and $b=1-\mu$.
\end{proof}

This eqution is also credited to Hoeffding and is found widely in literature.
So, if you have binary data and your sample mean $\hat{\mu}$ and you want know how likely it is at underestimating the data mean $\mu$ by $t$ then this equation gives this (the equivalent inequality for overestimation is similar)

\pagebreak

So, how much improvement is given by Theorem \ref{hoeffdings_inequality23} over our Hoeffding's inequality Theorem \ref{Hoeffdings_inequality_proper} over what might be achieved perfectly?\\ Consider the following figure:
\-\hspace{1cm}\\

\begin{figure}[h]
\centering

\begin{tikzpicture}[xscale=18.7, yscale=3]
\draw[->] (-0.1,0) -- (0.3,0) node[anchor=north] {$t$};
\draw[->] (0,0) -- (0,1.1) node[anchor=east] {};
\draw[smooth, domain=0:0.3, color=black, line width=0.20mm, samples=100]
    plot (\x,{e^(-2*9*(\x*\x))}) node [above] {};
\draw[smooth, domain=0:0.15, color=red, line width=0.20mm, samples=100]
    plot (\x,{((((1-0.85)/(1-\x-0.85))^(1-\x-0.85)) * (((0.85)/(\x+0.85))^(\x+0.85)))^9}) node [right] {};

%\draw[smooth, domain=0:0.85, color=black, line width=0.20mm]
%    plot (0,0)(1,1) node [above] {};

\draw [color=blue] plot coordinates {((0, 1.0) (0.01, 0.5858470553103369) (0.02, 0.541116769443986) (0.03, 0.4940266224163914) (0.04, 0.4448836480573838) (0.05, 0.39410018156932936) (0.060000000000000005, 0.34221013038093756) (0.07, 0.2898869021427757) (0.08, 0.23796310401811177) (0.09, 0.1874521323190363) (0.09999999999999999, 0.13957177653389025) (0.10999999999999999, 0.09576996688795385) (0.11999999999999998, 0.057752799775041316) (0.12999999999999998, 0.02751498069314216) (0.13999999999999999, 0.007372829712548668) (0.15, 0.0)
};

\draw (0.1,-0.1) -- (0.1,0.1);
\draw	(0.1,-0.25) node{{\scriptsize $0.1$}};

\draw (0.05,-0.1) -- (0.05,0.1);
\draw	(0.05,-0.25) node{{\scriptsize $0.05$}};

\draw (0.15,-0.1) -- (0.15,1);
\draw	(0.15,-0.25) node{{\scriptsize $0.15$}};

\draw	(0.0,-0.25) node{{\scriptsize $0$}};

\draw	(-0.01,1) node{{\scriptsize $1$}};

\end{tikzpicture}

\caption{for binary data, if your sample mean is $\hat{\mu}=0.85$, the probability envelopes of Hoeffdings inequality (black) and from Theorem \ref{hoeffdings_inequality23} (red) above a hypothetical minimum (blue) for $n=9$ }
\label{fig:graph111}
\end{figure}

And from this figure, we can see that the bound derived from Theorem \ref{hoeffdings_inequality23} as the red line is below the black line corresponding to Hoeffding's inequality (Theorem \ref{Hoeffdings_inequality_proper}).
The ideal blue bound (or atleast what I currently suspect is ideal (!) - contact me if you might like to do collaboration or something) is as follows:

$$\p(\hat{\mu}-\mu>t)\le\max\left(\left(1+\frac{t}{\hat{\mu}-1}\right)^n,\sum_{m=0}^{\lfloor \hat{\mu}n\rfloor}\binom{n}{m}
\left(\frac{(\hat{\mu}+t-1)(n-\lfloor \hat{\mu}n \rfloor)}{n(1-\hat{\mu})}\right)^{n-m}
\left(1-\frac{(\hat{\mu}+t-1)(n-\lfloor \hat{\mu}n \rfloor)}{n(1-\hat{\mu})}\right)^{m}
\right)$$

Which I anticipate is the result of Optimal Uncertainty Quantification procedure \cite{doi:10.1137/10080782X} - which basically is the process of searching directly for the worst case probability distribution for your random variables constrained by what you know.\\

In anycase, the literature on concentration inequalities is simply vast, and there are lots of improvements to be made, and an array of techniques which can be applied.
Sticking to the original Hoeffding's inequality is safe for publication purposes - as people understand and know it.
but there is no reason to stay bound to what we are familiar with.

\section{an additional note}
So, in deriving (most, but importantly not all) concentration inequalities (such as Hoeffding's) we start with the assumption of distribution parameters and then deduce the likely error of sampling statistics from that. However the way in which we often use these inequalities is in reverse, starting from knowledge of sampling statistics and then making inferences about the distribution parameters.
And if you are a baysian like I am, you should be very worried... as this is technically invalid use.
However, I understand that ultimately the same problem is part of statistical hypothesis testing generally.

\bibliographystyle{plain}
%\bibliographystyle{authordate1}
\bibliography{bib}

\end{document}


\subsection{The infinite cyclic group (\(Z\))}

\subsubsection{The additive group of integers}

\subsubsection{Generating cyclic groups}

We can generate a group with a single element, it is a cyclic group.

For example, we can define a group \(G=<1>\) which gives us the additive group of integers.

\subsubsection{Infinite cyclic groups are isomorphic to the additive group of integers}

More generally, any infinite cyclic group is isomorphic to the additive group of integers.

Consider the multiplicative group of \(< i >\).

This contains \(\{1,-1,i,-i \} \).

This is also automorphic to the natural number and modulo addition group above.

We can define finite cyclic groups of size \(n\) using the generating element \(z^{\dfrac{1}{n}}\). This is isomorphic to the general cyclic group \(C_n\), and to \(Z/nZ\).

\subsubsection{Abelian cyclic groups}

Cyclic groups are abelian.


\documentclass[oneside, 11pt]{article}

\usepackage{amsfonts}
\usepackage{amsmath}
\usepackage{bm}
\usepackage{cleveref}
\usepackage{txfonts}

\DeclareMathOperator{\cov}{cov}

\allowdisplaybreaks

\begin{document}

Consider regression with Gaussian Processes. It relies on a matrix $K$ that represents the assumed covariance between samples. If we have two samples $\mathbf{x}$ and $\mathbf{x}'$, the covariance $\cov\left[f(\mathbf{x}),f(\mathbf{x}')\right] = k(\mathbf{x}, \mathbf{x}')$ for some $k$ that we assume. Usually \[
    \lim_{\left\|\mathbf{x} - \mathbf{x}'\right\| \to 0}k(\mathbf{x}, \mathbf{x}') = 1
\] and \[
    \lim_{\left\|\mathbf{x} - \mathbf{x}'\right\| \to \infty}k(\mathbf{x}, \mathbf{x}') = 0 \text{.}
\]

This works well when we know $\mathbf{x}$ and $\mathbf{x}'$ exactly, but fails if we only know them up to an uncertainty. Suppose all we know if that $\mathbf{x}$ is a sample from $\mathcal{N}(\bm{\mu}, \Sigma)$ and $\mathbf{x}'$ is a sample from $\mathcal{N}(\bm{\mu}', \Sigma')$ for some small $\|\bm{\mu} - \bm{\mu}'\|$ and very large spreads $\Sigma$ and $\Sigma'$. By our na\"ive definition,\[
    \cov\left[f(\mathbf{x}),f(\mathbf{x}')\right] = k(\bm{\mu}, \bm{\mu}') \approx 1 \text{.}
\] However, for $\Sigma$ and $\Sigma'$ large enough, we expect\[
    \cov\left[f(\mathbf{x}),f(\mathbf{x}')\right] \approx 0 \text{.}
\]

It is thus desirable to extend the definition of kernels $k$ such that they are not just a function $k(\mathbf{x}, \mathbf{x}')$ of known values, but a function $k(\mathcal{P}_\mathbf{x}, \mathcal{P}_{\mathbf{x}'})$ of their probability distributions. This permits us to account for uncertainties of input data in scientific contexts. We derive this extended definition.

For known values $\mathbf{x}$ and $\mathbf{x}'$, the definition of $\cov[f(\mathbf{x}),f(\mathbf{x}')]$ is \begin{align*}
    \cov\left[f(\mathbf{x}),f(\mathbf{x}')\right] &= \mathbb{E}\left[\left(f\left(\mathbf{x}\right) - \mathbb{E}\left[f\left(\mathbf{x}\right)\right]\right)\left(f\left(\mathbf{x}'\right) - \mathbb{E}\left[f\left(\mathbf{x}'\right)\right]\right)\right] \\
    &= \mathbb{E}\left[f\left(\mathbf{x}\right)f\left(\mathbf{x}'\right)\right] \\
    &= \int_{-\infty}^{\infty} \int_{-\infty}^{\infty} y y' p\left(y = f\left(\mathbf{x}\right), y' = f\left(\mathbf{x}'\right)\right) dy'dy \\
    &= k(\mathbf{x}, \mathbf{x}')
\end{align*} since $\mathbb{E}[f(\mathbf{x})] = \mathbb{E}[f(\mathbf{x}')] = 0$ by the Gaussian process prior.

For values of $\mathbf{x}$ and $\mathbf{x}'$ sampled from $\mathcal{P}_\mathbf{x}$ and $\mathcal{P}_{\mathbf{x}'}$ respectively,  \begin{align*}
    \cov\left[f(\mathbf{x}),f(\mathbf{x}')\right] &= \mathbb{E}\left[\left(f\left(\mathbf{x}\right) - \mathbb{E}\left[f\left(\mathbf{x}\right)\right]\right)\left(f\left(\mathbf{x}'\right) - \mathbb{E}\left[f\left(\mathbf{x}'\right)\right]\right)\right] \\
    &= \mathbb{E}\left[f\left(\mathbf{x}\right)f\left(\mathbf{x}'\right)\right] \\
    &= \int_{\mathcal{X}}\int_{\mathcal{X}}\int_{-\infty}^{\infty} \int_{-\infty}^{\infty} y y' p\left(y = f\left(\mathbf{x}\right), y' = f\left(\mathbf{x}'\right)\right) \mathcal{P}_{\mathbf{x}}(\mathbf{x}) \mathcal{P}_{\mathbf{x}'}(\mathbf{x}') dy'dyd\mathbf{x}d\mathbf{x}' \\
    &= \int_{\mathcal{X}}\int_{\mathcal{X}}\mathcal{P}_{\mathbf{x}}(\mathbf{x}) \mathcal{P}_{\mathbf{x}'}(\mathbf{x}')\int_{-\infty}^{\infty} \int_{-\infty}^{\infty} y y' p\left(y = f\left(\mathbf{x}\right), y' = f\left(\mathbf{x}'\right)\right)  dy'dyd\mathbf{x}d\mathbf{x}' \\
    &= \int_{\mathcal{X}}\int_{\mathcal{X}}\mathcal{P}_{\mathbf{x}}(\mathbf{x}) \mathcal{P}_{\mathbf{x}'}(\mathbf{x}')k(\mathbf{x}, \mathbf{x}')d\mathbf{x}d\mathbf{x}' \text{,}
\end{align*} where $k(\mathbf{x}, \mathbf{x}')$ is the kernel defined on $\mathbf{x}$ and $\mathbf{x}'$ with no uncertainty.

For $\mathcal{P}_\mathbf{x} = \mathcal{N}(\bm{\mu}, \Sigma)$ and $\mathcal{P}_{\mathbf{x}'} = \mathcal{N}(\bm{\mu}', \Sigma')$, and for the Gaussian kernel $k(\mathbf{x}, \mathbf{x}') = \exp(-\frac12(\mathbf{x} - \mathbf{x}')^\top S^{-1} (\mathbf{x} - \mathbf{x}')$, our kernel with uncertainty \begin{align*}
    k(\mathcal{P}_\mathbf{x}, \mathcal{P}_{\mathbf{x}'}) \\
    &= \int_{\mathcal{X}}\int_{\mathcal{X}}\mathcal{P}_{\mathbf{x}}(\mathbf{x}) \mathcal{P}_{\mathbf{x}'}(\mathbf{x}')k(\mathbf{x}, \mathbf{x}')d\mathbf{x}d\mathbf{x}' \\
    &= \sqrt{\frac{\det S}{\det\left(\Sigma + \Sigma' + S\right)}} \exp\left(-\frac{1}{2}\left(\bm{\mu}' - \bm{\mu}\right)^\top \left( \Sigma + \Sigma' + S\right)^{-1}\left(\bm{\mu}' - \bm{\mu}\right)\right) \text{.}
\end{align*} See \cref{sec:integral} for the derivation.

\appendix
\section{Finding the integral}
\label{sec:integral}

    We want to find the integral\begin{align*}
        &\int_{\mathbb{R}^d} \det(2\pi\Sigma)^{-1/2} \exp\left(-\frac12 \left(\mathbf{x}-\bm{\mu}\right)^\top \Sigma^{-1} \left(\mathbf{x}-\bm{\mu}\right) \right) \\
        &\hspace{30pt} \int_{\mathbb{R}^d} \det(2\pi\Sigma')^{-1/2} \exp\left(-\frac12 \left(\mathbf{x}'-\bm{\mu}'\right)^\top \Sigma'^{-1} \left(\mathbf{x'}-\bm{\mu}'\right) \right) \\
        &\hspace{60pt}\exp\left(-\frac12 \left(\mathbf{x}'-\mathbf{x}\right)^\top S^{-1} \left(\mathbf{x}'-\mathbf{x}\right) \right) d\mathbf{x}' d\mathbf{x} \text{.}
    \end{align*}

    We first tackle the inner integral, \begin{align*}
        &\int_{\mathbb{R}^d} \det(2\pi\Sigma')^{-1/2} \exp\left(-\frac12 \left(\mathbf{x}'-\bm{\mu}'\right)^\top \Sigma'^{-1} \left(\mathbf{x'}-\bm{\mu}'\right) \right) \\
        &\hspace{60pt}\exp\left(-\frac12 \left(\mathbf{x}'-\mathbf{x}\right)^\top S^{-1} \left(\mathbf{x}'-\mathbf{x}\right) \right) d\mathbf{x}' \text{,}
    \end{align*} where we treat $\mathbf{x}$ as a constant.

    Changing variables, \[
        \mathbf{y}' \coloneqq S^{-1/2} (\mathbf{x}' - \mathbf{x}) \text{,}
    \] so\[
        \mathbf{x}' = S^{1/2} \mathbf{y}' + \mathbf{x} \text{.}
    \] We note that \[
        d\mathbf{y}' = S^{-1/2} d\mathbf{x}' \text{.}
    \]

    Then \begin{align*}
        &\int_{\mathbb{R}^d} \det(2\pi\Sigma')^{-1/2} \exp\left(-\frac12 \left(\mathbf{x}'-\bm{\mu}'\right)^\top \Sigma'^{-1} \left(\mathbf{x'}-\bm{\mu}'\right) \right) \\
        &\hspace{60pt}\exp\left(-\frac12 \left(\mathbf{x}'-\mathbf{x}\right)^\top S^{-1} \left(\mathbf{x}'-\mathbf{x}\right) \right) d\mathbf{x}' \\
        &= \det(2\pi\Sigma')^{-1/2} \left|\det S^{1/2}\right| \int_{\mathbb{R}^d} \exp\left(-\frac12 \left(S^{1/2} \mathbf{y}' + \mathbf{x} - \bm{\mu}'\right)^\top \Sigma'^{-1} \left(S^{1/2} \mathbf{y}' + \mathbf{x} - \bm{\mu}'\right) \right) \\
        &\hspace{150pt}\exp\left(-\frac12 \mathbf{y}'^\top \mathbf{y}' \right)  d\mathbf{y}' \text{.}
    \end{align*}

    Setting \[
        C' \coloneqq S^{\frac12}\Sigma'^{-1}S^{\frac12}
    \] and \[
        \mathbf{m'} \coloneqq -S^{-\frac12} (\mathbf{x} - \bm{\mu}') \text{,}
    \] this changes to \begin{align*}
        &\int_{\mathbb{R}^d} \exp\left(-\frac12 \left(S^{1/2} \mathbf{y}' + \mathbf{x} - \bm{\mu}'\right)^\top \Sigma'^{-1} \left(S^{1/2} \mathbf{y}' + \mathbf{x} - \bm{\mu}'\right) \right) \exp\left(-\frac12 \mathbf{y}'^\top \mathbf{y}' \right)  d\mathbf{y}' \\
        &= \int_{\mathbb{R}^d} \exp\left(-\frac12 \left(\mathbf{y}' - \mathbf{m}'\right)^\top C' \left(\mathbf{y}' - \mathbf{m}'\right) \right) \exp\left(-\frac12 \mathbf{y}'^\top \mathbf{y}' \right)  d\mathbf{y}' \text{.}
    \end{align*}

    In the 1-dimensional case we get \begin{align*}
        &\int_{-\infty}^{\infty} \exp\left(-\frac{c'}2 \left(y' - m'\right)^2 \right) \exp\left(-\frac12 y'^2 \right)  dy \\
        &= \frac{\sqrt{2\pi}}{\sqrt{1 + c'}} \exp\left(-\frac{1}{2}\frac{c'}{1+c'}m'^2 \right) \text{.}
    \end{align*}

    In the 2-dimensional case, \begin{align*}
        &\int_{\mathbb{R}^2} \exp\left(-\frac12 \left(\mathbf{y}' - \mathbf{m}'\right)^\top C' \left(\mathbf{y}' - \mathbf{m}'\right) \right) \exp\left(-\frac12 \mathbf{y}'^\top \mathbf{y}' \right)  d\mathbf{y}' \\
        &= 2 \pi \sqrt{\frac{1}{\det(I + C')}} \exp\left(-\frac{1}{2}\mathbf{m}'^\top \left(C'^{-1} + I\right)^{-1}\mathbf{m}'\right) \text{.}
    \end{align*}

    In the 3-dimensional case Mathematica shits itself and outputs hundreds of lines of symbols. For a simplified scenario where $C'$ is diagonal, we get\begin{align*}
        &\int_{\mathbb{R}^3} \exp\left(-\frac12 \left(\mathbf{y}' - \mathbf{m}'\right)^\top C' \left(\mathbf{y}' - \mathbf{m}'\right) \right) \exp\left(-\frac12 \mathbf{y}'^\top \mathbf{y}' \right)  d\mathbf{y}' \\
        &= (2 \pi)^{3/2} \sqrt{\frac{1}{\det(I + C')}} \exp\left(-\frac{1}{2}\mathbf{m}'^\top \left(C'^{-1} + I\right)^{-1}\mathbf{m}'\right) \text{.}
    \end{align*}

    We recognise the pattern, \begin{align*}
        &\int_{\mathbb{R}^d} \exp\left(-\frac12 \left(\mathbf{y}' - \mathbf{m}'\right)^\top C' \left(\mathbf{y}' - \mathbf{m}'\right) \right) \exp\left(-\frac12 \mathbf{y}'^\top \mathbf{y}' \right)  d\mathbf{y}' \\
        &= (2 \pi)^{d/2} \sqrt{\frac{1}{\det(I + C')}} \exp\left(-\frac{1}{2}\mathbf{m}'^\top \left(C'^{-1} + I\right)^{-1}\mathbf{m}'\right) \text{.}
    \end{align*}

    Substituting back,\begin{align*}
        &(2 \pi)^{d/2} \sqrt{\frac{1}{\det(I + C')}} \exp\left(-\frac{1}{2}\mathbf{m}'^\top \left(C'^{-1} + I\right)^{-1}\mathbf{m}'\right) \\
        &= (2 \pi)^{d/2} \sqrt{\frac{1}{\det\left(I + S^{\frac12}\Sigma'^{-1}S^{\frac12}\right)}} \exp\left(-\frac{1}{2}\mathbf{m}'^\top \left(S^{-\frac12}\Sigma'S^{-\frac12} + I\right)^{-1}\mathbf{m}'\right) \\
        &= (2 \pi)^{d/2} \sqrt{\frac{1}{\det\left(I + S^{\frac12}\Sigma'^{-1}S^{\frac12}\right)}} \exp\left(-\frac{1}{2}\left(-S^{-\frac12} \left(\mathbf{x} - \bm{\mu}'\right)\right)^\top \left(S^{-\frac12}\Sigma'S^{-\frac12} + I\right)^{-1}\left(-S^{-\frac12} \left(\mathbf{x} - \bm{\mu}'\right)\right)\right) \\
        &= (2 \pi)^{d/2} \sqrt{\frac{1}{\det\left(I + S^{\frac12}\Sigma'^{-1}S^{\frac12}\right)}} \exp\left(-\frac{1}{2}\left(\mathbf{x} - \bm{\mu}'\right)^\top \left(\Sigma' + S\right)^{-1}\left(\mathbf{x} - \bm{\mu}'\right)\right) \text{.}
    \end{align*}

    Hence, \begin{align*}
        &\int_{\mathbb{R}^d} \det(2\pi\Sigma')^{-1/2} \exp\left(-\frac12 \left(\mathbf{x}'-\bm{\mu}'\right)^\top \Sigma'^{-1} \left(\mathbf{x'}-\bm{\mu}'\right) \right) \\
        &\hspace{60pt}\exp\left(-\frac12 \left(\mathbf{x}'-\mathbf{x}\right)^\top S^{-1} \left(\mathbf{x}'-\mathbf{x}\right) \right) d\mathbf{x}' \\
        &= \det(2\pi\Sigma')^{-1/2} \left|\det S^{1/2}\right| \int_{\mathbb{R}^d} \exp\left(-\frac12 \left(S^{1/2} \mathbf{y}' + \mathbf{x} - \bm{\mu}'\right)^\top \Sigma'^{-1} \left(S^{1/2} \mathbf{y}' + \mathbf{x} - \bm{\mu}'\right) \right) \\
        &\hspace{150pt}\exp\left(-\frac12 \mathbf{y}'^\top \mathbf{y}' \right)  d\mathbf{y}' \\
        &= \det(2\pi\Sigma')^{-1/2} \left|\det S^{1/2}\right| (2 \pi)^{d/2} \sqrt{\frac{1}{\det\left(I + S^{\frac12}\Sigma'^{-1}S^{\frac12}\right)}} \exp\left(-\frac{1}{2}\left(\mathbf{x} - \bm{\mu}'\right)^\top \left(\Sigma' + S\right)^{-1}\left(\mathbf{x} - \bm{\mu}'\right)\right) \\
        &=  \sqrt{\frac{\det S}{\det\Sigma'\det\left(I + S^{\frac12}\Sigma'^{-1}S^{\frac12}\right)}} \exp\left(-\frac{1}{2}\left(\mathbf{x} - \bm{\mu}'\right)^\top \left(\Sigma' + S\right)^{-1}\left(\mathbf{x} - \bm{\mu}'\right)\right) \\
        &=  \sqrt{\frac{1}{\det\left(\Sigma'S^{-1} + I\right)}} \exp\left(-\frac{1}{2}\left(\mathbf{x} - \bm{\mu}'\right)^\top \left(\Sigma' + S\right)^{-1}\left(\mathbf{x} - \bm{\mu}'\right)\right)
    \end{align*}

    Now tackling the outer integral, \begin{align*}
        &\int_{\mathbb{R}^d} \det(2\pi\Sigma)^{-1/2} \exp\left(-\frac12 \left(\mathbf{x}-\bm{\mu}\right)^\top \Sigma^{-1} \left(\mathbf{x}-\bm{\mu}\right) \right) \\
        &\hspace{30pt} \int_{\mathbb{R}^d} \det(2\pi\Sigma')^{-1/2} \exp\left(-\frac12 \left(\mathbf{x}'-\bm{\mu}'\right)^\top \Sigma'^{-1} \left(\mathbf{x'}-\bm{\mu}'\right) \right) \\
        &\hspace{60pt}\exp\left(-\frac12 \left(\mathbf{x}'-\mathbf{x}\right)^\top S^{-1} \left(\mathbf{x}'-\mathbf{x}\right) \right) d\mathbf{x}' d\mathbf{x} \\
        &= \int_{\mathbb{R}^d} \det(2\pi\Sigma)^{-1/2} \exp\left(-\frac12 \left(\mathbf{x}-\bm{\mu}\right)^\top \Sigma^{-1} \left(\mathbf{x}-\bm{\mu}\right) \right) \\
        &\hspace{30pt} \sqrt{\frac{1}{\det\left(\Sigma'S^{-1} + I\right)}} \exp\left(-\frac{1}{2}\left(\mathbf{x} - \bm{\mu}'\right)^\top \left(\Sigma' + S\right)^{-1}\left(\mathbf{x} - \bm{\mu}'\right)\right) d\mathbf{x} \\
        &= \det(2\pi\Sigma)^{-1/2} \sqrt{\frac{1}{\det\left(\Sigma'S^{-1} + I\right)}} \\
        &\hspace{30pt}\int_{\mathbb{R}^d} \exp\left(-\frac12 \left(\mathbf{x}-\bm{\mu}\right)^\top \Sigma^{-1} \left(\mathbf{x}-\bm{\mu}\right) \right) \exp\left(-\frac{1}{2}\left(\mathbf{x} - \bm{\mu}'\right)^\top \left(\Sigma' + S\right)^{-1}\left(\mathbf{x} - \bm{\mu}'\right)\right) d\mathbf{x} \text{.}
    \end{align*}

    As before, we change variables with \[
        \mathbf{y} \coloneqq \left(\Sigma' + S\right)^{-\frac12}\left(\mathbf{x} - \bm{\mu}'\right) \text{,}
    \] which implies \[
        \mathbf{x} = \left(\Sigma' + S\right)^{\frac12}\mathbf{y} + \bm{\mu}' \text{,}
    \] so \[
        d\mathbf{x} = \left(\Sigma' + S\right)^{\frac12}d\mathbf{y} \text{.}
    \] Then \begin{align*}
        &\int_{\mathbb{R}^d} \exp\left(-\frac12 \left(\mathbf{x}-\bm{\mu}\right)^\top \Sigma^{-1} \left(\mathbf{x}-\bm{\mu}\right) \right) \exp\left(-\frac{1}{2}\left(\mathbf{x} - \bm{\mu}'\right)^\top \left(\Sigma' + S\right)^{-1}\left(\mathbf{x} - \bm{\mu}'\right)\right) d\mathbf{x} \\
        &= \left|\det(\Sigma' + S)^{\frac12} \right|\int_{\mathbb{R}^d} \exp\left(-\frac12 \left(\left(\Sigma' + S\right)^{\frac12}\mathbf{y} + \bm{\mu}'-\bm{\mu}\right)^\top \Sigma^{-1} \left(\left(\Sigma' + S\right)^{\frac12}\mathbf{y} + \bm{\mu}'-\bm{\mu}\right) \right) \\
        &\hspace{120pt}\exp\left(-\frac{1}{2}\mathbf{y}^\top\mathbf{y}\right) d\mathbf{y} \text{.}
    \end{align*}

    Setting \[
        C \coloneqq \left(\Sigma' + S\right)^{\frac12} \Sigma^{-1} \left(\Sigma' + S\right)^{\frac12}
    \] and \[
        \mathbf{m} \coloneqq -\left(\Sigma' + S\right)^{-\frac12} (\bm{\mu}' - \bm{\mu}) \text{,}
    \] we get \begin{align*}
        &\int_{\mathbb{R}^d} \exp\left(-\frac12 \left(\left(\Sigma' + S\right)^{\frac12}\mathbf{y} + \bm{\mu}'-\bm{\mu}\right)^\top \Sigma^{-1} \left(\left(\Sigma' + S\right)^{\frac12}\mathbf{y} + \bm{\mu}'-\bm{\mu}\right) \right) \exp\left(-\frac{1}{2}\mathbf{y}^\top\mathbf{y}\right) d\mathbf{y} \\
        &= \int_{\mathbb{R}^d} \exp\left(-\frac12 \left(\mathbf{y} - \mathbf{m}\right)^\top C \left(\mathbf{y} - \mathbf{m}\right) \right) \exp\left(-\frac{1}{2}\mathbf{y}^\top\mathbf{y}\right) d\mathbf{y} \text{.}
    \end{align*}

    By our previous result, \begin{align*}
        &\int_{\mathbb{R}^d} \exp\left(-\frac12 \left(\mathbf{y} - \mathbf{m}\right)^\top C \left(\mathbf{y} - \mathbf{m}\right) \right) \exp\left(-\frac{1}{2}\mathbf{y}^\top\mathbf{y}\right) d\mathbf{y} \\
        &= (2 \pi)^{d/2} \sqrt{\frac{1}{\det(I + C)}} \exp\left(-\frac{1}{2}\mathbf{m}^\top \left(C^{-1} + I\right)^{-1}\mathbf{m}\right) \\
        &= (2 \pi)^{d/2} \sqrt{\frac{1}{\det\left(I + \left(\Sigma' + S\right)^{\frac12} \Sigma^{-1} \left(\Sigma' + S\right)^{\frac12}\right)}} \exp\left(-\frac{1}{2}\mathbf{m}^\top \left(\left(\Sigma' + S\right)^{-\frac12} \Sigma \left(\Sigma' + S\right)^{-\frac12} + I\right)^{-1}\mathbf{m}\right) \\
        &= (2 \pi)^{d/2} \sqrt{\frac{1}{\det\left(I + \left(\Sigma' + S\right)^{\frac12} \Sigma^{-1} \left(\Sigma' + S\right)^{\frac12}\right)}} \\
        &\hspace{60pt}\exp\left(-\frac{1}{2}\left(\left(\Sigma' + S\right)^{-\frac12} \left(\bm{\mu}' - \bm{\mu}\right)\right)^\top \left(\left(\Sigma' + S\right)^{-\frac12} \Sigma \left(\Sigma' + S\right)^{-\frac12} + I\right)^{-1}\left(\left(\Sigma' + S\right)^{-\frac12} \left(\bm{\mu}' - \bm{\mu}\right)\right)\right) \\
        &= (2 \pi)^{d/2} \sqrt{\frac{1}{\det\left(I + \left(\Sigma' + S\right)^{\frac12} \Sigma^{-1} \left(\Sigma' + S\right)^{\frac12}\right)}} \exp\left(-\frac{1}{2}\left(\bm{\mu}' - \bm{\mu}\right)^\top \left( \Sigma + \Sigma' + S\right)^{-1}\left(\bm{\mu}' - \bm{\mu}\right)\right) \text.
    \end{align*}

    Then \begin{align*}
        &\det(2\pi\Sigma)^{-1/2} \sqrt{\frac{1}{\det\left(\Sigma'S^{-1} + I\right)}} \\
        &\hspace{30pt}\int_{\mathbb{R}^d} \exp\left(-\frac12 \left(\mathbf{x}-\bm{\mu}\right)^\top \Sigma^{-1} \left(\mathbf{x}-\bm{\mu}\right) \right) \exp\left(-\frac{1}{2}\left(\mathbf{x} - \bm{\mu}'\right)^\top \left(\Sigma' + S\right)^{-1}\left(\mathbf{x} - \bm{\mu}'\right)\right) d\mathbf{x} \\
        &= \det(2\pi\Sigma)^{-1/2} \sqrt{\frac{1}{\det\left(\Sigma'S^{-1} + I\right)}} \left|\det(\Sigma' + S)^{\frac12} \right| (2 \pi)^{d/2} \sqrt{\frac{1}{\det\left(I + \left(\Sigma' + S\right)^{\frac12} \Sigma^{-1} \left(\Sigma' + S\right)^{\frac12}\right)}} \\
        &\hspace{30pt} \exp\left(-\frac{1}{2}\left(\bm{\mu}' - \bm{\mu}\right)^\top \left( \Sigma + \Sigma' + S\right)^{-1}\left(\bm{\mu}' - \bm{\mu}\right)\right) \\
        &= \sqrt{\frac{\det S}{\det\left(\Sigma + \Sigma' + S\right)}} \exp\left(-\frac{1}{2}\left(\bm{\mu}' - \bm{\mu}\right)^\top \left( \Sigma + \Sigma' + S\right)^{-1}\left(\bm{\mu}' - \bm{\mu}\right)\right) \text{.}
    \end{align*}

    Hence,\begin{align*}
        &\int_{\mathbb{R}^d} \det(2\pi\Sigma)^{-1/2} \exp\left(-\frac12 \left(\mathbf{x}-\bm{\mu}\right)^\top \Sigma^{-1} \left(\mathbf{x}-\bm{\mu}\right) \right) \\
        &\hspace{30pt} \int_{\mathbb{R}^d} \det(2\pi\Sigma')^{-1/2} \exp\left(-\frac12 \left(\mathbf{x}'-\bm{\mu}'\right)^\top \Sigma'^{-1} \left(\mathbf{x'}-\bm{\mu}'\right) \right) \\
        &\hspace{60pt}\exp\left(-\frac12 \left(\mathbf{x}'-\mathbf{x}\right)^\top S^{-1} \left(\mathbf{x}'-\mathbf{x}\right) \right) d\mathbf{x}' d\mathbf{x} \\
        &= \sqrt{\frac{\det S}{\det\left(\Sigma + \Sigma' + S\right)}} \exp\left(-\frac{1}{2}\left(\bm{\mu}' - \bm{\mu}\right)^\top \left( \Sigma + \Sigma' + S\right)^{-1}\left(\bm{\mu}' - \bm{\mu}\right)\right) \text{.}
    \end{align*}

    Functionally, this makes sense. For our uncertain kernel, if $\Sigma=\Sigma'=0$, then we fall back to the original case. However, if they are large, then that places an upper bound on the maximum covariance that we assume. It is also a function of the difference $\bm{\mu}' - \bm{\mu}$, instead of taking each in to account separately. It is symmetric in $\Sigma$ and $\Sigma'$ and $\bm{\mu}$ and $\bm{\mu}'$.

\end{document}


\textbf{Download the program \textsc{wave2D\_leap\_frog.m}. This code solves the wave equation,
\begin{align*}
u_{tt}=u_{xx}+u_{yy},~~~~(x,y)\in [-1,1]\times [-1,1],~~t>0,
\end{align*}
with homogeneous Dirichlet boundary conditions and initial conditions
\begin{align*}
u(0,x,y) = exp(-40((x−0.2)^2+y^2)),~~ u_t(0, x, y) = 0.
\end{align*}
Modify this code to solve the same problem but with homogeneous Neumann boundary instead of Dirichlet. Plot your solution at time $t= 10$. How accurate is your solution? (number of digits)
}
\newline

To impose homogeneous Neumann boundary conditions we can simply ignore some rows and colums on our matrix multiplications. Taking $u_{yy}$:
\begin{align*}
u_{yy} = DDu = Du_y,
\end{align*}
where $D$ is the Chebyshev differentiation matrix. We know that $u_y$ has zero components in the first and last rows. This allows us to ignore the first and last rows of $D$ when taking the first multiplication. In addition, we can also ignore the first and last column of the second multiplication. Hence, we can simply impose Neumann boundary conditions by making,
\begin{align*}
u_{yy} = D(:,2:N)D(2:N,:)u,
\end{align*}
following \textsc{Matlab} notation. Analogous discussion can be made about $u_{xx}$, obtaining
\begin{align*}
u_{yy} = uD(2:N,:)'D(:,2:N)',
\end{align*}
where the $D'$ represents the transpose of $D$. The solution at $t=10$ is shown in the next figure.

\begin{figure}[H]
\centering
\includegraphics[scale=1]{P4.png}\caption{Numerical solution to the wave equation with homogeneous Neumann boundary conditions at $t=10$.}
\end{figure}

\subsection*{Matlab code for this problem}
\begin{verbatim}
%% Homework 4, Problem 4 -  Francisco Castillo
clear all; close all; clc
labelfontsize = 14;
%% 2D wave equation Chebyshev+Leap-frog, ZERO Neumann BC'S
N = 64;
[D,x] = cheb(N);
% x = x(2:end-1);
% D2 = D^2;
% D2 = D2(2:end-1,2:end-1);

% 2D grid
[X,Y] = meshgrid(x);

% Initial condition
u0 = exp(-40*((X-0.2).^2+Y.^2));
u = u0;

h = 1-x(2);
dt = h/2;
t = 0;
tf = 10;
count=0;

while t<tf
    if t+dt>tf
        dt = tf-t;
    else
        dt = h/2;
    end
    uyy = D(:,2:N)*D(2:N,:)*u;
    uxx = u*D(2:N,:)'*D(:,2:N)';
    u2 = 2*u- u0 + dt^2*(uxx+uyy);
    u0 = u;
    u = u2;

    if count == 10
        surf(X,Y,u)
        zlim([-1 1])
        drawnow
        shg
        count = 0;
    end

    count = count+1;
    t = t+dt;

    if t == tf
        surf(X,Y,u)
        zlim([-1 1])
        drawnow
        shg
        xlabel('$x$','interpreter','latex','fontsize',labelfontsize)
        ylabel('$y$','interpreter','latex','fontsize',labelfontsize)
        zlabel('$u(x,y)$','interpreter','latex','fontsize',labelfontsize)
        set(get(gca,'ZLabel'),'Rotation',0)
        saveas(gcf,'Latex/FIGURES/P4','png')
    end
end
\end{verbatim}


%================================
\section{Metrizability}
%================================

%--------------------------------
\begin{definition}
	[metric spaces]
	\label{def: metric spaces}
	Let $X$ be any set. A \textit{metric} $\rho$ on $X$ is a function $\rho: X \times X \to \mathbb R$ satisfying the following conditions: for all $x,y,z \in X$
	\begin{enumerate}[(i)]
		\item $\rho(x,y) \ge 0$, and $\rho(x,y) = 0$ iff $x = y$;
		\item $\rho(x,y) = \rho(y,z)$;
		\item $\rho(x,z) + \rho(z,y) \ge \rho(x,y)$.
	\end{enumerate}
\end{definition}
%--------------------------------

%--------------------------------
\begin{definition}
	[balls]
	\label{def: balls}
	Let $(X, \rho)$ be a metric space, let $x \in X$, and let $\varepsilon \in \mathbb R_{> 0}$. The \textit{open $\varepsilon$-ball about $x$} or just \textit{$\varepsilon$-ball about $x$} is defined to be
	$$
	B(x, \varepsilon) = \left\{ y \in X : \rho(x,y) < \varepsilon \right\}.
	$$
	The \textit{closed $\varepsilon$-ball about $x$} is defined to be
	$$
	\overline B (x, \varepsilon) = \{ y \in X : \rho(x,y) \le \varepsilon \}.
	$$
\end{definition}
%--------------------------------

%--------------------------------
\begin{example}
	\label{eg: Euclidean metrics}
	Let $X$ be any set, and let metric $\rho_p$ on $X^n$ ($n \in \mathbb Z_{>0}$) defined by
	$$
	\rho_p (x,y) = \left( \sum_{i = 1}^n |x_i - y_i|^p \right)^\frac{1}{p},
	$$
	where $p \in \mathbb R_{\ge 1}$. $\rho_2$ is so called the \textit{standard Euclidean metric}. If $X = \mathbb R$, then the metric space $(\mathbb R^n, \rho_2)$ is so-called \textit{Euclidean $n$-space}.

	For all $p,q \in \mathbb R_{\ge 1}$, if $p < q$, then for all $\varepsilon \in \mathbb R_{> 0}$ and for all $x,y \in X$, $\rho_p(x,y) \ge \rho_q(x,y)$; in particular, $\rho_p = \rho_q$ iff there is a unique $k \in \{1, \ldots, n\}$, such that for all $i \in \{1, \ldots, n\} \setminus \{k\}$, $x_i = 0$. As $\rho_p(x,y)$ is always ``overestimated'' than $\rho_q(x,y)$, we have $B_{\rho_p}(x, \varepsilon) \supseteq B_{\rho_q}(x,\varepsilon)$.
\end{example}
%--------------------------------

%--------------------------------
\begin{example}
	\label{eg: discrete metric}
	Let $X$ be any set. The \textit{discrete metric} $\rho$ on $X$ is defined to be
	$$
	\rho(x,y) =
	\begin{cases}
		0 & \text{if $x = y$}, \\
		1 & \text{otherwise}.
	\end{cases}
	$$
\end{example}
%--------------------------------

%--------------------------------
\begin{example}
	Let $a,b \in \mathbb R$ with $a< b$, and let metric $\rho_p$ on $C[a,b]$ defined by
	$$
	\rho_p (f, g) = \left( \int_a^b |f(t) - g(t)|^p dt \right)^\frac{1}{p},
	$$
	where $p \ge 1$. In particular,
	$$
	\rho_\infty (f,g) = \sup_{t \in [a,b]} |f(t) - g(t)|.
	$$
\end{example}
%--------------------------------

% todo: more examples
% 1. Hamming metric
% 2. Great circle metric
% 3. Hausdorff metric

%--------------------------------
\begin{theorem}
	Let $(X, \rho)$ be a metric space, and let
	$$
	\mathcal T_\rho = \left\{ \bigcup_{x \in U} B_\rho (x, \varepsilon) : U \subseteq X \land \varepsilon \in \mathbb R_{> 0} \right\}.
	$$
	$(X, \mathcal T_\rho)$ is a topological space; i.e., $\mathcal T_\rho$ satisfies the open set axioms (Definition \ref{def: topology}).

	\begin{proof}
		For O1. Apparently, there exists (for any) $\varepsilon \in \mathbb R_{> 0}$,
		$$
		X = \bigcup_{x \in X} B(x, \varepsilon).
		$$

		For O2. Let $I$ be an index set, and let $U_i \in T$ for all $i \in I$. There exists $\varepsilon \in \mathbb R_{> 0}$ we can define
		$$
		U = \bigcup_{i \in I} U_i = \bigcup_{i \in I} \bigcup_{x \in U_i} B(x, \varepsilon) = \bigcup_{x \in U} B(x, \varepsilon).
		$$
		Then, for all $x \in U$, there exists $B(x, \varepsilon) \ni x$. Thus $U \in \mathcal T$.

		For O3. Let $U, V \in \mathcal T$. If $U \cap V = \emptyset$, then the proof is done. Suppose $U \cap V \ne \emptyset$, and let $x \in U \cap V$. $U$ is open and $x \in U$, so there exists $r_1 \in \mathbb R_{> 0}$ such that $B(x, r_1) \subseteq U$; $V$ is open and $x \in V$, so there exists $r_2 \in \mathbb R_{> 0}$ such that $B(x, r_2) \subseteq V$.

		If $r_1 = r_2$, then $B(x, r_1) = B(x, r_2)$. If $r_1 \ne r_2$, say $r_1 < r_2$, then $B(x, r_1) \subseteq B(x, r_2) \subseteq V$. Above all, for any $x \in U \cap V$, there exits $r \in \mathbb R_{>0}$ such that $B(x, r) \subseteq U \cap V$. Thus, there exists $\varepsilon \in \mathbb R_{> 0}$ such that
		$$
		U \cap V = \bigcup_{x \in U \cap V} B(x, \varepsilon),
		$$
		i.e., $U \cap V \in \mathcal T_\rho$.
	\end{proof}
\end{theorem}
%--------------------------------

%--------------------------------
\begin{theorem}
	\label{prop: metric spaces are Hausdorff}
	Let $(X, \rho)$ be a metric space, then for all $x, y \in X$ ($x \ne y$), there is an $\varepsilon > 0$ such that $B(x, \varepsilon) \cap B (y, \varepsilon) = \emptyset$.

	\begin{proof}
		Suppose for all $\varepsilon > 0$, $B(x, \varepsilon) \cap B(y, \varepsilon) \ne \emptyset$, then there must be a $z \in X$ such that $z \in B(x, \varepsilon) \cap B(y, \varepsilon)$. $z \in B(x, \varepsilon)$ only if $\rho(x,z) < \varepsilon$, and $z \in B(y, \varepsilon)$ only if $\rho (z,y) < \varepsilon$. Thus
		$$
		\rho(x, z) + \rho(y,z) < 2\varepsilon.
		$$
		As the assumption holds for all $\varepsilon > 0$, we may put
		$$
		\varepsilon = \frac{\rho(x,y)}{2}.
		$$
		Then, we have
		$$
		\rho(x,z) + \rho(y,z) < \rho(x,y),
		$$
		which is impossible.
	\end{proof}
\end{theorem}
%--------------------------------

%--------------------------------
\begin{definition}
	[induced topologies]
	\label{def: induced topologies}
	Let $(X, \rho)$ be a metric space. A topology $\mathcal T$ on $X$ is said to be \textit{induced} by $\rho$ iff for all $\varepsilon > 0$, any $U \in \mathcal T$ is the union of ball(s) in $X$; i.e.,
	$$
	\mathcal T = \left\{ U \subseteq X :  U = \bigcup_{x \in X} B(x, \varepsilon) \right\}.
	$$

	In this case, $\mathcal T$ is called the \textit{underlying topology} of $\rho$.
\end{definition}
%--------------------------------

%--------------------------------
\begin{definition}
	[metrizable spaces]
	\label{def: metrizable spaces}
	Let $(X, \mathcal T)$ be a topological space. If there is any $\rho$ induce $\mathcal T$, then $(X, \mathcal T)$ is said to be \textit{metrizable}.
\end{definition}
%--------------------------------

%--------------------------------
\begin{definition}
	[Lipschitz equivalence]
	\label{def: Lipschitz equivalence}
	Let $X$ be any set, and let $\rho$ and $\rho'$ be metrics on $X$. $\rho$ and $\rho'$ are said to be \textit{Lipschitz equivalent} iff there exist $c, C > 0$, such that for all $x,y \in X$,
	$$
	c \rho(x,y) \le \rho'(x,y) \le C \rho(x,y).
	$$
\end{definition}
%--------------------------------

%--------------------------------
\begin{theorem}
	\label{prop: Lipschitz equivalence is a equivalence relation}
	Lipschitz equivalence is an equivalence relation.

	\begin{proof}
		Clearly, Definition \ref{def: Lipschitz equivalence} also holds for $\rho = \rho'$. So, Lipschitz equivalence is reflexive. In Definition \ref{def: Lipschitz equivalence}, the relation also holds for $\frac{1}{C} \rho' \le \rho \le \frac{1}{c} \rho'$. So Lipschitz equivalence is symmetric.

		If there is another $\rho''$ be Lipschitz equivalent to $\rho'$, then there is $r, R > 0$, such that for all $x,y \in X$,
		$$
		r\rho''(x,y) \le \rho'(x,y) \le R\rho''(x,y).
		$$
		By the conditions in Definition \ref{def: Lipschitz equivalence}, we have
		$$
		\frac{c}{r} \rho(x,y) \le \rho''(x,y) \le  \frac{C}{R} \rho(x,y),
		$$
		i.e., $\rho$ and $\rho''$ are also Lipschitz equivalent. So Lipschitz equivalence is transitive.

		Above all, Lipschitz equivalence is an equivalence relation.
	\end{proof}
\end{theorem}
%--------------------------------

%--------------------------------
\begin{theorem}
	\label{prop: Lipschitz equivalent metrics induces the same topology}
	Let $X$ be any set, and let $\rho$ and $\rho'$ be metrics on $X$. If $\rho$ and $\rho'$ are Lipschitz equivalent, then $\rho$ and $\rho'$ induce the same topology.

	\begin{proof}
		As $\rho$ and $\rho'$ are Lipschitz equivalent, by Definition \ref{def: Lipschitz equivalence}, there is a $c > 0$ such that for all $x,y \in X$,
		$$
		c \rho(x,y) \le \rho'(x,y).
		$$

		Given $r \in \mathbb R_{> 0}$ and for all $x \in X$, we have
		$$
		B_{\rho'}(x, cr) \subseteq B_{c \rho}(x, r) = B_\rho \left( x, \frac{1}{c} r \right).
		$$
		For all $U \in \mathcal T_\rho$, for all $x \in U$, there is an $\varepsilon \in \mathbb R_{> 0}$, such that
		$$
		B_{\rho'} (x, \varepsilon) \subseteq B_{\rho}(x, \varepsilon) \subseteq U.
		$$
		So $U \in \mathcal T_\rho'$. Then we have $\mathcal T_{\rho} \subseteq \mathcal T_{\rho'}$.

		Similarly, $U \in \mathcal T_{\rho'}$ only if $U \in \mathcal T_\rho$. Then we have $\mathcal T_{\rho'} \subseteq \mathcal T_{\rho}$.

		Above all, $\mathcal T_{\rho} = \mathcal T_{\rho'}$.
	\end{proof}
\end{theorem}
%--------------------------------

%--------------------------------
\begin{note}
	In this proposition, $\mathcal T_\rho$ and $\mathcal T_{\rho'}$ are said to be homeomorphic or topologically equivalent (see Definition \ref{def: homomorphic}). And $\rho$ and $\rho'$ are also said to be topologically equivalent.
\end{note}
%--------------------------------

%--------------------------------
\begin{example}
	In Example \ref{eg: Euclidean metrics}, for all $p,q \ge 1$, all $\rho_p$ and $\rho_q$ induce the same topology. Let $X$ be any subset of $\mathbb R^n$, then for all $x,y \in X$, if $p < q$, then
	$$
	\rho_p (x,y) \ge \rho_q (x,y).
	$$
	Thus, if $\rho_1$ and $\rho_\infty$ are Lipschitz equivalent, then any other $\rho_p$ and $\rho_q$ are Lipschitz equivalent. We have
	$$
	\rho_1 (x,y) = \sum_{i = 1}^n |x_i - y_i| \ge \max_{i \in \{1, \ldots, n\}} |x_i - y_i| = \rho_\infty (x,y).
 	$$
 	Clearly,
 	$$
 	\rho_\infty (x,y) \le \rho_1 (x,y) \le n \rho_\infty (x,y).
 	$$
 	By Definition \ref{def: Lipschitz equivalence}, $\rho_1$ and $\rho_\infty$ are Lipschitz equivalent, hence for all $p, q \ge 1$, $\rho_p$ and $\rho_q$ are Lipschitz equivalent. Thus, by Proposition \ref{prop: Lipschitz equivalent metrics induces the same topology}, they induce the same topology.
\end{example}
%--------------------------------


\section{Some Matrix Groups}
\begin{definition}
    Let $\mathbb F$ be a field (e.g. $\mathbb C,\mathbb Z/p\mathbb Z$), then we define
    $$\operatorname{GL}_n(\mathbb F)=\{M\in\mathcal M_{n\times n}(\mathbb F):\det M\neq 0\}$$
    $$\operatorname{SL}_n(\mathbb F)=\{M\in\mathcal M_{n\times n}(\mathbb F):\det M\neq 0\}=\ker\det|_{\operatorname{GL}_n(\mathbb F)}\unlhd\operatorname{GL}_n(\mathbb F)$$
\end{definition}
Note that $Z=Z(\operatorname{GL}_n(\mathbb F))$ is the set of scalar matrices.
\begin{definition}
    $$\operatorname{PGL}_n(\mathbb F)=\operatorname{GL}_n(\mathbb F)/Z$$
    $$\operatorname{PSL}_n(\mathbb F)=\operatorname{SL}_n(\mathbb F)/(Z\cap \operatorname{SL}_n(\mathbb F))\cong Z\operatorname{SL}_n(\mathbb F)/Z\le\operatorname{PGL}_n(\mathbb F)$$
\end{definition}
\begin{example}
    Let $G=\operatorname{GL}_n(\mathbb Z/p\mathbb Z)$.
    A list of $n$ vectors in $(\mathbb Z/p\mathbb Z)^n$ are the columns of some $A\in G$ iff they are linearly independent.
    Hence
    $$|G|=(p^n-1)(p^n-p)(p^n-p^2)\cdots (p^n-p^{n-1})=p^{n(n-1)/2}\prod_{i=1}^n(p^i-1)$$
    So the Sylow $p$-subgroups have order $p^{n(n-1)/2}$.
    Indeed, the subgroup of upper-triangular matrices with $1$'s on the diagonal gives one (hence all by conjugation) of such subgroups.
\end{example}
Just like $\operatorname{PSL}_2(\mathbb C)$ act on $\mathbb C\cup\{\infty\}$ by Mobius transformations, the group $\operatorname{PSL}_2(\mathbb Z/p\mathbb Z)$ can act on $\mathbb Z/p\mathbb Z\cup\{\infty\}$ in this way as well:
$$\begin{pmatrix}
    a&b\\
    c&d
\end{pmatrix}:z\mapsto\frac{az+b}{cz+d}$$
where the infinity cases are dealt with in the same way.
\begin{lemma}\label{psl_mobius}
    The permutation representation $\operatorname{PSL}_2(\mathbb Z)\to S_{p+1}$ by Mobius transformation is injective.
\end{lemma}
And indeed, by considering the size, for $p=2,3$, this is an isomorphism.
\begin{proof}
    Suppose there is some $a,b,c,d$ such that $\forall z\in\mathbb C,z=(az+b)/(cz+d)$, then putting $z=0$ gives $b=0$, and $z=\infty$ gives $c=0$, and $z=1$ gives $a=d$, but these only gives one element that is the identity of $\operatorname{PSL}_2(\mathbb Z/p\mathbb Z)$.
    Done.
\end{proof}
\begin{lemma}
    If $p$ is an odd prime, then $|\operatorname{PSL}_2(\mathbb Z/p\mathbb Z)|=p(p-1)(p+1)/2$.
\end{lemma}
\begin{proof}
    We already know that
    $$|\operatorname{GL}_2(\mathbb Z/p\mathbb Z)|=p(p-1)(p^2-1)$$
    Then by Isomorphism Theorem
    $$|\operatorname{SL}_2(\mathbb Z/p\mathbb Z)|=|\operatorname{GL}_2(\mathbb Z/p\mathbb Z)|/|(\mathbb Z/p\mathbb Z)^\star|=p(p-1)(p+1)$$
    And
    $$|\operatorname{PSL}_2(\mathbb Z/p\mathbb Z)|=|\operatorname{SL}_2(\mathbb Z/p\mathbb Z)|/|\{\pm I\}|=\frac{p(p-1)(p+1)}{2}$$
    Here $Z\cap\operatorname{SL}_2(\mathbb Z/p\mathbb Z)=\{\pm I\}$ because $a^2\equiv 1\pmod{p}$ only has two solutions, namely $\pm 1$, as $p$ is prime.
\end{proof}
Note that for $p=2$, things go wrong on the fact that in this case $I=-I$.
\begin{example}
    Consider the group $G=\operatorname{PSL}_2(\mathbb Z/5\mathbb Z)$, then $|G|=60=2^2\cdot3\cdot5$.
    We shall show that $G$ is simple, and in fact, it is isomorphic to $A_5$.\\
    Let $G$ act on $\mathbb Z/5\mathbb Z\cup\{\infty\}$ by Mobius transformation.
    By Lemma \ref{psl_mobius}, we have an injective group homomorphism $\phi:G\to S_6$.\\
    Our first claim that, if we embed $G$ into $S_6$, then $G\le A_6$.
    Equivalently the map $\psi: G\to S_6\to \{\pm 1\}$ is trivial, where the first arrow is $\phi$ and the second is the signature.\\
    Note that for odd $m$, we have $\psi(g)=1\iff\psi(g^m)=1$, so it suffices to study the elements of order being a power of $2$, but this means to study the elements contained in every Sylow $2$-subgroup of $G$, but all of them are conjugate, it is enough to check one of them (since $\{\pm 1\}$ is abelian).
    We spot the following one
    $$H=\left\{\pm\begin{pmatrix}
        2&0\\
        0&3
    \end{pmatrix},
    \pm\begin{pmatrix}
        0&1\\
        4&0
    \end{pmatrix}\right\}$$
    By simple computation, $\psi$ does vanish on $H$, so indeed we have $G\le A_6$.
    By a result in Example Sheet 1, if $G\le A_6$ and $|G|=60$, then $G\cong A_5$, which completes the proof.
\end{example}
The following facts will not be proved in the course, but are very important:
\begin{proposition}
    $\operatorname{PSL}_n(\mathbb Z/p\mathbb Z)$ is simple for $n\ge 2$ and $p$ prime except if $(n,p)=(2,2)$ or $(n,p)=(2,3)$.
    Also, the two smallest nonabelian simple groups are $A_5\cong\operatorname{PSL}_2(\mathbb Z/5\mathbb Z)$ with order $60$ and $\operatorname{PSL}_2(\mathbb Z/7\mathbb Z)\cong\operatorname{GL}_3(\mathbb Z/2\mathbb Z)$ with order $168$.
\end{proposition}


\section{Schur Decomposition}

For any square matrix $\mathbf{A}$, there exist a unitary matrix $\mathbf{H}$ so that:

\begin{equation}
\mathbf{T} = \mathbf{U}^{\dagger}\mathbf{A}\mathbf{U}
\end{equation}

Where $\mathbf{T}$ is an upper triangular matrix. This concept is the same as saying that every square matrix is similar to an upper triangular matrix.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS624: Analysis of Algorithms
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 3}

Prove that $\log_a x = O(log_b x)$ for any $a > 0$ and $b > 0$.

\subsection*{Solution}
It suffices to introduce two constants $c$ and $x_0$ such that

\begin{equation}
\log_a x \leq c\times \log_b x
\label{eq7}
\end{equation}

for any $x \geq x_0$.
To make Equation \ref{eq7} into equality, $c$ is obtained as

\begin{equation}
c = \frac{log_a x}{log_b x} = log_a b
\end{equation}

Thus Equation \ref{eq7} is valid for all $x > x_0 = 0$ and for any $c \geq log_a b$.


\subsection{Examples of linear operators on real functions}

For a function \(v\) we can define operators \(Ov\).

Here we consider some examples and their properties.

\subsubsection{Real multiplication}

\(Rv = rf(x)\)

This operator is hermitian. This is equivalent to a finite operator of the form \(rI\).

\subsubsection{Multiplication by underlying real number}

\(Xv = xf(x)\)

This operator is hermitian. This is equivalent to a finite operator of the form \(M_{ii}=i\) and \(M_{ij}=0\).

\subsubsection{Differentiation}

\(Dv = \dfrac{\delta }{\delta x}f(x)\)

While this operator is not hermitian, the following is:

\(-iDv = \dfrac{\delta }{\delta x}[-if(x)]\)


\section{General ReLU DNN as linear finite element functions}

Here we will show that any ReLU DNN function can be viewed as linear finite element function.
Let us first consider a ReLU DNN function with $L$ hidden layers
\begin{equation}\label{key}
f(x) = \theta^{L}\circ \sigma \circ \theta^{L-1} \circ \sigma \cdots \circ \theta^1 \circ \sigma \circ \theta^0(x),
\end{equation}
for any $X \in \mathbb{R}^d$.
Based on the property of ${\rm ReLU}$ activation function, we know that $f(x)$ is a continuous
piecewise linear function. More precisely, let us consider a closed bounded domain, for example $\Omega = [-1, 1]^d$ and
\begin{equation}\label{key}
\left. f(x) \right|_{E_i} = \ell_i(x),  \quad \text{ for } i =1,2,\cdots M,
\end{equation}
for any $x\in \Omega$, where $\ell_i(x) = w_i x + b_i$ is a linear function on $\mathbb{R}^d$ and
$\{E_i\}_{i=1}^M$ are the domains which consist a partition of $\Omega$.
Furthermore, we know that the boundary of $E_i$ consists of some hyperplanes
of $d$-dimensional space and $E_i \cap E_j = \emptyset$ or $E_i \cap E_j $ belongs to
hyperplanes of lower dimensional space.

Then we have the following representation theory to represent any ReLU DNN function
by linear finite element function.
\begin{theorem}
Any ReLU DNN function on $[-1,1]^d$ can be represent
by a linear finite element function.
\end{theorem}
The proof of this theorem can be found in~\cite{he2020approximation}.

Here we show a brief description of the main steps in the proof, which will appear in~\cite{he2020approximation}.
\begin{enumerate}
	\item Based on the discussion before, we have a partition of $\Omega$ as
	\begin{equation}\label{key}
	\Omega = \bigcup_{i=1}^M E_i.
	\end{equation}
	\item We can extend all the $d$-dimensional hyperplanes on the boundary of all $E_i$ to achieve
	a fine partition of $\Omega$ noted as
	\begin{equation}\label{key}
	\Omega = \bigcup_{i=1}^{\tilde M} \tilde E_i,
	\end{equation}
	with an important feature that each $\tilde E_i$ is a convex n-dimensional polytope.
	\item For these $\{\tilde E_i\}_{i=1}^{\tilde M}$, we can further partition each $\tilde E_i$
	to be a collection of n-dimensional simples. Finally, we can rewrite the continuous
	piecewise linear function $f(x)$ on a simplicial mesh of $\Omega$, which means that $f(x)$
	will can be written as a linear finite element function.
\end{enumerate}


\section{Multistep Method}
\begin{defn}
An $m$-step multistep method for solving the initial-value problem
\[
y'=f(t,y),\quad a\leq t\leq b,\quad y(a)=\alpha
\]]
has a difference equation for finding the approximation $\omega_{i+1}$ at the mesh point $t_{i+1}$ represented by the following equation, where $m$ is an integer greater than 1:
\begin{align*}
    \omega_{i+1} =& a_{m-1}\omega_{i}+a_{m-2}\omega_{i-1}+\cdots+a_{0}\omega_{i+1-m} \\
    & + h\left[b_{m}f(t_{i+1},\omega_{i+1})+b_{m-1}f(t_{i},\omega_{i})\right. \\
    \left.+\cdots+b_{0}f(t_{i+1-m},\omega_{i+1-m})\right]
\end{align*}
for $i=m-1,m,\cdots,N-1$, where $h=\frac{b-a}{N}$, the $a_0,a_1,\cdots a_{m-1}$ and $b_0,b_1,\cdots,b_m$ are constant, and the starting values
\[
\omega_{0}=\alpha_{0},\quad\omega_{1}=\alpha_{1},\cdots,\omega_{m-1}=\alpha_{m-1}
\]
are specified.

\begin{equation*}
    \begin{cases}
    \text{When $b_m=0$, the method is called explicit, or open.} \\
    \text{When $b_m\neq 0$, the method is called implicit, or closed.}
    \end{cases}
\end{equation*}
\end{defn}


\subsection{Kronecker product}

The Kronecker product takes the concept of the outer product and applies to to matrices.

We can essentially repace every element in the matrix on the left with the element multiplied by the entire matrix on the right.

Like outer products, Kronecker products are written as:

\(u\otimes v=w\)


\documentclass[12pt]{scrartcl}

\input{preamble}

\makeatletter
\title{Hack 5.0}\let\Title\@title
\subtitle{Computer Science I\\
{\small
\vskip1cm
Department of Computer Science \& Engineering \\
University of Nebraska--Lincoln}
\vskip-1cm}
%\author{Dr.\ Chris Bourke}
\date{~}
\makeatother

\begin{document}

\maketitle

\hrule

\input{instructions.tex}

\section*{Problem Statement}

To get some practice designing and using functions, you will
create a small library of utility functions by implementing
the following functions with the given prototypes and specified
functionality.
\begin{enumerate}
  \item \mintinline{c}{double degreesToRadians(double degree);} - Write a
  	function to convert degrees to radians using the formula
		$$\frac{d\cdot \pi}{180}$$
  \item Write a function to compute the air distance between two locations
    identified by their latitude/longitude.
\begin{minted}{c}
double getAirDistance(double originLatitude,
                      double originLongitude,
                      double destinationLatitude,
                      double destinationLongitude);
\end{minted}
The air distance between two latitude/longitude points can be calculated
using the Spherical Law of Cosines:
 $$d = \arccos{(\sin(\varphi_1) \sin(\varphi_2) + \cos(\varphi_1) \cos(\varphi_2) \cos(\Delta) )} \cdot R$$
where
\begin{itemize}
  \item $\varphi_1$ is the latitude of location $A$, $\varphi_2$ is the latitude of location $B$
  \item $\Delta$ is the difference between location $B$'s longitude and location $A$'s longitude
  \item $R$ is the (average) radius of the earth, 6,371 kilometers
\end{itemize}
Note: the formula above assumes that latitude and longitude are measured
in radians $r$, $-\pi \leq r \leq \pi$, but the function will expect
the latitude/longitude to be in degrees.  Latitude should be in the range
$[-90, 90]$ and longitude in the range $[-180, 180]$.  Negative values
correspond to the southern and western hemispheres.

  \item An object traveling at a velocity $v$ experiences time dilation
  relative to a stationary object which is quantified by the Lorentz equation:
  $$T = \frac{t}{\sqrt{(1-\frac{v^2}{c^2})}}$$
  where $t$ is the normal amount of lapsed time (stationary object)
  and $T$ is the dilated time experienced by the traveling object.
  For small velocities, the dilation is small, but for velocities
  approaching a \emph{percentage} (on the scale $[0, 1]$) of the
  speed of light, $c$, the dilation becomes significant.

  For example, at 25\% the speed of light, a year for the object
  traveling would correspond to 1.032796 years at the stationary object
  (or nearly 12 extra days).   A person traveling at high velocity
  would experience ``slowed'' time
  relative to the stationary \emph{frame}.

  Implement a function to compute the dilated time given the
  normal time $t$ (units may vary) and the \emph{percentage}
  (i.e.\ $\frac{v}{c}$) of the speed of light.
\begin{minted}{c}
double lorentzTimeDilation(double t, double percentC);
\end{minted}

\end{enumerate}

\section*{Instructions}

\begin{itemize}
  \item You are encouraged to collaborate any number of students
  before, during, and after your scheduled hack session.
  \item Design at least 3 test cases for each function
  \emph{before} you begin
  designing or implementing your program.  Test cases are
  input-output pairs that are known to be correct using means
  other than your program.
  \item Include the name(s) of everyone who worked together on
  this activity in your source file's header.
  \item Place your prototypes and documentation in a header file
  named \mintinline{text}{utils.h} and your source in a file
  named \mintinline{text}{utils.c}.
  \item In addition, implement all of your test cases in a
  \emph{test driver} file named \mintinline{text}{utilsTester.c}
  which should output the expected output, the actual output and
  a message on whether or not the test case passed.  You must have
  at least 3 test cases for \emph{each} of your functions.
  You should \emph{not} prompt for input or use command line
  arguments.  Your test cases should be hardcoded in your test
  driver's \mintinline{c}{main} function.
  \item Turn in all of your files via webhandin, making sure that
  it runs and executes correctly in the webgrader.  Each individual
  student will need to hand in their own copy and will receive
  their own individual grade.
\end{itemize}

\end{document}


\section{Conics in Polar Coordinates}\label{sec:Conics in Polar Coordinates}
A \dfont{conic section}\index{conic section}\index{polar coordinates!conic section} is a curve obtained as the intersection of a
cone and a plane. One useful geometric definition that only involves the plane
is that a conic consists of those points whose distances to some point, called a
\dfont{focus}, and some line, called a \dfont{directrix}, are in a fixed ratio,
called the \dfont{eccentricity}.

The three types of conic sections are the ellipse, parabola and hyperbola (with the circle being a degenerate case of an ellipse).

$$\includegraphics[width=3in]{images/ConicSections}$$

Let $F$ be a fixed point (the focus), $L$ be a line (the directrix) not containing $F$ and $e$ be a nonnegative real number (the eccentricity).
The conics sections are obtained by the set of all points $P$ whose distance to $F$ equals $e$ times their distance to $L$, that is:
$$\frac{|PF|}{|PL|}=e.$$
In the case that:
\begin{itemize}\setlength{\itemsep}{0 in}
\item $e<1$ we obtain an ellipse (and when $e=0$ we obtain the degenerate case: A circle),
\item $e=1$ we obtain a parabola,
\item $e>1$ we obtain a hyperbola.
\end{itemize}

To obtain a simple polar equation we place the focal point at the origin.
The formulation for a conic section is then given in the polar form by
$$r=\frac{pe}{1\pm e\cos\theta}\qquad\mbox{and}\qquad r=\frac{pe}{1\pm e\sin\theta}$$
where $e$ is the eccentricity and $p$ is the focal parameter representing the distance from the focus (or one of the two foci) to the directrix.

The three different types of conic sections are shown below. Focal points corresponding to all conic sections are placed at the origin.
First is the parabola.
$$\includegraphics[width=5in]{images/conics-polar1}$$
Next is the ellipse.
$$\includegraphics[width=5in]{images/conics-polar2}$$
Finally, we have the hyperbola.
$$\includegraphics[width=5in]{images/conics-polar3}$$

Some things to keep in mind with respect to the denominator of polar equations of various conics are the following:
\begin{itemize}\setlength{\itemsep}{0 in}
\item If the denominator is $1 + e \sin\theta$, it has a horizontal directrix above the focal point.
\item If the denominator is $1 - e \sin\theta$, it has a horizontal directrix below the focal point.
\item If the denominator is $1 + e \cos\theta$, it has a vertical directrix to the right of the focal point.
\item If the denominator is $1 - e \cos\theta$, it has a vertical directrix to the left of the focal point.
\end{itemize}

\begin{example}{Polar Equations for a Parabola}{polareqnparabola}
Find the equation of a parabola with focus at the origin and whose directrix is the line $x=-1$.
\end{example}

\begin{solution}
Since we have a parabola, $e=1$.
Furthermore, $p=1$.
Since the graph has a vertical directrix, the equation will use $1 - e \cos\theta$ in the denominator.
Thus, the equation is:
$$r=\frac{2}{1-\sin\theta}$$
\end{solution}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for \ref{sec:Conics in Polar Coordinates}}

\begin{enumialphparenastyle}

%%%%%%%%%%
\begin{ex}
\noindent Identify the following conics and find the eccentricity.
\begin{multicols}{2}
\begin{enumerate}
	\item	$\ds r=\frac{2}{1+\sin\theta}$
	\item	$\ds r=\frac{4}{2+\cos\theta}$
	\item	$\ds r=\frac{3}{1-\sin\theta}$
	\item	$\ds r=\frac{5}{2+2\sin\theta}$
\end{enumerate}
\end{multicols}
\end{ex}

%%%%%%%%%%
\begin{ex}
Write the polar equation of a parabola with focus at the origin and directrix $x=3$.
\end{ex}

%%%%%%%%%%
\begin{ex}
Write the polar equation of a hyperbola with focus at the origin, directrix $x=4$ and eccentricity $2$.
\end{ex}

%%%%%%%%%%
\begin{ex}
Write the polar equation of an ellipse with focus at the origin, directrix $x=4\sec\theta$ and eccentricity $1/2$.
\end{ex}

\end{enumialphparenastyle}


\textbf{Theorem 6 makes a prediction about the geometric rate of convergence in the third pane of Output 12. Exactly what is this prediction? How well does it match the observed rate of convergence?}
\newline

Since $f(x) = \frac{1}{1+x^2}$ is analytic everywhere exept $x=\pm i$, where it has singularities, it is analytic on and inside the ellipse with foci $\{-1,1\}$ and semiminor axis $b < 1$ on which the Chebishev potential takes the value
\begin{align*}
\phi_f = \log\left(\frac{1}{2}(a+b)\right) \approx \log\left(\frac{1}{2}(\sqrt{2}+1)\right),
\end{align*}
where we have obtained a from the foci identity $ f = a^2 - b^2$. Hence, by \textsc{Theorem 6}, the convergence should be
\begin{align*}
|w_j-u^{(v)}(x_j)| = \mathcal{O}\left(e^{-N(\phi_f+\log{2})}\right) = \mathcal{O}\left(e^{-(\sqrt{2}+1)N}\right),
\end{align*}
which is proved to be right by the next figure.
\begin{figure}[H]
\centering
\includegraphics[scale=0.75]{P6_7.png}\caption{Acuracy of the Chebishev spectral derivative for $f(x) = \frac{1}{1+x^2}$.}
\end{figure}

\subsection*{Matlab code for this problem}
\begin{verbatim}
%% Problem 3 - 6.7 Trefethen
Nmax = 50; E = zeros(Nmax,1);
for N = 1:Nmax
    [D,x] = cheb(N);
    v = 1./(1+x.^2); vprime = -2*x.*v.^2;    % analytic in [-1,1]
    E(N) = norm(D*v-vprime,inf);
end
% Define ellipse and potential on it.
a = sqrt(2) ; b = 1;
phif = log(0.5*(a+b));
% Plot results:
figure
semilogy(1:Nmax,E(:),'r*')
hold on
semilogy(1:Nmax,exp(-(phif+log(2))*(1:Nmax)))
axis([0 Nmax 1e-16 1e3]), grid on
set(gca,'xtick',0:10:Nmax,'ytick',(10).^(-15:5:0))
xlabel N, ylabel error
txt='Latex/FIGURES\P6_7';
saveas(gcf,txt,figformat)
\end{verbatim}


\subsection{Maximum Likelihood Estimator (MLE) and OLS equivalence}

\subsubsection{The OLS estimator}

\(\hat \theta_{OLS}=(X^TX)^{-1}X^Ty\)

\(E[\hat \theta_{OLS}]=w\)

\(Var[\hat \theta_{OLS}]=\sigma^2 (X^TX)^{-1}\)

\subsubsection{The MLE estimator}

\(y_i=\mathbf x_i\theta +\epsilon_i \)

\(P(y=y_i|x=x_i)=P(\epsilon_i=y_i-\mathbf x_i \theta )\)

If we assume \(\epsilon_i \sim N(0, \sigma^2_\epsilon )\) we have:

\(P(y=y_i|x=x_i)=\dfrac{1}{\sqrt {2\pi \sigma^2_\epsilon }}e^{-\dfrac{(y_i-\mathbf x_i\theta )^2}{2\sigma_\epsilon^2}}\)

\(L(X, \theta )=\prod_{i=1}^n\dfrac{1}{\sqrt {2\pi \sigma^2_\epsilon }}e^{-\dfrac{(y_i-\mathbf x_i\theta )^2}{2\sigma_\epsilon^2}}\)

\(l(X, \theta )=\sum_{i=1}^n -\dfrac{1}{2}\ln (2\pi \sigma_\epsilon^2)-\dfrac{(y_i-\mathbf x_i\theta )^2}{2\sigma_\epsilon^2}\)

\(\dfrac{\delta l}{\delta \theta_j }=\sum_{i=1}^n2x_{ij}\dfrac{y_i-\mathbf x_{i}\theta }{2\sigma^2_\epsilon}\)

\(\sum_{i=1}^nx_{ij}(y_i-\hat \theta_{MLE}\mathbf x_{i} )=0\)

\(X^T(y-X\hat \theta_{MLE} )=0\)

\(X^Ty=X^TX\hat \theta_{MLE} \)

\(\hat \theta_{MLE}=(X^TX)^{-1}X^Ty\)

\subsubsection{Equivalence}

If errors are normally IID then:

\(\hat \theta_{OLS}=\hat \theta_{MLE}\)


\section{Hermite Interpolation}
The osculating polynomial approximating a function $f\in C^m[a,b]$ at $x_i$ for each $i=0:n$, of which the derivatives of order less than or equal to $m_i$, then the degree of this osculating polynomial is at most $M=\sum_{i=0}^{n}m_i+n$.
\[
\diff{^kP(x_i)}{x^k}\,=\,\diff{^kf(x_i)}{x^k},\quad\text{for each $i=0:n$, $k=0:m_i$}.
\]
\begin{equation*}
    \begin{cases}
    n=0 & \text{$m_0$ Taylor polynomial for $f$ at $x_0$} \\
    m_i=0(\text{each $i$}) & \text{$n$th Lagrange polynomial}
    \end{cases}
\end{equation*}

\subsection{Hermite Polynomials}
\begin{theo}
$f\in C'[a,b]$ and $x_0,\cdots,x_n\in [a,b]$, the unique polynomial of least degree agreeing with $f$ and $f'$ at $x_0,\cdots,x_n$ is the Hermite polynomial of degree at most $2n+1$.
\begin{align*}
    H_{2n+1}(x)\,=\,\sum_{j=0}^nf(x_j)H_{n,j}(x) + \sum_{j=0}^nf'(x_j)\hat{H}_{n,j}(x) \\
    \begin{cases}
    H_{n,j}(x) &= \left[1-2(x-x_j)L_{n,j}'(x_j)\right]L_{n,j}^2(x) \\
    \hat{H}_{n,j}(x) &= (x-x_j)L_{n,j}^2(x)
    \end{cases}
\end{align*}
Moreover, if $f\in C^{2n+2}[a,b]$, then
\[
f(x)=H_{2n+1}(x)+\frac{(x-x_0)^2\cdots(x-x_n)^2}{(2n+2)!}f^{(2n+2)}\left(\xi(x)\right).
\]
\end{theo}
\begin{proof}
\begin{align*}
    & H_{n,j}(x_i)=\delta_{i,j} & & \hat{H}_{n,j}(x_i)=0 & \\
    & H_{n,j}'(x_i)=0 & & \hat{H}_{n,j}'(x_i)=\delta_{i,j} & \\
\end{align*}
\end{proof}

\subsection{Hermite Polynomials Using Divided Differences}
Suppose that the distinct numbers $x_0,\cdots,x_n$ are given together with values of $f$ and $f'$. Define a new sequence $z_0,\cdots,z_{2n+1}$ by
\[
z_{2i} = z_{2i+1} = x_i\quad\text{for $i=0:n$}.
\]
We have $H_{2n+1}(x)=f[z_0]+\sum_{k=1}^{2n+1}f[z_0,\cdots,z_k](x-z_0)\cdots(x-z_{k-1})$.
\vspace{1cm}\\
\begin{tabular}{@{}llll@{}}
\toprule
$z$ & $f(z)$ & First divided differences & $\cdots$ \\ \midrule
$z_0=x_0$ & $f[z_0]=f(x_0)$ & $f[z_0,z_1]=f'(x_0)$ & $\vdots$ \\
$z_1=x_0$ & $f[z_1]=f(x_0)$ & $f[z_1,z_2]=\frac{f[z_2]-f[z_1]}{z_2-z_1}$ & $\vdots$ \\
$z_2=x_1$ & $f[z_2]=f(x_1)$ & $f[z_2,z_3]=f'(x_1)$ & $\vdots$ \\
$z_3=x_1$ & $f[z_3]=f(x_1)$ & $f[z_3,z_4]=\frac{f[z_3]-f[z_4]}{z_3-z_4}$ & $\vdots$ \\
$z_4=x_2$ & $f[z_4]=f(x_2)$ & $f[z_4,z_5]=f'(x_2)$ &  \\
$z_5=x_2$ & $f[z_5]=f(x_2)$ &  &  \\ \bottomrule
\end{tabular}


\subsection{Inequalities}

\subsubsection{Less than or equal}

Orderings define relations between elements in a set, where one element can preceed the other.

Orderings are antisymmetric. That is, the only case where the relation is satisfied in both directions is if the elements are equal.

\((a\le b)\land (b\le a)\rightarrow (a=b)\)

Orderings are transitive. That is:

\((a\le b)\land (b\le c)\rightarrow (a\le c)\)

\subsubsection{Greater than or equal}

\subsubsection{Less than and greater than}

The relation \(\le \) is refered to as non-strict.

There is a similar strict relation relation, \(<\):

\((a\le b)\land \neg (b\le a)\rightarrow (a < b)\)


\section*{Exercises}

\begin{ex}
  Let $X=\begin{mymatrix}{rrr}
    -1 & -1 & 1
  \end{mymatrix}$ and $Y=\begin{mymatrix}{rrr}
    0 & 1 & 2
  \end{mymatrix}$. Find $X^TY$ and $XY^T$ if
  possible.
  \begin{sol}
    $X^TY = \begin{mymatrix}{rrr}
      0 & -1 & -2 \\
      0 & -1 & -2 \\
      0 & 1 & 2
    \end{mymatrix}$, $XY^T = 1$.
  \end{sol}
\end{ex}

\begin{ex}
  Consider the matrices
  \begin{equation*}
    A =\begin{mymatrix}{rr}
      1 & 2 \\
      3 & 2 \\
      1 & -1
    \end{mymatrix},
    \quad
    B=\begin{mymatrix}{rrr}
      2 & -5 & 2 \\
      -3 & 2 & 1
    \end{mymatrix},
    \quad
    C =\begin{mymatrix}{rr}
      1 & 2 \\
      5 & 0
    \end{mymatrix},
    \quad
    D=\begin{mymatrix}{rr}
      -1 & 1 \\
      4 & -3
    \end{mymatrix},
    \quad
    E=\begin{mymatrix}{r}
      1 \\
      3
    \end{mymatrix}.
  \end{equation*}
  Find the following if possible. If it is not possible explain why.
  \begin{enumerate}
  \item $-3A{^T}$.
  \item $3B - A^T$.
  \item $E^TB$.
  \item $EE^T$.
  \item $B^TB$.
  \item $CA^T$.
  \item $D^TBE$.
  \end{enumerate}

  \begin{sol}
    \begin{enumerate}
    \item $\begin{mymatrix}{rrr}
        -3 & -9 & -3 \\
        -6 & -6 & 3
      \end{mymatrix}$.
    \item $\begin{mymatrix}{rrr}
        5 & -18 & 5 \\
        -11 & 4 & 4
      \end{mymatrix}$.
    \item $\begin{mymatrix}{rrr}
        -7 & 1 & 5
      \end{mymatrix}$.
    \item $\begin{mymatrix}{rr}
        1 & 3 \\
        3 & 9
      \end{mymatrix}$.
    \item $\begin{mymatrix}{rrr}
        13 & -16 & 1 \\
        -16 & 29 & -8 \\
        1 & -8 & 5
      \end{mymatrix}$.
    \item $\begin{mymatrix}{rrr}
        5 & 7 & -1 \\
        5 & 15 & 5
      \end{mymatrix}$.
    \item Not possible because $B$ is a $2\times 3$-matrix and $E$ is
      a $2\times 1$-matrix, cannot multiply $BE$.
    \end{enumerate}
  \end{sol}
\end{ex}

\begin{ex}
  Which of the following matrices are symmetric, antisymmetric, both,
  or neither?
  \begin{equation*}
    A = \begin{mymatrix}{rr}
      0 & 1 \\
      -1 & 0 \\
    \end{mymatrix},
    \quad
    B = \begin{mymatrix}{rr}
      2 & 1 \\
      1 & 3 \\
    \end{mymatrix},
    \quad
    C = \begin{mymatrix}{rr}
      1 & 2 \\
      -2 & 0 \\
    \end{mymatrix},
    \quad
    D = \begin{mymatrix}{rr}
      0 & 0 \\
      0 & 0 \\
    \end{mymatrix}.
  \end{equation*}
  \begin{sol}
    $A$ is antisymmetric, $B$ is symmetric, $C$ is neither, and $D$ is both.
  \end{sol}
\end{ex}

\begin{ex}
  Suppose $A$ is a matrix that is both symmetric and
  antisymmetric. Show that $A=0$.
  \begin{sol}
    We have $A=A^T=-A$. Therefore, each entry $a_{ij}$ of $A$ is equal
    to its own negation. This implies that $a_{ij}=0$, and therefore
    $A$ is the zero matrix.
  \end{sol}
\end{ex}

\begin{ex}
  Let $A$ be an $n\times n$-matrix. Show $A$ equals the sum of a
  symmetric and an antisymmetric matrix.  \textbf{Hint:} Show that
  $\frac{1}{2}(A^T+A)$ is symmetric and then consider using
  this as one of the matrices.
  \begin{sol}
    $A=\frac{1}{2}(A+A^T)+\frac{1}{2}(A-A^T)$.
  \end{sol}
\end{ex}

\begin{ex}
  Show that the main diagonal of every antisymmetric matrix consists
  of only zeros. Recall that the main diagonal consists of every entry
  of the matrix which is of the form $a_{ii}$.
  \begin{sol}
    If $A$ is antisymmetric then $A=-A^T$. It follows that
    $a_{ii}=-a_{ii}$ and so each $a_{ii}=0$.
  \end{sol}
\end{ex}

\begin{ex}
  Show that for $m \times n$-matrices $A,B$ and scalars $r, s$, the
  following holds:
  \begin{equation*}
    (rA + sB)^T = rA^T + sB^T.
  \end{equation*}
  \vspace{-4ex}
  \begin{sol}
    This follows from properties {\ref{matrix-transpose-2}} and
    {\ref{matrix-transpose-3}} of
    Proposition~\ref{prop:transpose-properties}.
  \end{sol}
\end{ex}

\begin{ex} \label{exer-Rn3}  Let $A$ be a real $m\times n$-matrix and
  let $\vect{u}\in \R^n$ and $\vect{v}\in \R^m$. Show
  $(A\vect{u})\dotprod \vect{v}=\vect{u}\dotprod (A^T\vect{v})$.
  \begin{sol}
    $(A\vect{u})\dotprod \vect{v} = (A\vect{u})^T\vect{v} =
    (\vect{u}^TA^T)\vect{v}  = \vect{u}^T(A^T\vect{v}) =
    \vect{u}\dotprod (A^T\vect{v})$.
  \end{sol}
\end{ex}

\begin{ex}
  Show that if $A$ is an invertible $n\times n$-matrix, then so is
  $A^T$ and $(A^T)^{-1}=(A^{-1})^T$.
  \begin{sol}
    We need to show that $(A^{-1})^T$ is the inverse of
    $A^T$. From properties of the transpose,
    \begin{eqnarray*}
      A^T(A^{-1})^T &=& (A^{-1}A)^T=I^T=I, \\
      (A^{-1})^TA^T &=& (AA^{-1})^T=I^T=I.
    \end{eqnarray*}
    Hence $A^T$ is invertible and $(A^T)^{-1}=(A^{-1})^T$.
  \end{sol}
\end{ex}

\begin{ex}
  Suppose $A$ is invertible and symmetric. Show that $A^{-1}$ is
  symmetric.
  \begin{sol}
    We have $(A^{-1})^T = (A^T)^{-1} = A^{-1}$, and therefore $A^{-1}$
    is equal to its own transpose, hence symmetric.
  \end{sol}
\end{ex}


\chapter{From Abstract Algebra to Finite Fields}
\label{app:A}
% handbook of applied cryptography

In this Appendix we recall basic formal definitions from \cite{Menezes97handbookof} to properly understand what a finite field is.

\section{Groups}
\begin{mydef} A binary operation $*$ on a set $S$ is a mapping  $S \times S$ to $S$. That is, $*$ is a rule which assigns to each ordered pair of elements from $S$ an element of $S$. \end{mydef}

\begin{mydef}
	A group $(G, *)$ consists of a set $G$ with a binary operation $*$ on $G$ satisfying
	the following three axioms.
	\begin{itemize}
		\item[(i)] The group operation is associative. That is, $a* (b * c) = (a* b) * c \quad \forall a, b, c \in G$.
		\item[(ii)] There is an element $1 \in G$, called the identity element, such that $a * 1 = 1 * a = a \quad \forall a \in G$.
		\item[(iii)] For each $a \in G $ there exists an element $a^{-1} \in G$, called the inverse of $a$, such that $a *a^{-1}= a^{-1} * a = 1$.
	\end{itemize}
	A group G is Abelian (or commutative) if, furthermore,
	\begin{itemize}
		\item[(iv)] $a * b = b * a \quad \forall a, b \in G$.
	\end{itemize}
\end{mydef}

\begin{mydef}
	A group $G$ is finite if $|G|$ is finite. The number of elements in a finite group is called its order.
\end{mydef}

\begin{myexample}
	The set of integers $\mathbb{Z}$ with the operation of addition forms a group.
	The identity element is 0 and the inverse of an integer $a$ is the integer $-a$.
\end{myexample}

\begin{myexample}
	The set $\mathbb{Z}_n=\{a \in \mathbb{Z} | 0 \leq a \leq n - 1\}$, with the operation of addition modulo $n$, forms a group of order $n$.
	The set $\mathbb{Z}_n$ with the operation of multiplication modulo $n$ is not a group, since not all elements have multiplicative inverses.
\end{myexample}

\begin{mydef}
	The multiplicative group of $\mathbb{Z}_n$ is $\mathbb{Z}_n^* = \{a \in \mathbb{Z}_n | gcd(a, n) = 1\}$.
	In particular, if n is a prime, then $\mathbb{Z}_n^* = \{a | 1 \leq a \leq n - 1\}$.
\end{mydef}

\begin{myexample}
	The set $\mathbb{Z}_n^*$ is a group under the operation of multiplication modulo $n$, with identity element 1.\end{myexample}

\begin{mydef}
	A non-empty subset $H$ of a group $G$ is a subgroup of $G$ if $H$ is itself a group
	with respect to the operation of $G$. If $H$ is a subgroup of $G$ and $H \neq G$, then $H$ is called a proper subgroup of $G$.
\end{mydef}

\begin{mydef}
	A group $G$ is cyclic if there is an element $\alpha \in G $such that for each $b \in G$ there is an integer $i$ with $b= \alpha^i$. Such an element $\alpha$ is called a generator of $G$.
\end{mydef}

\section{Rings}

\begin{mydef}
	A ring $(R,+,\times)$ consists of a set $R$ with two binary operations arbitrarily denoted $+$ (addition) and $\times$ (multiplication) on $R$, satisfying the following axioms.
	\begin{itemize}
		\item[(i)] $(R, +)$ is an abelian group with identity denoted $0$.
		\item[(ii)] The operation $\times$ is associative. That is, $a \times (b \times c) = (a \times b)\times c \quad \forall a, b, c \in R$.
		\item[(iii)] There is a multiplicative identity denoted $1$, with $1 \neq 0$, such that $1 \times a = a \times 1 = a \forall a \in R.$.
		\item[(iv)] The operation $\times$ is distributive over $+$. That is, $a \times (b+c) = (a \times b)+(a \times c) \textmd{ and } (b + c) \times a = (b \times a) + (c \times a) \quad \forall a, b, c \in R.$

	\end{itemize}
	The ring is a commutative ring if $a \times b = b \times a \quad \forall a, b \in R$.
\end{mydef}

\begin{myexample}
	The set $\mathbb{Z}_n$ with addition and multiplication performed modulo $n$ is a commutative ring.
\end{myexample}

\begin{mydef}
	An element $a$ of a ring $R$ is called a unit or an invertible element if there is an element $b \in R $ such that $a \times b = 1$.
	The set of units in a ring $R$ forms a group under multiplication, called the group of units of $R$.
\end{mydef}

\begin{myexample}
	The group of units of the ring $\mathbb{Z}_n$ is $\mathbb{Z}_n^*$.
\end{myexample}

\section{Fields}

\begin{mydef}
	A field is a commutative ring in which all non-zero elements have multiplicative inverses.
\end{mydef}

\begin{mydef}
	The characteristic of a field is 0 if $\overbrace{1+1+\dots+1}^{m times}$ is never equal to $0$ for any $m \geq 1$. Otherwise, the characteristic of the field is the least positive integer $m$ such that $\sum_{i=1}^{m}1 = 0$.
\end{mydef}

\begin{myexample}
the rational numbers $\mathbb{Q}$, the real numbers $\mathbb{R}$ and the complex numbers $\mathbb{C}$ form fields of characteristic $0$ under the usual operations.
\end{myexample}

\begin{myexample}
	$\mathbb{Z}_n$ is a field (under the usual operations of addition and multiplication modulo $n$) if and only if $n$ is a prime number. If $n$ is prime, then $\mathbb{Z}_n$ has characteristic $n$.
\end{myexample}

\section{Finite Fields}

\begin{mydef}
	A finite field is a field $F$ which contains a finite number of elements. The order of $F$ is the number of elements in $F$. \end{mydef}

\begin{myprop}
	(existence and uniqueness of finite fields)
	\begin{itemize}
		\item[(i)] If $F$ is a finite field, then $F$ contains $p^m$ elements for some prime $p$ and integer $m \geq 1$.
		\item[(ii)] For every prime power order $p^m$, there is a unique (up to isomorphism) finite field of order $p^m$. This field is denoted by $\mathbb{F}_{p^m}$, or sometimes by $GF(p^m)$\footnote{Galois Field}.
	\end{itemize}
	Informally speaking, two fields are isomorphic if they are structurally the same, although the representation of their field elements may be different. Note that if $p$ is a prime then $\mathbb{Z}_p$ is a field, and hence every field of order $p$ is isomorphic to $\mathbb{Z}_p$. Thus the finite field $\mathbb{F}_p$ can be identified with $\mathbb{Z}_p$.
\end{myprop}


\subsection{Linear algebra}

The $dot$ function is used to multiply tensors.
For example, let
\begin{equation*}
A=\begin{pmatrix}1&2\\3&4\end{pmatrix}
\quad{\rm and}\quad x=\begin{pmatrix}x_1\\x_2\end{pmatrix}
\end{equation*}

\noindent
The product $Ax$ is computed as follows.

{\color{blue}
\begin{verbatim}
A = ((1,2),(3,4))
x = (x1,x2)
dot(A,x)
\end{verbatim}
}

\noindent
$\begin{bmatrix}
x_1+2x_2\\
3x_1+4x_2
\end{bmatrix}$

\bigskip
\noindent
The following example shows how to use $dot$ and $inv$ to solve for
the vector $X$ in $AX=B$.

{\color{blue}
\begin{verbatim}
A = ((3,7),(1,-9))
B = (16,-22)
X = dot(inv(A),B)
X
\end{verbatim}
}

\noindent
$X=\begin{bmatrix}-\tfrac{5}{17}\\ \\ \tfrac{41}{17}\end{bmatrix}$

\bigskip
\noindent
The $dot$ function can have more than two arguments.
For example, $dot(A,B,C)$ can be used for the dot product of three tensors.

\bigskip
\noindent
Square brackets are used for component access.
Index numbering starts with 1.

{\color{blue}
\begin{verbatim}
A = ((a,b),(c,d))
A[1,2] = -A[1,1]
A
\end{verbatim}
}

\noindent
$\begin{bmatrix}a&-a\\c&d\end{bmatrix}$

\bigskip
\noindent
The following example demonstrates the relation
$A^{-1}=\frac{\operatorname{adj}A}{\operatorname{det}A}$.

{\color{blue}
\begin{verbatim}
A = ((a,b),(c,d))
inv(A)
\end{verbatim}
}

\noindent
$\begin{bmatrix}\frac{d}{ad-bc} & -\frac{b}{ad-bc}\\-\frac{c}{ad-bc} & \frac{a}{ad-bc}\end{bmatrix}$

{\color{blue}
\begin{verbatim}
adj(A)
\end{verbatim}
}

\noindent
$\begin{bmatrix}d & -b\\-c & a\end{bmatrix}$

{\color{blue}
\begin{verbatim}
det(A)
\end{verbatim}
}

\noindent
$ad-bc$

{\color{blue}
\begin{verbatim}
inv(A) - adj(A)/det(A)
\end{verbatim}
}

\noindent
$\begin{bmatrix}0 & 0\\0 & 0\end{bmatrix}$

\bigskip
\noindent
Sometimes a calculation will be simpler if it can be reorganized to use
$adj$ instead of $inv$.
The main idea is to try to prevent the determinant from appearing as a
divisor.
For example, suppose for matrices $A$ and $B$ you want to check that
\begin{equation*}
{A}-{B}^{-1}=0
\end{equation*}
Depending on the complexity of $\mathop{\rm det}B$, the software
may not be able to find a simplification that yields zero.
Should that occur, the following alternative formulation can be tried.
\begin{equation*}
A\operatorname{det}B-\operatorname{adj}B=0
\end{equation*}