% section6
\chapter{Linear Transformations}

\section{Matrices as Transformations}

\begin{exer} (\textit{Linear Transformation: Rotation})\\
Make a function file with a function name \textit{reflect\_pt} to find the reflection of the point $(a,b)$ about the line through the origin of the $xy$-plane that makes an angle of $\theta\,^{\circ}$ with the positive $x$-axis. Make $a$, $b$, and $\theta$ the inputs to the function and the reflection point $(x,y)$ the output. Using this function, find the result of this function for $(a,b)=(1,3)$ and $\theta=12$ by the following commands:

\vspace{2mm}

\begin{verbatim}
>> [x, y]=reflect_pt(1, 3, 12)
\end{verbatim}

\end{exer}

\begin{sol}
\begin{verbatim}

%--- The following is the function file 'reflect_pt.m'. ---%
function [x, y]=reflect_pt(a, b, ang)
  theta=ang*(pi/180);
  T = [cos(2*theta) sin(2*theta); sin(2*theta) -cos(2*theta)];

  result=T*[a b]';
  x=result(1); y=result(2);
end
\end{verbatim}

\begin{outputs}

\begin{verbatim}

x =
    2.1338

y =
   -2.3339
\end{verbatim}
\end{outputs}
\end{sol}

\vspace{3mm}

\begin{exer} (\textit{Linear Transformation: Projection})\\
Consider successive rotations of $\mathbb{R}^{3}$ by an angle $\theta_{1}$ degree about the $x$-axis, then by an angle $\theta_{2}$ degree about the $y$-axis, and then by an angle $\theta_{3}$ degree about the $z$-axis. Make a function file named \textit{comp\_Rot} to find an appropriate axis and angle of rotation that achieves the same result in one rotation. Make $\theta_{1}, \theta_{2},$ and $\theta_{3}$ degrees the inputs to the function and the axis and angle of rotation the outputs. Give the output axis as a unit vector. You may make the standard matrix for the composition of the rotations by multiplication of the standard matrices for the rotations about the position $x$-, $y$-, and $z$-axes, respectively. Using the function \textit{comp\_Rot}, check the result of the following commands:

\vspace{2mm}

\begin{verbatim}
>> [L ang]=comp_Rot(45, 45, 45)
\end{verbatim}

\end{exer}

\begin{sol}

\begin{verbatim}

%--- The following is the function file 'comp_Rot.m'. ---%
function [L, ang] = comp_Rot(ang1, ang2, ang3)

  % Angle as a degree.
  angle=[ang1 ang2 ang3];

  % Convert angles from degrees to radians.
  theta=angle*(pi/180);

  % Rotation about x-axis, y-axis, and z-axis.
  Rx = [1 0 0;
        0 cos(theta(1)) -sin(theta(1));
        0 sin(theta(1)) cos(theta(1))];
  Ry = [cos(theta(2)) 0 sin(theta(2));
        0 1 0;
        -sin(theta(2)) 0 cos(theta(2))];
  Rz = [cos(theta(3)) -sin(theta(3)) 0;
        sin(theta(3)) cos(theta(3)) 0;
        0 0 1];

  % Composition of the three rotation matrices R = Ry*Rx*Rz.
  R = Rz*Ry*Rx;

  % Find the axis of rotation of R,
  % by finding the eigenvector of R
  % corresponding to the eigenvalue lambda=1.

  % Find a nonzero vector satisfying Rx = x.
  L = null(eye(3) - R);

  % Make the axis of rotation unit vector.
  L = L/norm(L);

  % Find the angle of rotation of R.
  % Note that w=(-x2,x1,0) is
  % orthogonal to the axis of rotation x=(x1,x2,x3).
  w = [-L(2) L(1) 0]';
  rot_theta = acos((dot(w, R*w))/((norm(w)*norm(R*w))));
  ang = ((rot_theta)*(180/pi));
end
\end{verbatim}

\begin{outputs}

\begin{verbatim}

L =
   -0.3574
   -0.8629
   -0.3574

ang =
   64.7368
\end{verbatim}
\end{outputs}

\end{sol}
\newpage

\section{Geometry of Linear Operators}

\begin{exer} (\textit{Rotation as an Orthogonal Transformation})\\
Let
\begin{displaymath}
A = \left[\begin{array}{rrrr} \displaystyle -\frac{3}{7}&\quad \displaystyle -\frac{2}{7}&\quad \displaystyle -\frac{6}{7} \vspace{3mm} \\ \vspace{3mm} \displaystyle \frac{6}{7} & \displaystyle -\frac{3}{7} & \displaystyle -\frac{2}{7}\\ \displaystyle -\frac{2}{7} & \displaystyle -\frac{6}{7} & \displaystyle \frac{3}{7} \end{array} \right].
\end{displaymath}

\vspace{2mm}
\noindent Show that $A$ represents a rotation, and use Formulas $(16)$ and $(17)$ in Section $6.2$ to find the axis and angle of rotation.
\end{exer}

\begin{sol}
\begin{verbatim}

A = [-3/7 -2/7 -6/7; 6/7 -3/7 -2/7; -2/7 -6/7 3/7]; format short;

% Check that A*A' is the identity matrix.
A*A'

% Although the off diagonal entries are not exactly all zeros,
% the scaling factor suggests that roundoff error prevents
% the computed matrix from being the identity matrix.
% You can see that the product is exactly the identity matrix,
% when the symbolic computation is used.

% Check that det(A)=1 to conclude that A is orthogonal.
det(A)

% Since A*A'=I and det(A) = 1, A represents a rotation.

% By (16) of Section 6.2,
% find the angle of rotation
% and convert the angle from radians to degrees.
theta = acos((trace(A)-1)/2); ang = ((theta)*(180/pi));
disp('The angle of rotation in degrees is'); disp(ang);

% By (17) of Section 6.2, find the axis of rotation.
% The initial point at the origin.
e1 = [1 0 0]';

% v is along the axis of rotation.
v = (A+A'+((1-trace(A))*eye(3))) * e1;
disp('The axis of rotation passes through the point'); disp(v');
\end{verbatim}

\begin{outputs}

\begin{verbatim}

The angle of rotation in degrees is
  135.5847
The axis of rotation passes through the point
    0.5714    0.5714   -1.1429
\end{verbatim}
\end{outputs}
\end{sol}

\section{Kernel and Range}

\begin{exer}
(\textit{Invertible Matrix as a Bijective Linear Transformation})\\
Consider the matrix
\begin{displaymath}
A = \left[\begin{array}{rrrr} 3& \hspace{1mm} -5& \hspace{1mm} -2&\hspace{3mm} 2\\ -4 & 7 & 4 & 4 \\ 4 & -9 & -3 & 7 \\ 2 & -6 & -3 & 2 \end{array} \right].
\end{displaymath}

\vspace{2mm}
Referring to the Theorem $6.3.15$ in Section $6.3$, show that $T_{A} : \mathbb{R}^{4} \rightarrow \mathbb{R}^{4}$ is onto in at least four different ways. You may use several related MATLAB commands.

\end{exer}

\begin{sol}
\begin{verbatim}

A = [3 -5 -2 2; -4 7 4 4; 4 -9 -3 7; 2 -6 -3 2];
format short;

% By (a) in Theorem 6.3.15, use the command rref of A.
rref(A)

% Since the reduced row echelon form of A is the identity matrix,
% the linear transformation T is onto.

% By (d) in Theorem 6.3.15, use the command null of A.

% Find a basis for the null space of A.
null(A, 'r')

% Since the null space contains only the zero vector,
% the linear transformation T is onto.

% By (g) in Theorem 6.3.15, use the command rank of A.

% Find the number of linearly independent columns of A.
rank(A)

% Since the column vectors of A are linearly independent,
% the linear transformation T is onto.

% By (i) in Theorem 6.3.15, use the command det of A.

% Find the determinant of A.
det(A)

% Since det(A) is nonzero,
% the linear transformation T is onto.

% By (j) in Theorem 6.3.15, use the command eig of A.

% Find the eigenvalues A.
eig(A)

% Since 0 is not an eigenvalue of A,
% the linear transformation T is onto.
\end{verbatim}
\end{sol}

\section{Composition and Invertibility of Linear Transformations}

\begin{exer}(\textit{Compositions of Linear Transformations})\\
Consider successive rotations of $\mathbb{R}^{3}$ by $30\,^{\circ}$ about the $z$-axis, then by $60\,^{\circ}$ about the $x$-axis, and then by $45\,^{\circ}$ about the $y$-axis. If it is desired to execute the three rotations by a single rotation about an appropriate axis, what axis and angle should be used?

\end{exer}

\begin{sol}
\begin{verbatim}

% Angle as a degree.
ang1=30; ang2=60; ang3=45;

% Convert angles from degrees to radians.
theta1=((ang1)*(pi/180));
theta2=((ang2)*(pi/180));
theta3=((ang3)*(pi/180));

format short;

% Rotation about z-axis with the angle 30.
Rz = [cos(theta1) -sin(theta1) 0; sin(theta1) cos(theta1) 0; 0 0 1];
% Rotation about x-axis with the angle 60.
Rx = [1 0 0; 0 cos(theta2) -sin(theta2); 0 sin(theta2) cos(theta2)];
% Rotation about y-axis with the angle 45.
Ry = [cos(theta3) 0 sin(theta3); 0 1 0; -sin(theta3) 0 cos(theta3)];

% Composition of the three rotation matrices R = Ry*Rx*Rz.
R = Ry*Rx*Rz;

% Find the axis of rotation of R,
% by finding the eigenvector of R corresponding to the eigenvalue lambda=1.

% Find a nonzero vector satisfying Rx = x.
x = null(eye(3) - R);

% Find the angle of rotation of R.
% Note that w=(-x2,x1,0) is orthogonal to the axis of rotation x=(x1,x2,x3).
w = [-x(2) x(1) 0]';
theta = acos((dot(w, R*w))/((norm(w)*norm(R*w))));

% Convert the angle from radians to degrees.
ang = ((theta)*(180/pi));

disp('The angle of rotation in degrees is'); disp(ang);
disp('The axis of rotation is'); disp(x');
\end{verbatim}

\begin{outputs}

\begin{verbatim}

The angle of rotation in degrees is
   69.3559

The axis of rotation is
   -0.9350   -0.3525   -0.0391
\end{verbatim}

\end{outputs}
\end{sol}

_codes/intro/Learning MATLAB with Linear Algebra (Jeon, Lee)/section6.tex

files/intro/Learning MATLAB with Linear Algebra (Jeon, Lee)/section6.tex


\subsection{FEM ans ${\rm DNN}_1$ in 1D}
Thanks to following connection between $\varphi(x)$ in \eqref{def_g} and ${\rm ReLU}(x) = \max(0,x )=x_+$
\begin{equation}\label{key}
\varphi(x) = 2{\rm ReLU}(x) - 4{\rm ReLu}({x-\frac{1}{2}}) + 2{\rm ReLU}(x-1),
\end{equation}
it suffices to show that each basis
function $\varphi_{\ell,i}$ can be represented by a ReLU DNN.
We first note that  the basis
function $\varphi_{\ell,i}$ has the support in $[x_{\ell,i-1},
x_{\ell,i+1} ]$ can be easily written as
\begin{equation}
\label{1d-basisu}
\varphi_{\ell,i}(x) = \frac{1}{h_{\ell}}{\rm ReLU}(x-x_{\ell,i-1}) -\frac{2}{h_{\ell}}{\rm ReLU}(x-x_{\ell,i}) +\frac{1}{h_\ell}{\rm ReLU}(x-x_{\ell,i+1}).
\end{equation}
More generally, consider a general  grid with vertex $\{x_i\}$, which is not necessarily uniform. The basis function $\varphi_i$ of the linear element with support $[x_{i-1},
x_{i+1} ]$ can be easily written as
\begin{equation}
\label{1d-basis}
\varphi_i(x) = \frac{1}{h_{i-1}}{\rm ReLU}(x-x_{i-1}) -(\frac{1}{h_{i-1}}+\frac{1}{h_i}){\rm ReLU}(x-x_i) +\frac{1}{h_i}{\rm ReLU}(x-x_{i+1}),
\end{equation}
where $h_i = x_{i+1} - x_i$.

Thus is to say, we have the next theorem.
\begin{theorem}\label{thm:1dLFEMDNN}
	For $d=1$, and  $\Omega\subset \mathbb R^d$ is
	a bounded interval, then ${\rm DNN}_1$ can be used to cover all linear finite element
	function in on $\Omega$.
\end{theorem}


\subsection{Properties of matrix multiplication}

We have already seen that matrix multiplication is not in general
commutative, i.e., $AB$ and $BA$ may be different, even if they are
both defined. Sometimes it can happen that $AB=BA$ for specific
matrices $A$ and $B$. In this case, we say that $A$ and $B$
\textbf{commute}%
\index{matrix!commuting matrices}%
\index{commuting matrices}.

The following are some properties of matrix multiplication.  Notice
that these properties hold only when the size of matrices are such
that the products are defined.

\begin{proposition}{Properties of matrix multiplication}{properties-of-matrix-multiplication}
  The following properties hold%
  \index{matrix!properties of multiplication}%
  \index{matrix!multiplication!properties}%
  \index{properties of multiplication!matrices} for matrices $A,B,C$
  of appropriate dimensions and for scalars $r$.
  \begin{itemize}
  \item The associative law of multiplication
    \begin{equation*}
      (AB)C ~=~ A(BC).
    \end{equation*}
  \item The existence of multiplicative units
    \begin{equation*}
      I_mA ~=~ A ~=~ AI_n,
    \end{equation*}
    where $A$ is an $m\times n$-matrix.
  \item Compatibility with scalar multiplication
    \begin{equation*}
      (rA)B ~=~ r(AB) ~=~ A(rB).
    \end{equation*}
  \item The distributive laws of multiplication over addition
    \begin{eqnarray*}
      A(B+C) &=& AB + AC, \\
      (B+C)A &=& BA + CA.
    \end{eqnarray*}
  \end{itemize}
\end{proposition}

\begin{proof}
  First, we will prove the associative law. In the proof, it will be
  useful to use {\em summation notation}%
  \index{summation notation}.  We write
  \begin{equation*}
    \sum_{i=1}^n x_i ~=~ x_1 + x_2 + \ldots + x_n
  \end{equation*}
  for the sum of the $n$ numbers $x_1,\ldots,x_n$. Assume $A$ is an
  $m\times n$-matrix, $B$ is an $n\times p$-matrix, and $C$ is a
  $p\times q$-matrix. Then both $(AB)C$ and $A(BC)$ are
  $n\times q$-matrices. We must show that they have the same
  entries. The $(i,\ell)$-entry of the matrix $(AB)C$ is
  \begin{equation*}
    ((AB)C)_{i\ell} = \sum_{k=1}^p (AB)_{ik}c_{k\ell}
    = \sum_{k=1}^p (\sum_{j=1}^n a_{ij}b_{jk})c_{k\ell}
    = \sum_{k=1}^p\sum_{j=1}^n a_{ij}b_{jk}c_{k\ell}.
  \end{equation*}
  On the other hand, the $(i,\ell)$-entry of the matrix $A(BC)$ is
  \begin{equation*}
    (A(BC))_{i\ell} = \sum_{j=1}^n a_{ij}(BC)_{j\ell}
    = \sum_{j=1}^n a_{ij} (\sum_{k=1}^p b_{jk} c_{k\ell})
    = \sum_{j=1}^n\sum_{k=1}^p a_{ij} b_{jk} c_{k\ell}.
  \end{equation*}
  Both sums are equal, since they are both summing over all the terms
  where $j=1,\ldots,n$ and $k=1,\ldots,p$. Therefore, $(AB)C=A(BC)$.
  The fact that identity matrices act as multiplicative units was
  already mentioned in Proposition~\ref{prop:identity-matrix}.  We
  leave compatibility with scalar multiplication as an exercise.
  To prove the first distributive law, assume $A$ is an
  $m\times n$-matrix, and $B$ and $C$ are $n\times p$-matrices. Then
  both $A(B+C)$ and $AB+AC$ are $m\times p$-matrices. We have
  \begin{equation*}
    (A(B+C))_{ik} = \sum_{j=1}^n a_{ij}(B+C)_{jk}
    = \sum_{j=1}^n a_{ij}(b_{jk}+c_{jk})
    = \sum_{j=1}^n a_{ij}b_{jk} + \sum_{j=1}^n a_{ij}c_{jk}
    = (AB+AC)_{ik}.
  \end{equation*}
  Thus $A(B+C) =AB+AC$ as claimed. The proof of the other distributive
  law is similar.
\end{proof}


%
% Chapter 1.7
%

\section*{1.7 Definition of a Limit}

\subsection*{Formal Definition of a Limit}

Let \(f\) be a function defined on some open interval that contains the number \(a\), except possibly \(a\) itself. Then we say that the limit of \(f(x)\) as \(x\) approaches \(a\) is \(L\) and we write
$$\lim_{x \to a}f(x)=L$$
if for every number \(\epsilon > 0\) there is a number \(\delta > 0\) such that
$$\text{if} \quad 0< \left| x-a \right| <\delta \quad \text{then} \quad \left| f(x)-L \right| <\epsilon$$

\subsection*{Formal Definition of a Left-Hand Limit}

$$\lim_{x \to a^-}f(x)=L$$
if for every number \(\epsilon > 0\) there is a number \(\delta > 0\) such that
$$\text{if} \quad a-\delta<x<a \quad \text{then} \quad \left| f(x)-L \right| <\epsilon$$

\subsection*{Formal Definition of a Right-Hand Limit}

$$\lim_{x \to a^+}f(x)=L$$
if for every number \(\epsilon > 0\) there is a number \(\delta > 0\) such that
$$\text{if} \quad a<x<a+\delta \quad \text{then} \quad \left| f(x)-L \right| <\epsilon$$

\subsection*{Formal Definition of an Infinite Limit}

Let \(f\) be a function defined on some open interval that contains the number \(a\), except \(a\) itself. Then
$$\lim_{x \to a}f(x)=\infty$$

means that for every positive number \(M\) there is a positive number \(\delta\) such that
$$\text{if} \quad 0< \left| x-a \right| <\delta \quad \text{then} \quad f(x)>M$$

\subsection*{Formal Definition of a Negative Infinite Limit}

Let \(f\) be a function defined on some open interval that contains the number \(a\), except \(a\) itself. Then
$$\lim_{x \to a}f(x)=-\infty$$
means that for every negative number \(N\) there is a positive number \(\delta\) such that
$$\text{if} \quad 0< \left| x-a \right| <\delta \quad \text{then} \quad f(x)>N$$

\subsection*{Famous Trigonometric Limits}

See Chapter 2.4


%!TEX root =  ../main.tex

\subsection{Restrained Growth}

\objective{Create and use logistic curves}

Exponential growth can rarely be carried on for long.  Bacteria fill the petri dish,
bunnies eat all the grass, and the shower-water has a maximum heat it can
reach.  In every case, the is a constraint, a ceiling value that the population
cannot grow beyond.  Typically, it is approached asymptotically.  This means there are
two horizontal asymptotes.  The left half of the graph is concave up (exponential
growth) and the right half is concave down (logarithmic growth).

How can these graphical quantities be achieved algebraically?  What can
be done to $y=e^x$ to make it have a horizontal asymptote?  As we saw last
chapter, rational functions have horizontal asymptotes at the level of the ratio
of the numerator to the denominator ``at infinity''.  So we need to create a
denominator.  But we don't want the denominator to ever equal 0, lest we
create a vertical asymptote.  Dividing by $e^x$ is almost right.  Let's make it
$e^x+1$.

$$
f(x) = \frac{e^x}{e^x+1}
$$

\begin{figure}[h]
\begin{centering}
\includegraphics[scale=0.5]{\chapdir/pics/basicalogistic}
\caption{A basic logistic curve}
\end{centering}
\end{figure}

That worked out nicely, but it made our equation slightly more complicated
than we might otherwise like.  If we want to do any inside transformation (e.g.
shift left/right, widen/narrow) then we will need to modify $x$ in two places.  Can
we consolidate the $x$ to be in only one place?  What could we do to the
numerator and denominator ?

Dividing top and bottom by $e^x$ will cancel the two terms with $x$ in them.
Furthermore, we can eliminate the need for complex fraction by changing
a denominator under 1 to be a negative exponent.

$$
f(x) = \frac{1}{1+e^{-x}}
$$

Notice which numbers control the behavior of the graph.  The 1 in the numerator
is the maximum height of the graph, which typically called $c$ for ceiling. If we
insert a multiplying constant next to $x$ in the exponent, it will contract the
graph in the $x$-direction, making the curve of the S steeper.  If we insert
a multiplying constant next to x, it will shift the turning point to the right.  No
matter what we do, the inflection point will occur half-way up.

$$
f(x) = \frac{c}{1+ae^{-bx}}
$$

\subsection{Regression}
The TI-8* has a LOGISTICREG function which is excellent for computing
regressions of this kind, and outputs functions of the kind already discussed.
However, if we need to make an equation ourselves by hand, $e$ is slightly
awkward number to use.  Algebra manipulation is much easier with this
equation:

$$
f(x) = \frac{c}{1+ab^{-x}}
$$


\section{Function Reference}

\subsection*{abs($x$)}

Returns the absolute value or vector length of $x$.

{\color{blue}
\begin{verbatim}
X = (x,y,z)
abs(X)
\end{verbatim}
}

\noindent
$\left(x^2+y^2+z^2\right)^{1/2}$

\subsection*{adj($m$)}

Returns the adjunct of matrix $m$.
Adjunct is equal to determinant times inverse.

{\color{blue}
\begin{verbatim}
A = ((a,b),(c,d))
adj(A) == det(A) inv(A)
\end{verbatim}
}

\noindent
$1$

\subsection*{and($a,b,\ldots$)}

Returns 1 if all arguments are true (nonzero).
Returns 0 otherwise.

{\color{blue}
\begin{verbatim}
and(1=1,2=2)
\end{verbatim}
}

\noindent
$1$

\subsection*{arccos($x$)}

Returns the arc cosine of $x$.

{\color{blue}
\begin{verbatim}
arccos(1/2)
\end{verbatim}
}

\noindent
$\tfrac{1}{3}\pi$

\subsection*{arccosh($x$)}

Returns the arc hyperbolic cosine of $x$.

\subsection*{arcsin($x$)}

Returns the arc sine of $x$.

{\color{blue}
\begin{verbatim}
arcsin(1/2)
\end{verbatim}
}

\noindent
$\tfrac{1}{6}\pi$

\subsection*{arcsinh($x$)}

Returns the arc hyperbolic sine of $x$.

\subsection*{arctan($y,x$)}

Returns the arc tangent of $y$ over $x$.
If $x$ is omitted then $x=1$ is used.

{\color{blue}
\begin{verbatim}
arctan(1,0)
\end{verbatim}
}

\noindent
$\tfrac{1}{2}\pi$

\subsection*{arctanh($x$)}

Returns the arc hyperbolic tangent of $x$.

\subsection*{arg($z$)}

Returns the angle of complex $z$.

{\color{blue}
\begin{verbatim}
arg(2 - 3i)
\end{verbatim}
}

\noindent
$\arctan(-3,2)$

\subsection*{besselj($x,n$)}

Returns a solution to the Bessel differential equation.

{\color{blue}
\begin{verbatim}
besselj(x,1/2)
\end{verbatim}
}

\noindent
$\displaystyle \frac{2^{1/2}\sin(x)}{\pi^{1/2}\,x^{1/2}}$

\subsection*{binding($s$)}

The result of evaluating a symbol can differ from the symbol's binding.
For example, the result may be expanded.
The {\tt binding} function returns the actual binding of a symbol.

{\color{blue}
\begin{verbatim}
p = quote((x + 1)^2)
p
\end{verbatim}
}

\noindent
$p=x^2+2x+1$

{\color{blue}
\begin{verbatim}
binding(p)
\end{verbatim}
}

\noindent
$(x+1)^2$

\subsection*{binomial($n,k$)}

Returns the coefficient of $x^ky^{n-k}$ in $(x+y)^n$.

{\color{blue}
\begin{verbatim}
binomial(52,5)
\end{verbatim}
}

\noindent
$2598960$

\bigskip
\noindent
Note:
\verb$binomial$
and
\verb$choose$
are the same function.

\subsection*{ceiling($x$)}

Returns the smallest integer greater than or equal to $x$.

{\color{blue}
\begin{verbatim}
ceiling(1/2)
\end{verbatim}
}

\noindent
$1$

\subsection*{check($x$)}

If $x$ is true (nonzero) then continue, else stop.
Expression $x$ can include the relational operators
\verb$=$,
\verb$==$,
\verb$<$,
\verb$<=$,
\verb$>$,
\verb$>=$.
Use the
\verb$not$
function to test for inequality.

{\color{blue}
\begin{verbatim}
A = 1
B = 1
check(A=B) -- stop here if A not equal to B
\end{verbatim}
}

\subsection*{choose($n,k$)}

Returns the number of combinations of $n$ items taken $k$ at a time.
The following example computes the number of poker hands.

{\color{blue}
\begin{verbatim}
choose(52,5)
\end{verbatim}
}

\noindent
$2598960$

\subsection*{circexp($x$)}

Returns expression $x$ with circular and hyperbolic functions
converted to exponentials.

{\color{blue}
\begin{verbatim}
circexp(cos(x) + i sin(x))
\end{verbatim}
}

\noindent
$\exp(ix)$

\subsection*{clear}

Clears all symbol definitions.

\subsection*{clock($z$)}

Returns complex $z$ in polar form with base of negative 1 instead of $e$.

{\color{blue}
\begin{verbatim}
clock(2 - 3i)
\end{verbatim}
}

\noindent
$13^{1/2}\,(-1)^{\arctan(-3,2)/\pi}$

\subsection*{coeff($p,x,n$)}

Returns the coefficient of $x^n$ in polynomial $p$.

{\color{blue}
\begin{verbatim}
p = x^3 + 6x^2 + 12x + 8
coeff(p,x,2)
\end{verbatim}
}

\noindent
$6$

\subsection*{cofactor($m,i,j$)}

Returns a cofactor of matrix $m$.
The cofactor matrix is the transpose of the adjunct of $m$.
This function returns the cofactor component at row $i$ and column $j$.

{\color{blue}
\begin{verbatim}
A = ((a,b),(c,d))
cofactor(A,1,2) == transpose(adj(A))[1,2]
\end{verbatim}
}

\noindent
$1$

\subsection*{conj($z$)}

Returns the complex conjugate of $z$.

{\color{blue}
\begin{verbatim}
conj(2 - 3i)
\end{verbatim}
}

\noindent
$2 + 3 i$

\subsection*{contract($a,i,j$)}

Returns tensor $a$ summed over indices $i$ and $j$.
If $i$ and $j$ are omitted then 1 and 2 are used.
The expression {\tt contract(m)} computes the trace of matrix $m$.

{\color{blue}
\begin{verbatim}
A = ((a,b),(c,d))
contract(A)
\end{verbatim}
}

\noindent
$a + d$

\subsection*{cos($x$)}

Returns the cosine of $x$.

{\color{blue}
\begin{verbatim}
cos(pi/4)
\end{verbatim}
}

\noindent
$\displaystyle \frac{1}{2^{1/2}}$

\subsection*{cosh($x$)}

Returns the hyperbolic cosine of $x$.

{\color{blue}
\begin{verbatim}
circexp(cosh(x))
\end{verbatim}
}

\noindent
$\tfrac{1}{2}\exp(-x)+\tfrac{1}{2}\exp(x)$

\subsection*{cross($u,v$)}

Returns the cross product of vectors $u$ and $v$.
It is OK to redefine {\tt cross}.
This is the default definition.

{\color{blue}
\begin{verbatim}
cross(u,v) = (u[2] v[3] - u[3] v[2],
              u[3] v[1] - u[1] v[3],
              u[1] v[2] - u[2] v[1])
\end{verbatim}
}

\subsection*{curl($u$)}

Returns the curl of vector $u$.
It is OK to redefine {\tt curl}.
This is the default definition.

{\color{blue}
\begin{verbatim}
curl(u) = (d(u[3],y) - d(u[2],z),
           d(u[1],z) - d(u[3],x),
           d(u[2],x) - d(u[1],y))
\end{verbatim}
}

\subsection*{d($f,x$)}

Returns the partial derivative of $f$ with respect to $x$.

{\color{blue}
\begin{verbatim}
d(x^2,x)
\end{verbatim}
}

\noindent
$2x$

\bigskip
\noindent
Argument $f$ can be a tensor of any rank.
Argument $x$ can be a vector.
When $x$ is a vector the result is the gradient of $f$.

{\color{blue}
\begin{verbatim}
F = (f(),g(),h())
X = (x,y,z)
d(F,X)
\end{verbatim}
}

\noindent
$\displaystyle \begin{bmatrix}
\operatorname{d}(f(),x) & \operatorname{d}(f(),y) &  \operatorname{d}(f(),z)\\
\operatorname{d}(g(),x) & \operatorname{d}(g(),y) &  \operatorname{d}(g(),z)\\
\operatorname{d}(h(),x) & \operatorname{d}(h(),y) &  \operatorname{d}(h(),z)
\end{bmatrix}
$

\bigskip
\noindent
It is OK to use {\tt d} as a variable name.
It will not conflict with function {\tt d}.

\bigskip
\noindent
It is OK to redefine {\tt d} as a different function.
The function {\tt derivative}, a synonym for {\tt d},
can still be used to obtain a partial derivative.

\subsection*{defint($f,x,a,b$)}

Returns the definite integral of $f$ with respect to $x$
evaluated from $a$ to $b$.
The argument list can be extended for multiple integrals
as shown in the following example.

{\color{blue}
\begin{verbatim}
f = (1 + cos(theta)^2) sin(theta)
defint(f, theta, 0, pi, phi, 0, 2pi) -- integrate over theta then over phi
\end{verbatim}
}

\noindent
$\tfrac{16}{3}\pi$

\subsection*{deg($p,x$)}

Returns the degree of polynomial $p(x)$.

{\color{blue}
\begin{verbatim}
p = (2x + 1)^3
deg(p,x)
\end{verbatim}
}

\noindent
$3$

\subsection*{denominator($x$)}

Returns the denominator of expression $x$.

{\color{blue}
\begin{verbatim}
denominator(a/b)
\end{verbatim}
}

\noindent
$b$

\subsection*{det($m$)}

Returns the determinant of matrix $m$.

{\color{blue}
\begin{verbatim}
A = ((a,b),(c,d))
det(A)
\end{verbatim}
}

\noindent
$a d - b c$

\subsection*{dim($a,n$)}

Returns the dimension of the $n$th index of tensor $a$.
Index numbering starts with 1.

{\color{blue}
\begin{verbatim}
A = ((1,2),(3,4),(5,6))
dim(A,1)
\end{verbatim}
}

\noindent
$3$

\subsection*{div($u$)}

Returns the divergence of vector $u$.
It is OK to redefine {\tt div}.
This is the default definition.

{\color{blue}
\begin{verbatim}
div(u) = d(u[1],x) + d(u[2],y) + d(u[3],z)
\end{verbatim}
}

\subsection*{do($a,b,\ldots$)}

Evaluates each argument from left to right.
Returns the result of the final argument.

{\color{blue}
\begin{verbatim}
do(A=1,B=2,A+B)
\end{verbatim}
}

\noindent
$3$

\subsection*{dot($a,b,\ldots$)}

Returns the dot product of vectors, matrices, and tensors.
Also known as the matrix product.

{\color{blue}
\begin{verbatim}
-- solve for X in AX=B
A = ((1,2),(3,4))
B = (5,6)
X = dot(inv(A),B)
X
\end{verbatim}
}

\noindent
$\displaystyle \begin{bmatrix}-4\\ \tfrac{9}{2}\end{bmatrix}$

\subsection*{draw($f,x$)}

Draws a graph of $f(x)$.
Drawing ranges can be set with {\tt xrange} and {\tt yrange}.

{\color{blue}
\begin{verbatim}
xrange = (0,1)
yrange = (0,1)
draw(x^2,x)
\end{verbatim}
}

\subsection*{e}

Symbol {\tt e} is initialized to the natural number $e=\exp(1)$.

{\color{blue}
\begin{verbatim}
e^x
\end{verbatim}
}

\noindent
$\exp(x)$

\bigskip
\noindent
Note: It is OK to clear or redefine {\tt e} and use the symbol for something else.

\subsection*{eigen($m$)}

Computes eigenvalues and eigenvectors numerically.
Matrix $m$ is required to be both numerical and symmetric.
Eigenvectors are returned in Q and eigenvalues are returned in D.
Each row of Q is an eigenvector.
Each diagonal element of D is an eigenvalue.

{\color{blue}
\begin{verbatim}
A = ((1,2),(2,1))
eigen(A)
dot(transpose(Q),D,Q)
\end{verbatim}
}

\noindent
$\displaystyle \begin{bmatrix}
1.0 & 2.0\\
2.0 & 1.0
\end{bmatrix}
$

\subsection*{erf($x$)}

Error function of $x$.

\subsection*{erfc($x$)}

Complementary error function of $x$.

\subsection*{eval($f,x,a$)}

Returns expression $f$ evaluated at $x$ equals $a$.
The argument list can be extended for multivariate expressions.
For example,
\verb$eval(f,x,a,y,b)$
is equivalent to
\verb$eval(eval(f,x,a),y,b)$.

{\color{blue}
\begin{verbatim}
eval(x + y,x,a,y,b)
\end{verbatim}
}

\noindent
$a+b$

\subsection*{exp($x$)}

Returns the exponential of $x$.

{\color{blue}
\begin{verbatim}
exp(i pi)
\end{verbatim}
}

\noindent
$-1$

\subsection*{expand($r,x$)}

Returns the partial fraction expansion of the ratio of polynomials $r$ in $x$.

{\color{blue}
\begin{verbatim}
p = (x + 1)^2
q = (x + 2)^2
expand(p/q,x)
\end{verbatim}
}

\noindent
$\displaystyle -\frac{2}{x+2}+\frac{1}{x^2+4x+4}+1$

\subsection*{expcos($z$)}

Returns the cosine of $z$ in exponential form.

{\color{blue}
\begin{verbatim}
expcos(z)
\end{verbatim}
}

\noindent
$\displaystyle \tfrac{1}{2}\exp(iz)+\tfrac{1}{2}\exp(-iz)$

\subsection*{expcosh($z$)}

Returns the hyperbolic cosine of $z$ in exponential form.

{\color{blue}
\begin{verbatim}
expcosh(z)
\end{verbatim}
}

\noindent
$\displaystyle \tfrac{1}{2}\exp(-z)+\tfrac{1}{2}\exp(z)$

\subsection*{expsin($z$)}

Returns the sine of $z$ in exponential form.

{\color{blue}
\begin{verbatim}
expsin(z)
\end{verbatim}
}

\noindent
$\displaystyle -\tfrac{1}{2}i\exp(iz)+\tfrac{1}{2}i\exp(-iz)$

\subsection*{expsinh($z$)}

Returns the hyperbolic sine of $z$ in exponential form.

{\color{blue}
\begin{verbatim}
expsinh(z)
\end{verbatim}
}

\noindent
$\displaystyle -\tfrac{1}{2}\exp(-z)+\tfrac{1}{2}\exp(z)$

\subsection*{exptan($z$)}

Returns the tangent of $z$ in exponential form.

{\color{blue}
\begin{verbatim}
exptan(z)
\end{verbatim}
}

\noindent
$\displaystyle \frac{i}{\exp(2iz)+1}-\frac{i\exp(2iz)}{\exp(2iz)+1}$

\subsection*{exptanh($z$)}

Returns the hyperbolic tangent of $z$ in exponential form.

{\color{blue}
\begin{verbatim}
exptanh(z)
\end{verbatim}
}

\noindent
$\displaystyle -\frac{1}{\exp(2z)+1}+\frac{\exp(2z)}{\exp(2z)+1}$

\subsection*{factor($n$)}

Factors numerical value $n$ and returns the result.

{\color{blue}
\begin{verbatim}
factor(12/35)
\end{verbatim}
}

\noindent
$\displaystyle \frac{2^2\times 3}{5\times 7}$

\bigskip
\noindent
If $n$ is a floating point value then a rational approximation of $n$ is factored and returned.

{\color{blue}
\begin{verbatim}
factor(float(pi))
\end{verbatim}
}

\noindent
$\displaystyle \frac{5\times 71}{113}$

\subsection*{factor($p,x$)}

Factors polynomial $p$ of $x$ and returns the result.
The argument list can be extended for multivariate polynomials.

{\color{blue}
\begin{verbatim}
p = 2x + x y + y + 2
factor(p,x,y)
\end{verbatim}
}

\noindent
$\displaystyle (x+1)(y+2)$

\bigskip
\noindent
Note:
\verb$factor$
returns an unexpanded expression.
If the result is assigned to a symbol, evaluating the symbol will expand the result.
Use
\verb$binding$
to retrieve the unexpanded expression.

{\color{blue}
\begin{verbatim}
q = factor(p,x,y)
binding(q)
\end{verbatim}
}

\noindent
$\displaystyle (x+1)(y+2)$

\subsection*{factorial($n$)}

Returns the factorial of $n$.
The expression {\tt n!} can also be used.

{\color{blue}
\begin{verbatim}
20!
\end{verbatim}
}

\noindent
$2432902008176640000$

\subsection*{filter($f,a,b,\ldots$)}

Returns $f$ excluding any terms containing $a$, $b$, etc.

{\color{blue}
\begin{verbatim}
p = x^2 + 3x + 2
filter(p,x^2)
\end{verbatim}
}

\noindent
$3x+2$

\subsection*{float($x$)}

Returns expression $x$ with rational numbers and integers converted to
floating point values.
The symbol {\tt pi} and the natural number are also converted.

{\color{blue}
\begin{verbatim}
float(212^17)
\end{verbatim}
}

\noindent
$\displaystyle 3.52947\times 10^{39}$

\subsection*{floor($x$)}

Returns the largest integer less than or equal to $x$.

{\color{blue}
\begin{verbatim}
floor(1/2)
\end{verbatim}
}

\noindent
$0$

\subsection*{for($i,j,k,a,b,\ldots$)}

For $i$ equals $j$ through $k$ evaluate $a$, $b$, etc.

{\color{blue}
\begin{verbatim}
for(k,1,3,A=k,print(A))
\end{verbatim}
}

\noindent
$A=1$\\
$A=2$\\
$A=3$

\bigskip
\noindent
Note: The original value of $i$ is restored after {\tt for} completes.
If symbol {\tt i} is used for index variable $i$
then the imaginary unit is overridden in the scope of {\tt for}.

\subsection*{gcd($a,b,\ldots$)}

Returns the greatest common divisor of expressions.

{\color{blue}
\begin{verbatim}
gcd(x,x y)
\end{verbatim}
}

\noindent
$x$

\subsection*{hermite($x,n$)}

Returns the $n$th Hermite polynomial in $x$.

{\color{blue}
\begin{verbatim}
hermite(x,3)
\end{verbatim}
}

\noindent
$\displaystyle 8x^3-12x$

\subsection*{hilbert($n$)}

Returns an $n$ by $n$ Hilbert matrix.

{\color{blue}
\begin{verbatim}
hilbert(3)
\end{verbatim}
}

\noindent
$\displaystyle
\begin{bmatrix}
1 & \tfrac{1}{2} & \tfrac{1}{3}\\ \\
\tfrac{1}{2} & \tfrac{1}{3} & \tfrac{1}{4}\\ \\
\tfrac{1}{3} & \tfrac{1}{4} & \tfrac{1}{5}
\end{bmatrix}
$

\subsection*{i}

Symbol {\tt i} is initialized to the imaginary unit $\sqrt{-1}$.

{\color{blue}
\begin{verbatim}
exp(i pi)
\end{verbatim}
}

\noindent
$-1$

\bigskip
\noindent
Note: It is OK to clear or redefine {\tt i} and use the symbol for something else.

\subsection*{imag($z$)}

Returns the imaginary part of complex $z$.

{\color{blue}
\begin{verbatim}
imag(2 - 3i)
\end{verbatim}
}

\noindent
$-3$

\subsection*{inner($a,b,\ldots$)}

Returns the inner product of vectors, matrices, and tensors.
Also known as the matrix product.

{\color{blue}
\begin{verbatim}
A = ((a,b),(c,d))
B = (x,y)
inner(A,B)
\end{verbatim}
}

\noindent
$\displaystyle
\begin{bmatrix}
ax+by\\
cx+dy
\end{bmatrix}
$

\bigskip
\noindent
Note: {\tt inner} and {\tt dot} are the same function.

\subsection*{integral($f,x$)}

Returns the integral of $f$ with respect to $x$.

{\color{blue}
\begin{verbatim}
integral(x^2,x)
\end{verbatim}
}

\noindent
$\displaystyle \tfrac{1}{3}x^3$

\subsection*{inv($m$)}

Returns the inverse of matrix $m$.

{\color{blue}
\begin{verbatim}
A = ((1,2),(3,4))
inv(A)
\end{verbatim}
}

\noindent
$\displaystyle
\begin{bmatrix}
-2 & 1\\
\tfrac{3}{2} & -\tfrac{1}{2}
\end{bmatrix}
$

\subsection*{isprime($n$)}

Returns 1 if $n$ is a prime number. Returns zero otherwise.

{\color{blue}
\begin{verbatim}
isprime(2^31 - 1)
\end{verbatim}
}

\noindent
1

\subsection*{j}

Set {\tt j=sqrt(-1)} to use {\tt j} for the imaginary unit instead of {\tt i}.

{\color{blue}
\begin{verbatim}
j = sqrt(-1)
1/sqrt(-1)
\end{verbatim}
}

\noindent
$-j$

\subsection*{laguerre($x,n,a$)}

Returns the $n$th Laguerre polynomial in $x$.
If argument $a$ is omitted then $a=0$ is used.

{\color{blue}
\begin{verbatim}
laguerre(x,3)
\end{verbatim}
}

\noindent
$\displaystyle -\tfrac{1}{6}x^3+\tfrac{3}{2}x^2-3x+1$

\subsection*{last}

The result of the previous calculation is stored in {\tt last}.

{\color{blue}
\begin{verbatim}
212^17
\end{verbatim}
}

\noindent
$3529471145760275132301897342055866171392$

{\color{blue}
\begin{verbatim}
last
\end{verbatim}
}

\noindent
$last=3529471145760275132301897342055866171392$

\bigskip
\noindent
Symbol {\tt last} is an implied argument when a function has no argument list.

{\color{blue}
\begin{verbatim}
float
\end{verbatim}
}

\noindent
$\displaystyle 3.52947\times10^{39}$

\subsection*{lcm($a,b,\ldots$)}

Returns the least common multiple of expressions.

{\color{blue}
\begin{verbatim}
lcm(x,x y)
\end{verbatim}
}

\noindent
$xy$

\subsection*{leading($p,x$)}

Returns the leading coefficient of polynomial $p(x)$.

{\color{blue}
\begin{verbatim}
leading(3x^2 + 1,x)
\end{verbatim}
}

\noindent
$3$

\subsection*{legendre($x,n,m$)}

Returns the $n$th Legendre polynomial in $x$.
If $m$ is omitted then $m=0$ is used.

{\color{blue}
\begin{verbatim}
legendre(x,3)
\end{verbatim}
}

\noindent
$\displaystyle \tfrac{5}{2}x^3-\tfrac{3}{2}x$

\subsection*{lisp($x$)}

Evaluates expression $x$ and returns the result as a
string in prefix notation.
Useful for debugging scripts.

{\color{blue}
\begin{verbatim}
lisp(x^2 + 1)
\end{verbatim}
}

\noindent
\verb$(+ (^ x 2) 1)$

\subsection*{log($x$)}

Returns the natural logarithm of $x$.

{\color{blue}
\begin{verbatim}
log(x^y)
\end{verbatim}
}

\noindent
$y\log(x)$

\subsection*{mag($z$)}

Returns the magnitude of complex $z$.
Function {\tt mag} treats undefined symbols as real while {\tt abs} does not.

{\color{blue}
\begin{verbatim}
mag(x + i y)
\end{verbatim}
}

\noindent
$\displaystyle (x^2+y^2)^{1/2}$

\subsection*{mod($a,b$)}

Returns the remainder of $a$ divided by $b$.

{\color{blue}
\begin{verbatim}
mod(10,7)
\end{verbatim}
}

\noindent
$3$

\subsection*{not($x$)}

Returns 0 if $x$ is true (nonzero).
Returns 1 otherwise.

{\color{blue}
\begin{verbatim}
not(1=1)
\end{verbatim}
}

\noindent
$0$

\subsection*{nroots($p,x$)}

Returns all roots, both real and complex,
of polynomial $p(x)$.
The roots are computed numerically.

\subsection*{number($x$)}

Returns 1 if $x$ is a rational or floating point number.
Returns 0 otherwise.

{\color{blue}
\begin{verbatim}
number(1/2)
\end{verbatim}
}

\noindent
$1$

\subsection*{numerator($x$)}

Returns the numerator of expression $x$.

{\color{blue}
\begin{verbatim}
numerator(a/b)
\end{verbatim}
}

\noindent
$a$

\subsection*{or($a,b,\ldots$)}

Returns 1 if at least one argument is true (nonzero).
Returns 0 otherwise.

{\color{blue}
\begin{verbatim}
or(1=1,2=2)
\end{verbatim}
}

\noindent
$1$

\subsection*{outer($a,b,\ldots$)}

Returns the outer product of vectors, matrices, and tensors.
Also known as the tensor product.

{\color{blue}
\begin{verbatim}
A = (a,b,c)
B = (x,y,z)
outer(A,B)
\end{verbatim}
}

\noindent
$\displaystyle
\begin{bmatrix}
a x & a y & a z\\
b x & b y & b z\\
c x & c y & c z
\end{bmatrix}
$

\subsection*{pi}

Symbol for $\pi$.

{\color{blue}
\begin{verbatim}
exp(i pi)
\end{verbatim}
}

\noindent
$-1$

\subsection*{polar($z$)}

Returns complex $z$ in polar form.

{\color{blue}
\begin{verbatim}
polar(x - i y)
\end{verbatim}
}

\noindent
$\displaystyle (x^2+y^2)^{1/2}\exp(i\arctan(-y,x))$

\subsection*{power}

Use \verb$^$ to raise something to a power.
Use parentheses for negative powers.

{\color{blue}
\begin{verbatim}
x^(-2)
\end{verbatim}
}

\noindent
$\displaystyle \frac{1}{x^2}$

\subsection*{prime($n$)}

Returns the $n$th prime number.
The domain of $n$ is 1 to 10000.
{\color{blue}
\begin{verbatim}
prime(100)
\end{verbatim}
}

\noindent
$541$

\subsection*{print($a,b,\ldots$)}

Evaluate expressions and print the results.
Useful for printing from inside a {\tt for} loop.

{\color{blue}
\begin{verbatim}
for(j,1,3,print(j))
\end{verbatim}
}

\noindent
$j=1$\newline
$j=2$\newline
$j=3$

\section*{product($i,j,k,f$)}

For $i$ equals $j$ through $k$ evaluate $f$.
Returns the product of all $f$.

{\color{blue}
\begin{verbatim}
product(j,1,3,x + j)
\end{verbatim}
}

\noindent
$\displaystyle x^3+6x^2+11x+6$

\bigskip
\noindent
Note: The original value of $i$ is restored after {\tt product} completes.
If symbol {\tt i} is used for index variable $i$
then the imaginary unit is overridden in the scope of {\tt product}.

\subsection*{quote($x$)}

Returns expression $x$ without evaluating it first.

{\color{blue}
\begin{verbatim}
quote((x + 1)^2)
\end{verbatim}
}

\noindent
$\displaystyle (x+1)^2$

\subsection*{quotient($p,q,x$)}

Returns the quotient of polynomial $p(x)$ over $q(x)$.

{\color{blue}
\begin{verbatim}
p = x^2 + 1
q = x + 3
quotient(p,q,x)
\end{verbatim}
}

\noindent
$x-3$

\subsection*{rank($a$)}

Returns the number of indices that tensor $a$ has.

{\color{blue}
\begin{verbatim}
A = ((a,b),(c,d))
rank(A)
\end{verbatim}
}

\noindent
2

\subsection*{rationalize($x$)}

Returns expression $x$ with everything over a common denominator.

{\color{blue}
\begin{verbatim}
rationalize(1/a + 1/b + 1/2)
\end{verbatim}
}

\noindent
$\displaystyle \frac{2a+ab+2b}{2ab}$

\bigskip
\noindent
Note:
\verb$rationalize$
returns an unexpanded expression.
If the result is assigned to a symbol, evaluating the symbol will expand the result.
Use
\verb$binding$
to retrieve the unexpanded expression.

{\color{blue}
\begin{verbatim}
f = rationalize(1/a + 1/b + 1/2)
binding(f)
\end{verbatim}
}

\noindent
$\displaystyle \frac{2a+ab+2b}{2ab}$

\subsection*{real($z$)}

Returns the real part of complex $z$.

{\color{blue}
\begin{verbatim}
real(2 - 3i)
\end{verbatim}
}

\noindent
2

\subsection*{rect($z$)}

Returns complex $z$ in rectangular form.

{\color{blue}
\begin{verbatim}
rect(exp(i x))
\end{verbatim}
}

\noindent
$\displaystyle \cos(x)+i\sin(x)$

\subsection*{roots($p,x$)}

Returns the $x$ such that $p(x)=0$.
Polynomial $p(x)$ should be factorable over integers.
Returns a vector for multiple roots.

{\color{blue}
\begin{verbatim}
roots(x^2 + 3x + 2,x)
\end{verbatim}
}

\noindent
$\displaystyle
\begin{bmatrix}
-2\\
-1
\end{bmatrix}
$

\subsection*{run({\it file})}

Run script {\it file}.
Useful for importing function libraries.

{\color{blue}
\begin{verbatim}
run("Downloads/EVA.txt")
\end{verbatim}
}

\noindent
Note: {\it file} must be in the Downloads folder due to security requirements for apps distributed on the Mac App Store.

\subsection*{simplify($x$)}

Returns expression $x$ in a simpler form.

{\color{blue}
\begin{verbatim}
simplify(sin(x)^2 + cos(x)^2)
\end{verbatim}
}

\noindent
1

\subsection*{sin($x$)}

Returns the sine of $x$.

{\color{blue}
\begin{verbatim}
sin(pi/4)
\end{verbatim}
}

\noindent
$\displaystyle \frac{1}{2^{1/2}}$

\subsection*{sinh($x$)}

Returns the hyperbolic sine of $x$.

{\color{blue}
\begin{verbatim}
circexp(sinh(x))
\end{verbatim}
}

\noindent
$\displaystyle -\tfrac{1}{2}\exp(-x)+\tfrac{1}{2}\exp(x)$

\subsection*{sqrt($x$)}

Returns the square root of $x$.

{\color{blue}
\begin{verbatim}
sqrt(10!)
\end{verbatim}
}

\noindent
$\displaystyle 720\; 7^{1/2}$

\subsection*{status}

Prints memory statistics.

\bigskip
\noindent
{\color{blue}
\verb$status$
}

\bigskip
\noindent
\verb$block_count 1$\\
\verb$free_count 99258$\\
\verb$gc_count 1$\\
\verb$bignum_count 370$\\
\verb$string_count 0$\\
\verb$tensor_count 5$

\subsection*{stop}

In a script, it does what it says.

\subsection*{string($x$)}

Evaluates expression $x$ and returns the result as a string.
Useful for testing scripts.

{\color{blue}
\begin{verbatim}
string((x + 1)^2) == "x^2 + 2 x + 1"
\end{verbatim}
}

\noindent
1

\subsection*{subst($a,b,c$)}

Substitutes $a$ for $b$ in $c$ and returns the result.

{\color{blue}
\begin{verbatim}
subst(x,y,y^2)
\end{verbatim}
}

\noindent
$\displaystyle x^2$

\subsection*{sum($i,j,k,f$)}

For $i$ equals $j$ through $k$ evaluate $f$.
Returns the sum of all $f$.

{\color{blue}
\begin{verbatim}
sum(j,1,5,x^j)
\end{verbatim}
}

\noindent
$\displaystyle x^5+x^4+x^3+x^2+x$

\bigskip
\noindent
Note: The original value of $i$ is restored after {\tt sum} completes.
If symbol {\tt i} is used for index variable $i$
then the imaginary unit is overridden in the scope of {\tt sum}.

\subsection*{tan($x$)}

Returns the tangent of $x$.

{\color{blue}
\begin{verbatim}
simplify(tan(x) - sin(x)/cos(x))
\end{verbatim}
}

\noindent
0

\subsection*{tanh($x$)}

Returns the hyperbolic tangent of $x$.

{\color{blue}
\begin{verbatim}
circexp(tanh(x))
\end{verbatim}
}

\noindent
$\displaystyle -\frac{1}{\exp(2x)+1}+\frac{\exp(2x)}{\exp(2x)+1}$

\subsection*{taylor($f,x,n,a$)}

Returns the Taylor expansion of $f(x)$ near $x=a$.
If argument $a$ is omitted then $a=0$ is used.
Argument $n$ is the degree of the expansion.

{\color{blue}
\begin{verbatim}
taylor(sin(x),x,5)
\end{verbatim}
}

\noindent
$\displaystyle \tfrac{1}{120}x^5-\tfrac{1}{6}x^3+x$

\subsection*{test($a,b,c,d,\ldots$)}

If argument $a$ is true (nonzero) then $b$ is returned, else if $c$ is true then $d$ is returned, etc.
If the number of arguments is odd then the final argument is returned if all else fails.
Expressions can include the relational operators
\verb$=$,
\verb$==$,
\verb$<$,
\verb$<=$,
\verb$>$,
\verb$>=$.
Use the
\verb$not$
function to test for inequality.
(The equality operator
\verb$==$
is available for contexts in which
\verb$=$
is the assignment operator.)

{\color{blue}
\begin{verbatim}
A = 1
B = 1
test(A=B,"yes","no")
\end{verbatim}
}

\noindent
yes

\subsection*{trace}

Set {\tt trace=1} in a script to print the script as it is evaluated.
Useful for debugging.

{\color{blue}
\begin{verbatim}
trace = 1
\end{verbatim}
}

\noindent
Note:
The
\verb$contract$
function is used to obtain the trace of a matrix.

\subsection*{transpose($a,i,j$)}

Returns the transpose of tensor $a$ with respect to indices $i$ and $j$.
If $i$ and $j$ are omitted then 1 and 2 are used.
Hence a matrix can be transposed with a single argument.

{\color{blue}
\begin{verbatim}
A = ((a,b),(c,d))
transpose(A)
\end{verbatim}
}

\noindent
$\displaystyle
\begin{bmatrix}
a & c\\
b & d
\end{bmatrix}
$

\bigskip
\noindent
Note:
The argument list can be extended for multiple transpose operations.
The arguments are evaluated from left to right.
For example,
\verb$transpose(A,1,2,2,3)$
is equivalent to
\verb$transpose(transpose(A,1,2),2,3)$.

\subsection*{tty}

Set {\tt tty=1} to print results in text format.

{\color{blue}
\begin{verbatim}
tty = 1
(x + 1/2)^2
\end{verbatim}
}

\noindent
\verb$x^2 + x + 1/4$

\subsection*{unit($n$)}

Returns an $n$ by $n$ identity matrix.

{\color{blue}
\begin{verbatim}
unit(3)
\end{verbatim}
}

\noindent
$\displaystyle
\begin{bmatrix}
1 & 0 & 0\\
0 & 1 & 0\\
0 & 0 & 1
\end{bmatrix}
$

\subsection*{zero($i,j,\ldots$)}

Returns a null tensor with dimensions $i$, $j$, etc.
Useful for creating a tensor and then setting component values.

{\color{blue}
\begin{verbatim}
A = zero(3,3)
for(k,1,3,A[k,k]=k)
A
\end{verbatim}
}

\noindent
$\displaystyle
A=
\begin{bmatrix}
1 & 0 & 0\\
0 & 2 & 0\\
0 & 0 & 3
\end{bmatrix}
$


\section{Hashing}
To make Maths people feel better about this exam, with the courtesy of Harald Räcke.

The purpose of hashing is having a data structure that tries to directly compute the memory location from the given key, obtaining constant search time.

Hash functions and their associated hash tables are used in data storage and retrieval applications to access and store information thanks to their efficiency.

Given an universe $U$ of keys and a set $S \leq U$ of used keys to be distributed over an array $T[0, \dots, n-1]$, an optimal hash function $H : U \rightarrow [0, \dots, n-1]$ should be fast to evaluate, have small storage requirements and a good distribution of elements over the whole table.

Ideally, the hash function would map all keys to different memory locations (direct addressing), yet this is unrealistic since the size of the table would be much larger than available memory.

Knowing previously the set $S$ of actual keys, it could be possible to design a simple function that maps all of them to different memory locations (perfect hashing); but this is not often the case.

The best expectation would be a function which distributes keys evenly across the table, yet this leads to the problem of collision: usually the universe $U$ is much larger than the table size $n$, hence there may be two elements $k_1, k_2$ from the same set $S$ so that $h(k_1) = h(k_2)$, and therefore getting mapped to the same memory location.

Typically, collisions do not appear once the size of the set $S$ of actual keys get close to $n$, but otherwise the probability of having a collision when hashing $m$ elements under uniform hashing is at least:
$$1 - e^{-\frac{m(m-1)}{2n}} \approx 1 - e^{-\frac{m^2}{2n}}$$

Practically, hash tables should be larger than the amount of keys to store, for instance the next power of two. There also are scaling coefficients such as 1.5 or 1.5, to avoid resizing too much and wasting space. Precomputing prime numbers is an useful option, but requires a range of about 10\% more than the number of keys.

There are methods to deal with collisions: open addressing, hashing with chaining, or just not resolving them at all.

Hashing with chaining arranges elements that map to the same position in a linear list, inserting at the front of the list. The average time required for an unsuccessful search is $A^- = 1 + \frac{m}{n}$, while for a successful search it is $A^+ \leq 1 + \frac{\nicefrac{m}{n}}{2}$.

Open addressing, in particular, stores all objects in the table defining a function that determines the position through a permutation of all possible $n - 1$ values. If an insertion fails, a new position is searched (either through a different function or looking for the first free slot).

A good choice for $h(i, j)$ to have an uniform distribution over the possible values is a modulo with a prime number.

Introducing prime numbers is relevant because the size of hash table should not have factors in common with the linear component of the function, otherwise keys would be mapped to the same position. Increasing has also to be done by odd numbers, to reduce the possibility of remainder zero.

Prime numbers can be generated through algorithms, of which the most basic is the prime sieve: a list of integers up to a desired number is created, and composite numbers are progressively removed until only primes are left.

For large primes used in cryptography, a random chosen range of odd numbers of the desired size is sieved against a number of relatively small primes, and the remaining are subject to primality testing (Miller-Rabin).

To keep uniformity of the distribution even allowing new elements to be added, rehashing might be necessary, or incremental of the prime.

\subsection{Fibonacci hashing}
Fibonacci hashing is a variation of classic hashing using instead of a modulo operation, a multiplication by the golden ratio:
$$\frac{x}{y} = \frac{x + y}{x} \implies \phi = \frac{1 + \sqrt{5}}{2}$$

The golden ratio has a close relationship with Fibonacci numbers: the previous equality, in fact, holds between every Fibonacci number and its successor.

The $n^{th}$ Fibonacci number is obtained calculating $F_n = \frac{1}{\sqrt{5}}(\phi^n - \varphi^n)$, where $\phi = \frac{(1 + \sqrt{5})}{2}$ and $\varphi = \frac{(1 - \sqrt{5})}{2}$.

In the context of Fibonacci hashing, it is taken the reciprocal of $\phi$, and hashing is performed multiplying the value $k$ by the integer relatively prime to $k$ which is closer to $\phi^{-1}k$.

Consecutive keys have an uniform spread distribution, therefore it is an efficient method.

\subsection{Neojoin}
Hashing is relevant in a distributed systems architecture because it can help determining how to locate relations on nodes.

While performing a distributed join, tuples may join with tuples on remote nodes: relations can be repartitioned and redistributed for local joins, so that tuples will only join with the corresponding partition.

The partitioning scheme defines how to assign partitions to nodes, using for instance a hash function. There are several options to implement a scheme:
\begin{enumerate}
	\item Minimizing network traffic, assigning a partition to the node which owns its largest part. This way, only the small fragments of a partition are sent over the network, but a schedule with minimal network traffic may have high duration;
	\item Minimizing response time, i. e. the time from request to result. This is dominated by network duration, which depends on maximum straggler;
	\item Minimizing maximum straggler, formulated as a NP-hard linear programming problem where nodes have the most similar possible amount of workload.
\end{enumerate}

Satisfying those constraints may lead to unused resources, for instance nodes not doing any work.

\subsection{Chord hashing}
Chord is a protocol for a peer-to-peer distributed hash table. it stores key-values assigning a node the values for which it is responsible, using a chord to assign and discover values.

Nodes and keys are given an hashed identifier of $m$ bits, and using the Chord lookup protocol nodes are put in a circle that has as most $2^m$ nodes.

Each node has a successor and a predecessor, going in clockwise direction, but normally there are holes in the sequence. In general, the successor of a node $k$ has the first identifier equal or greater than $k$.

Lookup is performed passing the query through the successors of a node, if the key cannot be found locally. This leads to linear worst-case performance, but can be avoided implementing a finger table in each node.

A finger table contains up to $m$ entries, and entry $i$ of node $n$ contains the successor of key $(n + 2^{i-1}) \mod 2^m$. At every lookup, the query is passed to closest successor or predecessor of $k$ in the finger table, narrowing worst-case performance to logarithmic.

Each peer is responsible for all keys larger than the predecessor number until its own. Introducing a factor of $2^k$, the distance always increases by 2, halving the number of hops each time.

\begin{figure}[h]
	\includegraphics[scale=0.45]{chord.png}
	\centering
\end{figure}

\section{Distributed data processing}
Distributed databases are a great solution to optimize memory and speed, yet not every operation is supported by them. Machine learning or graph algorithms, for instance, are hard to compute on a sharded relational schema.

Analysis can be implemented in several different ways, but certain tasks need to be handled in every implementation:
\begin{itemize}
	\item Parallelization/synchronization;
	\item Distribution of computation and data;
	\item Communication between nodes;
	\item Node failures.
\end{itemize}

Creating a custom solution takes time and is error prone; functional abstractions can be used to specify computation in parallel. The runtime system, this way, manages program execution, data distribution and persistence, node failures and communication.

When a cluster is necessary, it is possible to use pre-configured structures:
\begin{enumerate}
	\item Infrastructure as a Service, externalizing resources and storage;
	\item Platform as a Service, renting databases and web servers;
	\item Software as a Service, using developer tools and additional packages.
\end{enumerate}

\subsection{MapReduce}
MapReduce is a programming model with associated implementation for processing and generating big data sets with a parallel distributed algorithm on a cluster.

It is composed of:
\begin{enumerate}
	\item A Map procedure, which performs filtering and sorting, applied by each worker to the local data and collecting output to a temporary storage;
	\item A Shuffle phase, where workers redistribute data based on the same output keys, so that all data with the same key is located on the same node;
	\item A Reduce method, which executes a summary operation (such as counting) on local data in parallel.
\end{enumerate}

The model is inspired by functional programming, but its key contributions are the scalability and fault tolerance achieved by optimizing the execution engine, especially on multi-threaded implementations.

Maps can be performed in parallel, and a set of reducers can perform the reduction phase. Parallelism offers possibility of recovering from partial failure of servers or storage: the work can be rescheduled without recomputing all of it.

Logically, Map takes one pair of data with a type in one data domain, and returns a list of pairs in a different domain:
$$Map(k_1, v_1) \rightarrow list(k_2, v_2)$$

The Reduce function is then applied in parallel to each group, which in turn produces a collection of values in the same domain:
$$Reduce(k_2, list(v_2)) \rightarrow list((k_3, v_3))$$

Each Reduce call typically produces either one key-value par or an empty return.

Example (from Wikipedia):
\begin{lstlisting}
function map(String name, String document):
	// name: document name
	// document: document contents
	for each word w in document:
		emit (w, 1)

function reduce(String word, Iterator partialCounts):
	// word: a word
	// partialCounts: a list of aggregated partial counts
	sum = 0
	for each pc in partialCounts:
		sum += pc
	emit (word, sum)
\end{lstlisting}
This MapReduce performs a word count on a document. Each document is split into words, and every word is counted by Map; then all pairs with the same key are put together and sent to Reduce, which sums the relative counts.

MapReduce implementaton can be applied to many algorithms, such as joining of two relations or PageRank.

\subsubsection{PageRank}
PageRank is an algorithm used by Google Search to rank web pages in their search engine results, measuring the relevance of them.

It works assigning a numerical weight to each element of an hyperlinked set of documents, with the purpose of obtaining its relative importance within the set.

It outputs a probability distribution used to represent the likelihood that a person randomly clicking on links will arrive at any particular page. Links from a page to itself are ignored.

The PageRank transferred from a given page to the targets of its outbound links upon the next iteration is divided equally among all outbound links.

The value for a page $v$ is dependent on the values for each page  in the set containing all pages linking to $v$, divided by the number of links from $v$.

Pseudo-code PageRank:
\begin{enumerate}
	\item Retrieve total number of nodes $n$;
	\item Write initial values in each node, computed as $\frac{1}{n}$;
	\item Find next node as $\frac{1-d}{n}$, where $d$ is the correction factor;
	\item For each iteration of the algorithm:
	\begin{enumerate}
		\item Compute outgoing page rank fraction;
		\item Add fraction to all neighbors;
	\end{enumerate}
	\item Swap next node and current;
	\item Reset next reassigning it the value of $\frac{1-d}{n}$.
\end{enumerate}

This algorithm looks at each node in the graph, computes the fraction of rank it has to pass on to neighbors, and writes it.

For better performance, the value can be computed in reverse, collecting all fractions from reverse neighbors.

The sequential reads are optimized for modern computers, but the random access to next node exhibits performance problems. Caches help alleviating this, since they can store most visited nodes.

If the memory requirements exceed the available main memory, the system will allocate too much memory, running into several page faults. Sorting by degree will make often-accessed nodes all in the same page.

The distributed version sends computed values to other nodes, but can cause overhead and network congestion. MapReduce assigns the Shuffle phase the duty of message passing.

Computing PageRank using MapReduce is also performed using the function to repeatedly multiply a vector by the matrix representing how the distribution changes after every single step.

\subsection{Hadoop}
Hadoop is an open-source implementation of the MapReduce framework, written in Java (opposed to the proprietary of Google in C++). It is now an Apache project used for very large clusters to handle big data sets.

\subsubsection{MapReduce}
Hadoop MapReduce is an application on the cluster that negotiates resources with YARN: a JobTracker globally manages jobs, and every node has its own TaskTracker.

\begin{enumerate}
	\item An user submits job configuration to JobTracker;
	\item JobTracker submits mapper tasks to TaskTrackers, scheduling computation on all available cluster nodes;
	\item Each task works on the data local to its node;
	\item The Shuffler moves output produced my Map among the nodes in the cluster;
	\item Once shuffling is complete, JobTracker schedules Reduce to all nodes.
\end{enumerate}

Since the datasets are much larger than the main memory and processing happens on very big clusters, intermediate results must be written to disk and shuffled results should be ordered.

MapReduce is however a framework not suitable for all uses: not everything can be expressed through the functions it offers, plus it is not a good fit for workloads with a lot of computation and smaller datasets.

\subsubsection{HDFS}
Hadoop relies on a distributed file system (HDFS), the primary data storage set used by its applications. It supports the rapid transfer of data between compute nodes, breaking information down into separate blocks and distributing it.

The file system ensures durability and data distribution, storing data read by Map, and pre-sorting and saving its output.

Shuffling moves data between nodes and merges pre-sorted data, and results of Reduce are again written to HDFS.

HDFS is highly fault tolerant: the system replicates data multiple times, placing at least one copy on a different server rack than the others.

Files are split into blocks of 64MB, and by default each block has 3 replicas, so that missing data can be reconstructed.

It follows a master-slave architecture: Name node manages file system and regulates access, while data nodes manage attached storage.

I/O is boosted by striping blocks of data over multiple disks, writing in parallel. HDFS exposes append-only files to users.

JobTracker is responsible to cope with failing nodes: it notices that one TaskTracker is not responsive, and its task gets reassigned to another node.

If its job is in Reduce phase, the correspondent function is restarted on other TaskTrackers.

\subsubsection{YARN}
YARN stands for Yet Another Resource Negotiator, and is a redesigned resource manager and large-scale distributed operating system for Hadoop.

YARN splits up the functionalities of resource management and job scheduling into separate daemons, to obtain a global resouce manager and a master for every application.

The ResourceManager and the NodeManager form the data computation framework. A NodeManager is the agent in every machine which is responsible for containers, monitoring their resource usage and reporting results to the ResourceManager.

The Scheduler is responsible for allocating resources to the various running applications subject to constraints such as capacity, performing no tracking of application status.

\subsection{Optimizations}
To optimize MapReduce it is possible to add a combiner, an optional class that operated by accepting input from Map and passing it to Reduce after summarizing records with the same key.

It operates on each Map output key, having the same key-value output as the Reducer, and it helps segregating data into multiple groups.

Folding is a function to save disk space: it analyzes a recursive data structure, and returns recombined results by recursively processing its constituent parts.

Programs working well on large clusters require a deep understanding of warehouse-sized computers: different settings need different techniques. In general, HDFS should be portable.

Overall, Hadoop considers hardware failure to be the norm, with applications needing streaming access on large datasets. Its model, therefore, uses a model aiming to minimize writes and keeps computation local with the data.

\subsection{Scale up vs. out}
A data center is usually made of computers piled in racks, and racks are connected via a switch to form a cluster.

Accessing data on remote machines comes with a price: the faster is a local DRAM (20GB/S), but cluster DRAMs and disks are relatively slow.

In case no single machine is large enough to store all the data, it is necessary to choose between scaling up and out: a cluster of large expensive machines or a large cluster of cheap commodity machines.

A simple execution model calculates the total cost summing the expense of computation with all global data access. The data kept locally is inversely proportional to the size of cluster.
$$ TotalCost = \frac{1ms}{n} + f \times \Big[100ns \times \frac{1}{n} + 100\mu s \times \Big(1 - \frac{1}{n}\Big) \Big]$$

\begin{itemize}
	\item Light communication: $f = 1$;
	\item Medium communication: $f = 10$;
	\item Heavy communication: $f = 100$.
\end{itemize}

Workloads with light communication benefit from large clusters, while large amounts of random communication impede performance. Random reads must be avoided!

\subsection{Spark}
Spark is an alternative to Hadoop which provides more expressive fault-tolerant runtime for distributed batched data analytics. It can handle more complex analysis with interactive queries processing real time streams.

The purpose of a new programming model is extending the existing naive algorithms implementation:
\begin{itemize}
	\item Data can be outputted by sorted key;
	\item Locality can be increased using different kinds of partitioning;
	\item Window queries can be answered sorting by group.
\end{itemize}

The MapReduce model is extended too with complex data pipelines, but recovery mechanisms are lightweight and require less additional computation.

Sparks offers additional API for more functionalities, such as Spark SQL, GraphX and MLIB.

\subsubsection{RDDs}
Spark revolves around the concept of a resilient distributed dataset (RDD), which is a fault-tolerant collection of elements that can be operated on in parallel.

RDD is considered the evolution of the general Spark model, offering datasets that can be efficiently cached and manipulated.

RDDs support two types of operations: transformations, which create a new dataset from an existing one, and actions, which return a value to the driver program after running a computation on the dataset.

All transformations in Spark are lazy, in that they do not compute their results right away. Instead, they just remember the transformations applied to some base dataset (e.g. a file).

They are only computed when an action requires a result to be returned to the driver program, and the dataset gets only loaded when operations are performed on it. This design enables Spark to run more efficiently.

Pipelines allow to chain transformations and intermediate steps do not need to be materialized since they can just be recomputed on node failure.

\subsubsection{DataFrames}
DatFrames are dynamically strongly typed distributed collections of data organized into columns, conceptually equivalent to a table in a relational database.

They can be interacted with using SQL functions, allowing query optimization and avoiding interpretation overhead, although with a limited set of functions.

\subsubsection{Datasets}
A Dataset is a combination between DataFrame and RDD, optionally weakly typed, to retain code compilation. It is still mapped as a RDD.

Datasets can be created, analyzed (limited, filtered) and be subject of exploration, management or aggregation just like tables.

\subsubsection{Optimization}
RDDs track lineage information to rebuild lost data, logging everything in a graph to recompute missing or damaged partitions. Spark offers support for DAGs.

Snapshots are automatically created after repartitioning transformations and managed using a LRU cache to speed up recovery.

The optimizer builds a logical plan and applies transformation rules, to then generate the physical plan and select the cheapest cost model.

Spark's runtime focuses on three main points:
\begin{itemize}
	\item Memory Management and Binary Processing, leveraging semantics to eliminate the overhead of deserialization and garbage collection;
	\item Cache-aware computation, using columnar storage and compression to exploit memory hierarchy;
	\item Code generation to optimize usage of modern compilers and CPU.
\end{itemize}

Operations to take care of reading, managing ETL pipelines, training and query are performed within the same framework.

Foundations of Data Engineering/lectures/lecture5.tex


\subsection{The Moore-Penrose pseudoinverse}

For a matrix \(X\), the pseudoinverse is \((X^*X)^{-1}X^*\).

For real matrices, this is: \((X^TX)^{-1}X^T\)

The pseudoinverse can be written as \(X^+\)

Therefore \(\theta \) is the pseudoinverse of the inputs, multiplied by the outputs. Or:

\(\theta = X^+y\)

The pseudoinverse satisfies:

\(XX^+X=X\)

\(X^+XX^+=X^+\)


\subsection{Derivative of a Laplace Transform}
\noindent
Consider the $n^{th}$ derivative of the Laplace transform of $f$,
\begin{equation*}
	\frac{\mathrm{d}^n}{\mathrm{d}s^n}\left(\Laplace{f(t)}\right) = \frac{\mathrm{d}^n}{\mathrm{d}s^n}\left(\int_{0}^{\infty}{f(t)e^{-st} \mathrm{d}t}\right).
\end{equation*}
We're able to change the order of differentiation and integration here without affecting the result.
\begin{align*}
	&= \int_{0}^{\infty}{f(t)\frac{\mathrm{d}^n}{\mathrm{d}s^n}\left(e^{-st}\right) \mathrm{d}t} \\
	&= \int_{0}^{\infty}{f(t)(-t)^ne^{-st} \mathrm{d}t} \\
	&= (-1)^n\Laplace{t^nf(t)}.
\end{align*}
So,
\begin{equation*}
	\frac{\mathrm{d}^n}{\mathrm{d}s^n}\left(\Laplace{f(t)}\right) = (-1)^n\Laplace{t^nf(t)}.
\end{equation*}
This formula is useful in both directions: finding the derivatives of Laplace transforms, and finding the Laplace transforms of functions multiplied by $t^n$.


\section{Proof Methods}

\frame{{Part 2: Proof Methods}

\tableofcontents[currentsection,hideallsubsections, firstsection=2, sections={2-4}]}

\subsection{Inference Rules}

\begin{frame}{Inference Rules}{}
  Inference (or logic deductions) are used to prove new propositions by using propositions that have been proposed before.
  \bigskip

  We normally write an inference as follows:

  \[
  \frac{P, Q, R}{X}
  \]

  This means "propositions P, Q, R are true, meaning that proposition X is true".\bigskip

  \structure{Inference Rules} are inferences that are particularly useful to build proofs. Let's see a few:
\end{frame}

\begin{frame}{Inference Rules}{Modus Ponens}
  The \emph{Modus Ponens} inference rule is:
  \[
  \frac{P, P\implies Q}{Q}
  \]
  If P is true, and P implies Q is true, then Q is true.\bigskip

  A few other related inference rules:
  \[
  \frac{P \implies Q, Q \implies R}{P \implies R},
  \frac{not(P) \implies not(Q)}{Q \implies P}
  \]

  So one way to prove a proposition is to {\bf start with propositions that you know are true} and {\bf use inference rules to reach the proposition you want to prove}.
\end{frame}

\subsection{Proving Implications}

\begin{frame}{Proving an Implication}{Direct Proof}
  The \emph{Modus Ponens} rule says that:
  \[
  \frac{P, P\implies Q}{Q}
  \]
  To prove $Q$, we have to prove that $P$, and that {\bf $P$ implies $Q$}.\bigskip

  We can prove an implication directly, by assuming $P$ is true, and showing that $Q$ must be true, step by step.\bigskip
\end{frame}

\begin{frame}{Proving an Implication}{Direct Proof}
{\bf Theorem:} If $0 \leq x \leq 2$, then $-x^3 + 4x + 1 > 0$\bigskip

\begin{proof}
\begin{itemize}
  \item Let's assume $0 \leq x \leq 2$
  \item We can rewrite $-x^3 + 4x$ as $x(2-x)(2+x)$
  \item If $0 \leq x \leq 2$, then $x$, $(2-x)$, $(2+x)$ are all non-negative.
  \item $x(2-x)(2+x) \geq 0$
  \item $x(2-x)(2+x) + 1 > 0$
  \item $-x^3 + 4x + 1 > 0$
\end{itemize}
\end{proof}

\end{frame}

\begin{frame}{Proving an Implication}{Contrapositive}
  Another way to prove an implication is to "prove the contrapositive". This means using the following inference rule:

  \[
    \frac{\text{NOT}(Q) \implies \text{NOT}(P)}{P \implies Q}
  \]

  So if we show that when Q is false, then P is always false, it is equivalent to show that when P is true, then Q is always true.
\end{frame}

\begin{frame}{Proving an Implication}{Contrapositive}
  {\bf Theorem:} if $r$ is irrational, then $\sqrt{r}$ is also irrational.

  \begin{proof}
    We prove the contrapositive: If $\sqrt{r}$ is rational, then $r$ is also rational.
    \begin{itemize}
      \item If $\sqrt{r}$ is rational, then $\sqrt{r}=\frac{m}{n}$.
      \item $m$ and $n$ are integers (definition of rational numbers)
      \item Square both sides: $r = \frac{m^2}{n^2}$.
      \item $m^2$ and $n^2$ are also integers, so $r$ is rational.
    \end{itemize}
  \end{proof}
\end{frame}

\begin{frame}{Proving "If and only If"}
  Remember that "If and only If" can be defined as:
  \[
  \frac{P\implies Q, Q \implies P}{P\iff Q}
  \]

  So to prove $P\iff Q$, we can first prove the implication from $P$ to $Q$, and then prove the implication from $Q$ to $P$.\bigskip

  This is useful to show equivalence between two mathematical statements.
\end{frame}

\subsection{Proof by Cases}

\begin{frame}[fragile]{Proof By Cases}{Example}

  Let's say you are refactoring code, and you want to profe that the two code samples below are equivalent. How would you do it?
  \vfill

  \begin{block}{Code 1}
\begin{verbatim}
If (X > 0 OR (X <= 0 AND Y > 100))
  print("Hello!")
\end{verbatim}
  \end{block}
  \begin{block}{Code 2}
\begin{verbatim}
If (X > 0 OR Y > 100)
  print("Hello!")
\end{verbatim}
  \end{block}
\end{frame}

\begin{frame}{Proof By Cases}{Definition}
  \structure{Proof By Cases}, is a proof technique that uses the idea of "divide and conquer".\vfill

  You break one complicated problem into easier, smaller sub-problems.
  \vfill

  \alert{Important!} When you create the cases, make sure that all possible cases are covered!
\end{frame}

\begin{frame}{Example: Friends and Strangers}

  {\bf Theorem:} In a group of 6 people, where {\bf every pair} is either a friend or a stranger, then we {\bf always} have at least a set of \structure{3 mutual friends} or a set of \alert{3 mutual strangers}.

  \begin{center}
    \includegraphics[width=0.6\textwidth]{../img/friends_and_strangers}
  \end{center}
  \pagenote{Friends or Strangers Image from "Math for Computer Science" MIT-OCW slides}
\end{frame}

\begin{frame}{Example: Friends and Strangers}
  \begin{proof}
    The proof is by case analysis. Let "A" be one of the six people. There are two cases:
    \begin{enumerate}
      \item Among the 5 other people, at least 3 are friends with A;
      \item Among the 5 other people, at least 3 are strangers with A;
    \end{enumerate}\medskip

    Let's assume case (1). Let's call the three friends B, C, D. There are two subcases:
    \begin{enumerate}[A]
      \item B-C, C-D, or B-D are friends. We have now \structure{3 mutual friends} with A and the pair here.
      \item B-C, C-D and B-D are strangers. This makes a \structure{3 mutual strangers} set with the three pairs.
    \end{enumerate}\medskip

    This means that in case 1, the theorem holds. It is easy to see that case 2 is symmetrical to case 1.
  \end{proof}
\end{frame}

\begin{frame}{A WRONG Proof By Cases}

  {\bf Theorem:} $2a^2 > a$, for all $a\in \mathbb{Z}$.

  \begin{proof}
    The proof is by case analysis.
  \begin{enumerate}
    \item Case 1: $a > 0$;
    \begin{itemize}
      \item $2a^2$ is equal to $2a\times a$
      \item Since $a > 0$ and $a \in \mathbb{Z}$, then $a \geq 1$
      \item $2\times 1\times 1 > 1$
    \end{itemize}
    \item Case 2: $a < 0$
    \begin{itemize}
      \item Since $a < 0$ and $a \in \mathbb{Z}$, then $a \leq 1$
      \item For any negative $a$, $a^2$ is positive, so $a^2 > a$.
    \end{itemize}
    \end{enumerate}
    Because the theorem holds for case (1) and case (2), it holds for all possible cases.
  \end{proof}
  \bigskip

  What is wrong with this proof?
\end{frame}

\subsection{Proof by Contradiction}
\begin{frame}{Proof By Contradiction}{Definition}

  "Proof by Contradiction" is a technique where you show that {\bf the negative of the theorem implies a false fact to be true}.\bigskip

  For a simple example: "If gravity did not exist, then we would all be flying. Since we are not flying, then gravity must exist."\bigskip

  Sometimes, it can be easy to create a proof by contradiction by finding a good counter-example. Other times, we have to find an absurd consequence of the negative.\bigskip

  Use "Proof by Contradiction" to prove the following theorem:\\
  {\bf Theorem:} $\sqrt{2}$ is an irrational number.
\end{frame}

\begin{frame}{Proof by Contradiction}{Example}

  \begin{proof}
    We use proof by contradiction, and assume $\sqrt{2}$ is rational.
  \begin{enumerate}
  \item $\sqrt{2} = \frac{m}{n}$; $m,n \in \mathbb{Z}$; $n\neq 0$, and $m,n$ have no common factors.
  \item $n\sqrt{2} = m$ and squaring both sides give $2n^2 = m^2$.
  \item $m^2$ is even (because $n^2 = \frac{m^2}{2}$)
  \item If $m^2$ is even, then $m$ is even too. So $m = 2k$ for some integer $k$.
  \item So, $2n^2 = (2k)^2$, which leads to $n^2 = 2k^2$.
  \item Following the logic of (3) and (4), $n^2$ is even, and $n$ is even too.
  \item However, if $m$ and $n$ are even, it is a contradiction with (1).
  \end{enumerate}
  \end{proof}
\end{frame}

\subsection{The Well Ordering Outline}

\begin{frame}{Well Ordering Principle}{Definition}

  The Well Ordering Principle (WOP) is a very useful principle in mathematics, that can also look a little bit "obvious":\bigskip

  {\Large
  \begin{center}
    Every \structure{non-empty} set of\\
    \structure{Non-negative Integer Numbers} ($\mathbb{Z}^+$)\\
    has \alert{one smallest element}
  \end{center}}
\end{frame}

\begin{frame}{Well Ordering Principle}
  \frametitle{Well Ordering Examples}
    \begin{itemize}
      \item What is the smallest age among students in Tsukuba?
      \bigskip

      \item What is the smallest number of coins that adds to 876 yens?
      \bigskip

      \item What are the smallest integers $m$ and $n$ so that $x = \frac{m}{n}$?
    \end{itemize}
\end{frame}

\begin{frame}{Well Ordering Principle Proof Example}{}
  We can re-write the proof that $\sqrt{2}$ is irrational using WOP.

  \begin{proof}
  \begin{enumerate}
  \item $\sqrt{2} = \frac{m}{n}$; $m,n \in \mathbb{Z}$; $n\neq 0$;
  \item \alert{By WOP, there is a {\bf smallest} $m$ and $n$ so that $\sqrt{2} = \frac{m}{n}$}
  \item $n\sqrt{2} = m$ and squaring both sides give $2n^2 = m^2$.
  \item $m^2$ is even (because $n^2 = \frac{m^2}{2}$)
  \item If $m^2$ is even, then $m$ is even too. So $m = 2k$ for some integer $k$.
  \item So, $2n^2 = (2k)^2$, which leads to $n^2 = 2k^2$.
  \item Following the logic of (4) and (5), $n^2$ is even, and $n$ is even too.
  \item If $m$ and $n$ are even, then $\sqrt{2} = \frac{m/2}{n/2}$, and \alert{$m/2$, $n/2$ are smaller than $m,n$, contradicting the WOP}.
  \end{enumerate}
  \end{proof}
\end{frame}

\begin{frame}{Why is the WOP useful?}{General form for a WOP proof}

  The WOP gives us a general framework to produce proofs by contradiction:

  \begin{itemize}
    \item Structure your theorem around predicate $P(n)$, where $n \in \mathbb{N}$.
    \item Define a set $C$ of counter examples, so that $C ::=\{n \in \mathbb{N}| P(n) \text{ is false}\}$.
    \item By WOP, consider the minimum element $m \in C$.
    \item Find a contradiction, for example:
    \begin{itemize}
      \item if $m$ exists, then it implies in the existence of a smaller element $m' < m, m' \in C$.
      \item if $m$ exists, then actually $P(m)$ is true, and $m$ is not actually in $C$.
    \end{itemize}
    \item Therefore, the minimum element $m$ does not exist, the counter example set $C$ does not exist, and $P(n)$ is true for all $n$.
  \end{itemize}
\end{frame}

\begin{frame}{WOP Proof examples:}
  Let's see two quick examples of proofs using WOP. Try doing these two proofs by yourself first:
  \vfill

  \begin{itemize}
    \item {\bf Theorem:} Every $n > 1, n\in\mathbb{N}$ is a product of prime numbers.\bigskip

    \item {\bf Theorem:} For every $n\in\mathbb{N}$, $P(n): n+8 = 5a+3b; a,b \in\mathbb{N}$.\\(for every $n$, $n+8$ is composed of a sum of 3s and 5s)
  \end{itemize}
\end{frame}

\begin{frame}{WOP Proof example I: Prime factors}

  {\bf Theorem:} Every integers bigger than 1 is a product of prime numbers.
  \begin{proof}
    Proof by contradiction using the WOP.
    \begin{itemize}
      \item Assume, by WOP, that $m$ is the smallest $\mathbb{N}$ that is not a product of prime numbers.
      \item Obviously $m$ is not a prime, so $m = a_1a_2\ldots a_n$, where $a_i$ is not prime.
      \item Is $a_i$ a product of prime numbers?
      \begin{itemize}
        \item If $a_i$ is a product of prime numbers, then $a_i$ = $p_1p_2\ldots p_n$, and $m$ is now a product of prime numbers (\alert{contradiction})
        \item If $a_i$ is not a product of prime numbers, then $m$ is not the {\bf smallest} product of prime numbers. (\alert{contradiction})
      \end{itemize}
    \end{itemize}
  \end{proof}
\end{frame}

\begin{frame}{WOP Proof example II: Postal Numbers}
  {\bf Theorem:} For every $n$, $n+8$ is composed of 3s and 5s.

  \begin{proof}
    Proof by contradiction using the WOP
    \begin{itemize}
      \item First, we quickly verify that $P(n)$ is true for 0..8
      \item By WOP, we assume that there is some minimum $m > 8$ where $P(m)$ is false.
      \item If $P(m)$ is false, then $m+8$ cannot be composed of 3s and 5s.
      \item If $m$ is minimum, then $P(m-8)$ is true, and $m$ is composed of 3s and 5s.
      \item If $m$ is composed of 3s and 5s, then $m+8$ is $m+3+5$, and $P(m)$ is true! (\alert{Contradiction})
    \end{itemize}
  \end{proof}
\end{frame}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% CS624: Analysis of Algorithms
% Copyright 2015 Pejman Ghorbanzade <pejman@ghorbanzade.com>
% Creative Commons Attribution-ShareAlike 4.0 International License
% More info: https://github.com/ghorbanzade/beacon
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Question 1}

Show that for any integer $n \geq 0$,
\begin{equation}
\sum_{k=0}^{n} {n \choose k}k = n2^{n-1}
\end{equation}

\subsection*{Solution}

Proof is given by differentiating over binomial formula, given in Equation \ref{eq2}.

\begin{equation}
(x + y)^n = \sum_{k=0}^{n}{n \choose k}x^ky^{n-k}
\label{eq2}
\end{equation}

Substituting $1$ for $y$ leads to

\begin{equation}
(1 + x)^n = \sum_{k=0}^{n}{n \choose k}x^k
\label{eq3}
\end{equation}

where $(1 + x)^n$ is simply the ordinary generating function for a finite sequence of binomial coefficients.
By differentiating Equation \ref{eq3} over $x$, we will achieve

\begin{equation}
n(1+x)^{n-1} = \sum_{k=0}^{n} {n \choose k}kx^{k-1}
\end{equation}

Now substituting $1$ for $x$ will give us

\begin{equation}
\sum_{k=0}^{n}{n \choose k}k = n2^{n-1}
\end{equation}

and all equations are valid for any integer $n \geq 0$.


\subsection{Symmetric and antisymmetric tensors}

Consider a tensor, e g \(T_{abc}\).

In general, this is not symmetric, that is:

\(T_{abc}\ne T_{bac}\)

\subsubsection{Symmetric part of a tensor}

We can write the symmetric part of this with regard to \(a\) and \(b\).

\(T_{(ab)c}=\dfrac{1}{2}(T_{abc}+T_{bac})\)

Clearly, \(T_{(ab)c}=T_{(ba)c}\)

\subsubsection{Antisymmetric part of a tensor}

We can also have an an antisymmetric part with regard to \(a\) and \(b\).

\(T_{[ab]c}=\dfrac{1}{2}(T_{abc}-T_{bac})\)

Clearly, \(T_{[ab]c}=-T_{[ba]c}\)

\subsubsection{Tensors as sums of their symmetric and antisymmetric parts}

\(T_{(ab)c}+T_{[ab]c}=\dfrac{1}{2}(T_{abc}+T_{bac})+\dfrac{1}{2}(T_{abc}-T_{bac})\)

\(T_{(ab)c}+T_{[ab]c}=T_{abc}\)


\lab{The Finite Difference Method}{The Finite Difference Method}
\label{lab:finitedifference2}

Suppose we have a function $u(x)$, defined on an interval $[a,b]$.
Let $a = x_{-1}, x_0, x_1, \ldots x_{n-1}$ be a grid of evenly spaced points, with $x_i = a + (i+1)h$, $h = (b-a)/n$.
From a Taylor expansion of $u(x)$, we obtain the approximation
\[u''(x) = \frac{u(x+h) - 2u(x) + u(x-h)}{h^2}, \]
with error $E(h) = \mathcal{O}(h^2)$.
Thus
\[u''(x) = \frac{u(x_{i+1}) - 2u(x_i) + u(x_{i-1})}{h^2}, \quad i = 0, \ldots, n-2.\]
This can be written in matrix form as
\begin{align}
\frac{1}{h^2}
\begin{bmatrix}
-2 & 1 & \\
1 & -2 & 1  \\
& &\ddots & \\
 & 1 & -2 & 1 \\
 & & 1 & -2
\end{bmatrix}
\begin{bmatrix}
u(x_0) \\ u(x_1)\\ \vdots  \\ u(x_{n-3}) \\ u(x_{n-2})
\end{bmatrix} +
\begin{bmatrix}
u(x_{-1})/h^2 \\ 0 \\ \vdots  \\ 0 \\ u(x_{n-1})/h^2
\end{bmatrix} =
\begin{bmatrix}
u''(x_0) \\ u''(x_1)\\ \vdots  \\ u''(x_{n-3}) \\ u''(x_{n-2})
\end{bmatrix}\label{finitedifference2:upp}
\end{align}

Similarly, since  $u'(x) = (u(x+h) - u(x-h))/(2h)$ with error $E(h) = \mathcal{O}(h^2)$, we have the approximation
\begin{align}
\frac{1}{2h}
\begin{bmatrix}
0 & 1 & \\
-1 & 0 & 1  \\
& &\ddots & \\
 & -1 & 0 & 1 \\
 & & -1 & 0
\end{bmatrix}
\begin{bmatrix}
u(x_0) \\ u(x_1)\\ \vdots  \\ u(x_{n-3}) \\ u(x_{n-2})
\end{bmatrix} +
\begin{bmatrix}
-u(x_{-1})/(2h) \\ 0 \\ \vdots  \\ 0 \\ u(x_{n-1})/(2h)
\end{bmatrix} =
\begin{bmatrix}
u'(x_0) \\ u'(x_1)\\ \vdots  \\ u'(x_{n-3}) \\ u'(x_{n-2})
\end{bmatrix}\label{finitedifference2:up}
\end{align}

\begin{problem}
	Let $u(x) = \sin((x+\pi)^2-1)$. Use \eqref{finitedifference2:upp} and \eqref{finitedifference2:up} to approximate $\frac{1}{2}u'' - u'$ at the grid points. \label{prob:finitedifference2:prob1}
\end{problem}

Suppose that instead of knowing the function $u(x)$, we know that $\frac{1}{2}u'' - u = f$, where the function $f(x)$ is given.
How do we solve for $u$ at the grid points?

\section*{Finite Difference Methods}

Numerical methods for differential equations seek to approximate the exact solution $u(x)$ at some finite collection of points in the domain of the problem.
Instead of analytically solving the original differential equation, defined over an infinite-dimensional function space, they use a simpler finite system of algebraic equations to approximate the original problem.

Consider the following differential equation:
\begin{align}
	\begin{split}
	&{ } \epsilon u''(x)-u(x)'= f(x), \quad x \in (0,1), \\
	&{ } u(0) = 1, \quad u(1) = 3.
	\end{split} \label{eqn:singular_perturbed_BVP}
\end{align}
Equation \eqref{eqn:singular_perturbed_BVP} can be written $D u = f,$ where $D = \epsilon \frac{d^2}{dx^2} - \frac{d}{dx}$ is a differential operator defined on the infinite-dimensional space of functions that are twice continuously differentiable on $[0,1]$ and satisfy $u(0) = 1$, $u(1) = 3$.

We look for an approximate solution $\{U_i\}_{i=-1}^{N-1}$ on an evenly spaced grid of $N$ subintervals, $a = x_{-1}, x_0, \ldots, x_{N-1} = b$ with $h = x_{i+1}-x_i$ for each $i$.
Our finite difference method will replace the differential operator $D = \epsilon \frac{d^2}{dx^2} - \frac{d}{dx}$, defined on an infinite-dimensional space of functions, with difference operators defined on a finite vector space (the space of grid functions $\{U_i\}_{i=-1}^{N-1}$).
To do this, we replace derivative terms in the differential equation with appropriate difference expressions.

Recalling that
\begin{align*}
	\frac{d^2}{dx^2} u(x) &= \frac{u(x+h)- 2u(x) + u(x-h)}{h^2} + \mathcal{O}(h^2),\\
\frac{d}{dx} u(x) &= \frac{u(x+h)-u(x-h)}{2h} + \mathcal{O}(h^2).
\end{align*}
we define the finite difference operator $D_h$ by
\begin{align}
D_h U_i = \epsilon \frac{1}{h^2}\left(U_{i+1} -2U_i + U_{i-1}\right) - \frac{1}{2h} \left(U_{i+1}-U_{i-1} \right). \label{fd_operator}
\end{align}
Thus we discretize equation \eqref{eqn:singular_perturbed_BVP} using the equations
\begin{align*}
	\frac{\epsilon}{h^2} (U_{i+1}- 2U_i + U_{i-1}) - \frac{1}{2h} \left(U_{i+1}-U_{i-1} \right)  &= f(x_i), \quad i = 0, \ldots, N-2,
\end{align*}
along with boundary conditions $U_{-1} = 1,$ $U_{N-1} = 3$.

This gives $N+1$ equations and $N+1$ unknowns, and can be written in matrix form as
\[ \frac{1}{h^2} \begin{bmatrix}h^2 & 0 &0&\hdots &0 \\ (\epsilon + h/2) &-2\epsilon & (\epsilon - h/2) &\hdots &0\\ \vdots &  & \ddots & &\vdots \\
0 & \hdots & (\epsilon + h/2) &-2\epsilon & (\epsilon - h/2) \\ 0 & \hdots & & 0 & h^2
\end{bmatrix} \cdot \begin{bmatrix}U_{-1}\\U_0\\ \vdots \\U_{N-2} \\U_{N-1}\end{bmatrix} = \begin{bmatrix}f(x_{-1})\\f(x_0)\\ \vdots \\ f(x_{N-2}) \\ f(x_{N-1}) \end{bmatrix}.
\]
We can further modify the system to obtain an $(N-1)\times (N-1)$ tridiagonal matrix on the left:
\begin{align}
    \label{eqn:practical_fin_dif}
    \begin{split}
\frac{1}{h^2} \begin{bmatrix}-2\epsilon & (\epsilon - h/2) &0&\hdots &0 \\ (\epsilon + h/2) &-2\epsilon & (\epsilon - h/2) &\hdots &0\\ \vdots &  & \ddots & &\vdots \\
0 & \hdots & (\epsilon + h/2) &-2\epsilon & (\epsilon - h/2) \\ 0 & \hdots & & (\epsilon + h/2) &-2\epsilon
\end{bmatrix} \cdot \begin{bmatrix}U_{0}\\U_1\\ \vdots \\U_{N-3} \\U_{N-2}\end{bmatrix}\\
 = \begin{bmatrix}f(x_0) -\alpha(\epsilon + h/2)/h^2 \\f(x_1)\\ \vdots \\ f(x_{N-3})\\ f(x_{N-2})-\beta(\epsilon - h/2)/h^2 \end{bmatrix}.
    \end{split}
\end{align}

\begin{problem}
Use equation \eqref{eqn:practical_fin_dif} to solve the singularly perturbed BVP \eqref{eqn:singular_perturbed_BVP} with $\epsilon = 1/10$, $f(x) = -1$.
This BVP is called singularly perturbed because of the location of the parameter $\epsilon$.
For $\epsilon = 0$ the ODE has a drastically different character - it then becomes first order, and can no longer support two boundary conditions.
    \label{prob:finitedifference2:prob1}
\end{problem}

\begin{figure}[h]
\centering
\includegraphics[width=10cm]{figure2.pdf}
\caption{The solution to Problem \ref{prob:finitedifference2:prob1}.
The solution gets steeper near $x = 1$ as $\epsilon $ gets small.}
\end{figure}

\begin{comment}
\begin{lstlisting}
import numpy as np
from scipy.sparse import spdiags
from scipy.sparse.linalg import spsolve
import matplotlib.pyplot as plt

def bvp(func, epsilon, alpha, beta, N):
	a,b = 0., 1. 	# Interval for the BVP
	h = (b-a)/N 	# The length of each subinterval

	# Initialize and define the vector F on the right
	F = np.empty(N-1.)
	F[0] = func(a+1.*h)-alpha*(epsilon+h/2.)*h**(-2.)
	F[N-2] = func(a+(N-1)*h)-beta*(epsilon-h/2.)*h**(-2.)
	for j in xrange(1,N-2):
		F[j] = func(a + (j+1)*h)

	# Here we define the arrays that will go on the diagonals of A
	data = np.empty((3,N-1))
	data[0,:] = -2.*epsilon*np.ones((1,N-1)) # main diagonal
	data[1,:]  = (epsilon+h/2.)*np.ones((1,N-1))  	 # off-diagonals
	data[2,:] = (epsilon-h/2.)*np.ones((1,N-1))
	# Next we specify on which diagonals they will be placed, and create A
	diags = np.array([0,-1,1])
	A=h**(-2.)*spdiags(data,diags,N-1,N-1).asformat('csr')

	U = np.empty(N+1)
	U[1:-1] = spsolve(A,F)
	U[0], U[-1] = alpha, beta
	return np.linspace(a,b,N+1), U

x, y = bvp(lambda x:-1., epsilon=.05,alpha=1, beta=3, N=400)
plt.plot(x,y,'-k',linewidth=2.0)
plt.show()

\end{lstlisting}
\end{comment}

\section*{A heuristic test for convergence}
\begin{figure}[ht]
\centering
\includegraphics[width=12cm]{example_convergence.pdf}
\caption{Demonstration of second order convergence for the finite difference approximation \eqref{fd_operator} of the BVP given in \eqref{eqn:singular_perturbed_BVP} with $\epsilon = .5$. } \label{fig:finitedifference2}
\end{figure}

The finite differences used above are second order approximations of the first and second derivatives of a function.  It seems reasonable to expect that the numerical solution would converge at a rate of about $\mathcal{O}(h^2)$. How can we check that a numerical approximation is reasonable?

% One way to determine this is to compute solutions for various step sizes $h$ and see if the solutions are converging to something.
Suppose a finite difference method is $\mathcal{O}(h^p)$ accurate.
This means that the error $E(h) \approx Ch^p$ for some constant $C$ as $h \to 0$ (in other words, for $h>0$ small enough).

So compute the approximation $y_k$ for each stepsize $h_k$, $h_1 > h_2> \ldots>h_m$.
$y_m$ should be the most accurate approximation, and will be thought of as the true solution.
Then the error of the approximation for
stepsize $h_k, k < m,$ is
\begin{align*}
	E(h_k) &= \max( \abs{ y_k - y_m}) \approx C h_k^p ,\\
	\log(E(h_k)) &= \log(C) + p \log(h_k).
\end{align*}

Thus on a log-log plot of $E(h)$ vs. $h$, these values should be on a straight line with slope $p$ when $h$ is small enough to start getting convergence. We should note that demonstrating second-order convergence does NOT imply that the numerical approximation is converging to the correct solution.

\begin{problem}
Return to problem \ref{prob:finitedifference2:prob1}.   How many subintervals are needed to obtain 4 digits of accuracy?

This is a question about the convergence of your solution.  The following code generates the log-log plot in Figure \ref{fig:finitedifference2}, and demonstrates second-order convergence for our finite difference approximation of \eqref{eqn:singular_perturbed_BVP}. Use this code to determine what $h$ (and hence what $N$) is needed for the error to be less than $10^{-4}$.

\textsc{note}: The function \li{bvp} is not provided; you need to use your code from problem \ref{prob:finitedifference2:prob1} to define it.  Make sure your function is compatible with the code below.  It must take 5 parameters as input and return both the solution and the grid.

	% If $\alpha = 1$,  $\beta = 3$, and $f(x) = -1$, there is an exact solution:
	% \[y(x) = \alpha + x+ (\beta - \alpha -1)\frac{e^{x/\epsilon -1}}{e^{1/\epsilon -1}}\]
	\label{prob:finitedifference2:prob2}
\end{problem}

\begin{lstlisting}
num_approx = 10 # Number of Approximations
N = 5*np.array([2**j for j in range(num_approx)])
h, max_error = (1.-0)/N[:-1], np.ones(num_approx-1)

# Best numerical solution, used to approximate the true solution.
# bvp returns the grid, and the grid function, approximating the solution
# with N subintervals of equal length.
mesh_best, num_sol_best = bvp(lambda x:-1, epsilon=.1, alpha=1, beta=3, N=N[-1])
for j in range(len(N)-1):
    mesh, num_sol = bvp(lambda x:-1, epsilon=.1, alpha=1, beta=3, N=N[j])
    max_error[j] = np.max(np.abs( num_sol- num_sol_best[::2**(num_approx-j-1)] ) )
plt.loglog(h,max_error,'.-r',label="$E(h)$")
plt.loglog(h,h**(2.),'-k',label="$h^{\, 2}$")
plt.xlabel("$h$")
plt.legend(loc='best')
plt.show()
print "The order of the finite difference approximation is about ", ( (np.log(max_error[0]) -
    np.log(max_error[-1]) )/( np.log(h[0]) - np.log(h[-1]) ) ), "."
\end{lstlisting}

\begin{problem}
Extend your finite difference code to the case of a general second order linear BVP with Dirichlet conditions:
\begin{align*}
	&{ } a_1(x)y'' +a_2(x)y'+ a_3(x) y = f(x), \quad x \in (a,b),\\
	&{ } y(a) = \alpha, \quad y(b) = \beta.
\end{align*}
Use your code to solve the boundary value problem
\begin{align*}
	\epsilon y'' - 4(\pi - x^2)y = \cos x, \\
	y(0) = 0, \quad y(\pi/2) = 1,
\end{align*}
for $\epsilon = 0.1$.
\label{prob:finitedifference2:prob3}
(Hint: How should the finite difference operator $D_h$ in \eqref{fd_operator} be modified?)

\end{problem}

\begin{figure}[h]
\centering
\includegraphics[width=10cm]{figure3.pdf}
\caption{The solution to Problem \ref{prob:finitedifference2:prob3}.
% The solution gets steeper near $x = 1$ as $\epsilon $ gets small.
}
\end{figure}

The next few problems will help you troubleshoot your finite difference code.

\begin{problem}
Numerically solve the boundary value problem
\begin{align*}
	\epsilon y'' + xy' = -\epsilon \pi^2 \cos(\pi x) - \pi x \sin(\pi x), \\
	y(-1) = -2, \quad y(1) = 0,
\end{align*}
for $\epsilon = 0.1, 0.01,$ and $0.001$.
\label{prob:finitedifference2:prob4}
\end{problem}

\begin{problem}
Numerically solve the boundary value problem
\begin{align*}
	(\epsilon +x^2)y'' + 4xy' + 2y = 0, \\
	y(-1) = 1/(1+\epsilon), \quad y(1) = 1/(1+\epsilon),
\end{align*}
for $\epsilon = 0.05, 0.02$.
\label{prob:finitedifference2:prob5}
\end{problem}

\begin{figure}[h]
\centering
\includegraphics[width=11cm]{figure4.pdf}
\caption{The solution to Problem \ref{prob:finitedifference2:prob4}.
}
\end{figure}

\begin{figure}
\centering
\includegraphics[width=11cm]{figure5.pdf}
\caption{The solution to Problem \ref{prob:finitedifference2:prob5}.
}
\end{figure}


\section{Convolution}
For simplicity of exposition, we denote
\begin{equation}\label{eq:size_f}
m = m_1 = 2^{s} + 1,  \quad n = n_1 = 2^t  +1.
\end{equation}
\begin{definition}
A convolution defined on $\mathbb{R}^{m\times n}$ is a linear mapping
$K\ast: \mathbb{R}^{m\times n}\mapsto \mathbb{R}^{m\times n}$ defined with padding,
for any $g \in \mathbb{R}^{m\times n}$ by:
%We first consider $\theta$ a convolution operator (with stride $1$)
%and padding:
\begin{equation}\label{con1}
[K \ast g]_{i,j} = \sum_{p,q=-k}^k K_{p, q} g_{i + p, j + q}, \quad i=1:m, j = 1:n.
\end{equation}
\end{definition}
Here we note that the indices for the entries in $K$ are given un a special way.
For example, if $k=1, K\in \mathbb R^{3\times 3}$, and
$$
K=\begin{pmatrix}
	K_{-1,-1} &K_{-1,0} &K_{-1,1} \\
	K_{0,-1} &K_{0,0} &K_{0,1} \\
	K_{1,-1} &K_{1,0} &K_{1,1} \\
	\end{pmatrix}.
$$

The coefficients in \eqref{con1} constitute  a kernel matrix
\begin{equation}
K \in \mathbb{R}^{(2k+1) \times (2k+1)},
\end{equation}
where $k$ is often taken as a small integer.
Here padding means how $ g_{i+ p, j + q}$ is defined
when $(i+ p, j + q)$ is out of $1:m$ or $1:n$.
The following three choices are often used
\begin{equation}\label{eq:padding}
f_{i + p, j + q} = \begin{cases}
0,  \quad &\text{zero padding}, \\
f_{(i + p)\pmod{m}, (s + q)\pmod{n}},  \quad &\text{periodic padding}, \\
f_{|i-1 +p|, |j -1  +q|},  \quad &\text{reflected padding}, \\
\end{cases}
\end{equation}
if
\begin{equation}
i + p \notin \{1, 2, \dots, m\} ~\text{or} ~  j+ q \notin \{1, 2, \dots, n\}.
\end{equation}
Here $ d \pmod{m} \in \{1, \cdots, m\} $  means the remainder when $d$ is divided by $m$.

\section{Convolution with stride}
\begin{definition}
For any given integer $s\ge1$, a convolution with stride $s$ for $g \in \mathbb{R}^{m\times n}$ is defined as:
	\begin{equation}\label{stride}
	[K \ast_s g]_{i,j} = \sum_{p,q=-k}^k K_{p,q} g_{si + p, sj + q},
	\quad i = 1: \lceil  \frac{m+1}{s}\rceil, j = 1: \lceil  \frac{n+1}{s}\rceil.
	\end{equation}
	Here $ \lceil  \frac{m}{s}\rceil$ denotes the smallest integer that greater than $\frac{m}{s}$.
\end{definition}
Both in CNN and in multigrid method, convolution with stride $s=2$ is the most important case, namely
\begin{equation}\label{stride_2}
[K \ast_2 g]_{i,j} = \sum_{p,q=-k}^k K_{p,q} g_{2i + p, 2j + q},
\quad i = 1: \frac{m+1}{2} , j = 1: \frac{n+1}{2}.
\end{equation}

\begin{lemma}
The convolution with stride $2$ can be rewritten as:
\begin{equation}\label{eq:convstride_2_1}
K \ast_2 g = \mathcal S( K\ast g),
\end{equation}
where $\mathcal S$ is a stride operator defined by:
\begin{equation}\label{eq:strideopdim}
\mathcal S: \mathbb{R}^{m \times n} \mapsto \mathbb{R}^{\frac{m+1}{2} \times \frac{n+1}{2}},
\end{equation}
with
\begin{equation}\label{eq:strideop}
[\mathcal S(g)]_{i,j} = g_{2i, 2j}, \quad i  = 1: \frac{m+1}{2} j = 1: \frac{n+1}{2}.
\end{equation}
\end{lemma}