\subsection{Matrix multiplication}

The multiplication of a matrix and a vector from the previous section
is a special case of the operation of multiplying two matrices, which
we now define.

\begin{definition}{Matrix multiplication}{matrix-multiplication}
  Let $A=\mat{a_{ij}}$ be an $m\times n$-matrix, and let
  $B=\mat{b_{jk}}$ be an $n\times p$-matrix. Then their product%
  \index{matrix!multiplication}%
  \index{multiplication!of matrices}%
  \index{product|see{multiplication}}%
  \index{matrix!multiplication!componentwise method}%
  \index{componentwise method!matrix multiplication} is the
  $m\times p$-matrix $AB=\mat{c_{ik}}$ whose $(i,k)$-entry is defined
  by
  \begin{equation*}
    c_{ik} = a_{i1}b_{1k} + a_{i2}b_{2k} + \ldots + a_{in}b_{nk}.
  \end{equation*}
\end{definition}

For matrices $A$ and $B$, in order to form the product $AB$, the
number of columns of $A$ must equal the number of rows of
$B$. Consider a product $AB$ where $A$ has dimensions $m\times n$ and
$B$ has dimensions $n \times p$. Then the dimensions of the product
are given by
\begin{equation*}
  (\overset{\text{these must match!}}{m\times\widehat{n)\;(n}\times p})=m\times p.
\end{equation*}
Note that the two outside numbers give the dimensions of the
product. If the two middle numbers do not match, we cannot multiply
the matrices.

To better visualize the rule of matrix multiplication, suppose
\begin{equation*}
  A = \begin{mymatrix}{cccc}
    a_{11} & a_{12} & \cdots & a_{1n} \\
    a_{21} & a_{22} & \cdots & a_{2n} \\
    \vdots & \vdots & \ddots & \vdots \\
    a_{m1} & a_{m2} & \cdots & a_{mn} \\
  \end{mymatrix}
  \quad\mbox{and}\quad
  B = \begin{mymatrix}{cccc}
    b_{11} & b_{12} & \cdots & b_{1p} \\
    b_{21} & b_{22} & \cdots & b_{2p} \\
    \vdots & \vdots & \ddots & \vdots \\
    b_{n1} & b_{n2} & \cdots & b_{np} \\
  \end{mymatrix}.
\end{equation*}
Then their product
\begin{equation*}
  AB = \begin{mymatrix}{cccc}
    c_{11} & c_{12} & \cdots & c_{1p} \\
    c_{21} & c_{22} & \cdots & c_{2p} \\
    \vdots & \vdots & \ddots & \vdots \\
    c_{m1} & c_{m2} & \cdots & c_{mp} \\
  \end{mymatrix}
\end{equation*}
is an $m\times p$-matrix whose $(i,k)$-entry is defined by
\begin{equation*}
  c_{ik} = a_{i1}b_{1k} + a_{i2}b_{2k} + \ldots + a_{in}b_{nk}.
\end{equation*}
Note that we can also write this as
\begin{equation*}
  c_{ik} ~=~ \begin{mymatrix}{cccc}a_{i1} & a_{i2} & \cdots & a_{in}\end{mymatrix}
  \begin{mymatrix}{c}b_{1k} \\ b_{2k} \\ \vdots \\ b_{nk}\end{mymatrix}.
\end{equation*}
In other words, the $(i,k)$-entry of the matrix product $AB$ is a kind
of dot product of the $i\th$ row of $A$ with the $k\th$ column of $B$.

\begin{example}{Matrix multiplication}{matrix-multiplication}
  Find $AB$, where
  \begin{equation*}
    A = \begin{mymatrix}{rrr}
      1 & 2 & 1 \\
      0 & 2 & 1
    \end{mymatrix}
    \quad\mbox{and}\quad
    B = \begin{mymatrix}{rrr}
      1 & 2 & 0 \\
      0 & 3 & 1 \\
      -2 & 1 & 1
    \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  First, let us note that since $A$ has size $2\times 3$ and $B$ has
  size $3\times 3$, the product $AB$ is well-defined and has size
  $2\times 3$. Let $C=AB$. We compute each of the six entries of $C$:
  \begin{itemize}
  \item The $(1,1)$-entry is the first row of $A$ times the first
    column of $B$: $c_{11} = 1\cdot 1+2\cdot 0+1\cdot(-2) = -1$.
  \item The $(1,2)$-entry is the first row of $A$ times the second
    column of $B$: $c_{12} = 1\cdot 2+2\cdot 3+1\cdot 1 = 9$.
  \item The $(1,3)$-entry is the first row of $A$ times the third
    column of $B$: $c_{13} = 1\cdot 0+2\cdot 1+1\cdot 1 = 3$.
  \item The $(2,1)$-entry is the second row of $A$ times the first
    column of $B$: $c_{21} = 0\cdot 1+2\cdot 0+1\cdot(-2) = -2$.
  \item The $(2,2)$-entry is the second row of $A$ times the second
    column of $B$: $c_{22} = 0\cdot 2+2\cdot 3+1\cdot 1 = 7$.
  \item The $(2,3)$-entry is the second row of $A$ times the third
    column of $B$: $c_{23} = 0\cdot 0+2\cdot 1+1\cdot 1 = 3$.
  \end{itemize}
  Therefore, we have
  \begin{equation*}
    AB = \begin{mymatrix}{rrr}
      -1 & 9 & 3 \\
      -2 & 7 & 3
    \end{mymatrix}.
  \end{equation*}
\end{solution}

As this example shows, calculating matrix products one component at a
time can be an extremely repetitive and tedious process. Fortunately,
we can speed this up by considering whole columns at once.

\begin{proposition}{Matrix multiplication, column method}{matrix-multiplication-columns}
  Let $A$ be an $m\times n$-matrix, and let $B$ be an
  $n\times p$-matrix.  Suppose that the columns of $B$ are
  $\vect{b}_1,\vect{b}_2,\ldots,\vect{b}_p$. Then the columns of $AB$
  are%
  \index{matrix!multiplication!column method}%
  \index{column method!matrix multiplication}
  \begin{equation*}
    A\vect{b}_1,~A\vect{b}_2,~\ldots,~A\vect{b}_p.
  \end{equation*}
  In other words, the $k\th$ column of the matrix product $AB$ is
  equal to $A$ times the $k\th$ column of $B$.
\end{proposition}

\begin{example}{Matrix multiplication by the column method}{matrix-multiplication-columns}
  Find the matrix product $AB$ by the column method, where
  \begin{equation*}
    A = \begin{mymatrix}{rrr}
      1 & 2 & 1 \\
      0 & 2 & 1
    \end{mymatrix}
    \quad\mbox{and}\quad
    B = \begin{mymatrix}{rrr}
      1 & 2 & 0 \\
      0 & 3 & 1 \\
      -2 & 1 & 1
    \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  We multiply $A$ by each of the columns of $B$:
  \begin{equation*}
    \begin{mymatrix}{rrr}
      1 & 2 & 1 \\
      0 & 2 & 1
    \end{mymatrix}
    \begin{mymatrix}{r}
      1 \\
      0 \\
      -2
    \end{mymatrix}
    ~=~
    1 \begin{mymatrix}{r} 1 \\ 0 \end{mymatrix}
    + 0 \begin{mymatrix}{r} 2 \\ 2 \end{mymatrix}
    - 2 \begin{mymatrix}{r} 1 \\ 1 \end{mymatrix}
    ~=~
    \begin{mymatrix}{r}
      -1 \\
      -2
    \end{mymatrix},
  \end{equation*}
  \begin{equation*}
    \begin{mymatrix}{rrr}
      1 & 2 & 1 \\
      0 & 2 & 1
    \end{mymatrix}
    \begin{mymatrix}{r}
      2 \\
      3 \\
      1
    \end{mymatrix}
    ~=~
    2 \begin{mymatrix}{r} 1 \\ 0 \end{mymatrix}
    + 3 \begin{mymatrix}{r} 2 \\ 2 \end{mymatrix}
    + 1 \begin{mymatrix}{r} 1 \\ 1 \end{mymatrix}
    ~=~
    \begin{mymatrix}{r}
      9 \\
      7
    \end{mymatrix},
  \end{equation*}
  \begin{equation*}
    \begin{mymatrix}{rrr}
      1 & 2 & 1 \\
      0 & 2 & 1
    \end{mymatrix}
    \begin{mymatrix}{r}
      0 \\
      1 \\
      1
    \end{mymatrix}
    ~=~
    0 \begin{mymatrix}{r} 1 \\ 0 \end{mymatrix}
    + 1 \begin{mymatrix}{r} 2 \\ 2 \end{mymatrix}
    + 1 \begin{mymatrix}{r} 1 \\ 1 \end{mymatrix}
    ~=~
    \begin{mymatrix}{r}
      3 \\
      3
    \end{mymatrix}.
  \end{equation*}
  The resulting three column vectors form the columns of $AB$.  Thus,
  \begin{equation*}
    AB ~=~ \begin{mymatrix}{rrr}
      -1 & 9 & 3 \\
      -2 & 7 & 3
    \end{mymatrix}.
  \end{equation*}
\end{solution}

Of course, the answer in
Example~\ref{exa:matrix-multiplication-columns} is the same as that in
Example~\ref{exa:matrix-multiplication}. Please convince yourself that
both methods of matrix multiplication give the same answer, since they
each ultimately calculate the same thing. Nevertheless, with a bit of
practice, the column method is much faster, and you can even learn to
multiply matrices in your head! The key to understanding the column
method is that each column of $B$ provides instructions for taking a
linear combination of the columns of $A$. The method works especially
well if $B$ contains many zeros and ones.

Since column vectors are simply $n\times 1$-matrices, and row vectors
are $1\times m$-matrices, we can also multiply a column vector by a
row vector or vice versa.

\begin{example}{Column vector times row vector}{column-times-row}
  Multiply $\begin{mymatrix}{r}
    1 \\
    2 \\
    1
  \end{mymatrix}
  \begin{mymatrix}{rrrr}
    1 & 2 & 1 & 0
  \end{mymatrix}$.
\end{example}

\begin{solution}
  Here we are multiplying a $3\times 1$-matrix by a
  $1\times 4$-matrix, so the result will be a $3\times
  4$-matrix. Using the column method, we can compute this product as
  follows:
  \begin{equation*}
    \begin{mymatrix}{r}
      1 \\
      2 \\
      1
    \end{mymatrix} \begin{mymatrix}{rrrr}
      1 & 2 & 1 & 0
    \end{mymatrix} =
    \mat{
      \overset{\text{First column}}{
        \overbrace{
          \begin{mymatrix}{r}
            1 \\
            2 \\
            1
          \end{mymatrix}
          \begin{mymatrix}{r}
            1
          \end{mymatrix}
        }
      },
      \overset{\text{Second column}}{
        \overbrace{
          \begin{mymatrix}{r}
            1 \\
            2 \\
            1
          \end{mymatrix}
          \begin{mymatrix}{r}
            2
          \end{mymatrix}
        }
      },
      \overset{\text{Third column}}{
        \overbrace{
          \begin{mymatrix}{r}
            1 \\
            2 \\
            1
          \end{mymatrix}
          \begin{mymatrix}{r}
            1
          \end{mymatrix}
        }
      },
      \overset {\text{Fourth column}}{
        \overbrace{
          \begin{mymatrix}{r}
            1 \\
            2 \\
            1
          \end{mymatrix}
          \begin{mymatrix}{r}
            0
          \end{mymatrix}
        }}
    }
    ~=~
    \begin{mymatrix}{rrrr}
      1 & 2 & 1 & 0 \\
      2 & 4 & 2 & 0 \\
      1 & 2 & 1 & 0
    \end{mymatrix}
  \end{equation*}
\end{solution}

\begin{example}{Row vector times column vector}{row-times-column}
  Multiply $\begin{mymatrix}{rrrr}
    1 & 2 & 3
  \end{mymatrix}
  \begin{mymatrix}{r}
    1 \\
    2 \\
    -1
  \end{mymatrix}$.
\end{example}

\begin{solution}
  Here we are multiplying a $1\times 3$-matrix by a $3\times
  1$-matrix, so the result will be a $1\times 1$-matrix, or in other
  words, a scalar. (We regard a scalar and a $1\times 1$-matrix as the
  same thing). We have:
  \begin{equation*}
    \begin{mymatrix}{rrrr}
      1 & 2 & 3
    \end{mymatrix}
    \begin{mymatrix}{r}
      1 \\
      2 \\
      -1
    \end{mymatrix}
    ~=~
    1\cdot 1 + 2\cdot 2 + 3\cdot(-1) = 2.
  \end{equation*}
  Therefore, multiplying a row vector by a column vector works very
  similarly to an ordinary dot product (except that the dot product is
  defined between two column vectors, not a row vector and a column
  vector).
\end{solution}

\begin{example}{A multiplication that is not defined}{undefined-matrix-multiplication}
  Find $BA$, where
  \begin{equation*}
    B = \begin{mymatrix}{rrr}
      1 & 2 & 0 \\
      0 & 3 & 1 \\
      -2 & 1 & 1
    \end{mymatrix}
    \quad\mbox{and}\quad
    A = \begin{mymatrix}{rrr}
      1 & 2 & 1 \\
      0 & 2 & 1
    \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  The product $BA$ is not defined, since $B$ is a $3\times 3$-matrix
  and $A$ is a $2\times 3$-matrix. Since the number of columns of $B$
  does not match the number of rows of $A$, the product is not defined.
\end{solution}

Notice that the matrices in
Example~\ref{exa:undefined-matrix-multiplication} are the same as
those in Example~\ref{exa:matrix-multiplication}.  This
demonstrates an important property of matrix multiplication: it is
possible that $AB$ is defined by $BA$ is undefined. Even if $AB$ and
$BA$ are both defined, they may not be equal, as the following example
shows. Therefore, matrix multiplication is not commutative.

\begin{example}{Matrix multiplication is not commutative}{matrix-multiplication-not-commutative}
  Compute $AB$ and $BA$, where
  \begin{equation*}
    A ~=~ \begin{mymatrix}{rr}
      1 & 2 \\
      3 & 4
    \end{mymatrix}
    \quad\mbox{and}\quad
    B ~=~ \begin{mymatrix}{rr}
      0 & 1 \\
      1 & 0
    \end{mymatrix}
  \end{equation*}
  Are they equal?
\end{example}

\begin{solution}
  We have
  \begin{equation*}
    AB ~=~
    \begin{mymatrix}{rr}
      1 & 2 \\
      3 & 4
    \end{mymatrix}
    \begin{mymatrix}{rr}
      0 & 1 \\
      1 & 0
    \end{mymatrix}
    ~=~
    \begin{mymatrix}{rr}
      2 & 1 \\
      4 & 3
    \end{mymatrix}
  \end{equation*}
  and
  \begin{equation*}
    BA ~=~
    \begin{mymatrix}{rr}
      0 & 1 \\
      1 & 0
    \end{mymatrix}
    \begin{mymatrix}{rr}
      1 & 2 \\
      3 & 4
    \end{mymatrix}
    ~=~
    \begin{mymatrix}{rr}
      3 & 4 \\
      1 & 2
    \end{mymatrix}.
  \end{equation*}
  Therefore, $AB$ and $BA$ are not equal. Matrix multiplication is not
  commutative.
\end{solution}

We have seen two methods for matrix multiplication: one component at a
time, and by the column method. There is also a third method, called
the row method. It is exactly symmetric to the column method.

\begin{proposition}{Matrix multiplication, row method}{matrix-multiplication-rows}
  Let $A$ be an $m\times n$-matrix, and let $B$ be an
  $n\times p$-matrix.  Suppose that the rows of $A$ are
  $\vect{a}_1,\vect{a}_2,\ldots,\vect{a}_m$. Then the rows of $AB$
  are%
  \index{matrix!multiplication!row method}%
  \index{row method!matrix multiplication}
  \begin{equation*}
    \vect{a}_1B,~\vect{a}_2B,~\ldots,~\vect{a}_mB.
  \end{equation*}
  In other words, the $i\th$ column of the matrix product $AB$ is
  equal to the $i\th$ column of $A$ times $B$.
\end{proposition}

\begin{example}{Matrix multiplication by the row method}{matrix-multiplication-rows}
  Find the matrix product $AB$ by the row method, where
  \begin{equation*}
    A = \begin{mymatrix}{rrr}
      1 & 2 & 1 \\
      0 & 2 & 1
    \end{mymatrix}
    \quad\mbox{and}\quad
    B = \begin{mymatrix}{rrr}
      1 & 2 & 0 \\
      0 & 3 & 1 \\
      -2 & 1 & 1
    \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  We multiply each of the rows of $A$ by $B$:
  \begin{equation*}
    \begin{mymatrix}{rrr}
      1 & 2 & 1
    \end{mymatrix}
    \begin{mymatrix}{rrr}
      1 & 2 & 0 \\
      0 & 3 & 1 \\
      -2 & 1 & 1
    \end{mymatrix}
    ~=~
    1 \begin{mymatrix}{rrr} 1 & 2 & 0 \end{mymatrix}
    + 2 \begin{mymatrix}{rrr} 0 & 3 & 1 \end{mymatrix}
    + 1 \begin{mymatrix}{rrr} -2 & 1 & 1 \end{mymatrix}
    ~=~
    \begin{mymatrix}{rrr} -1 & 9 & 3 \end{mymatrix},
  \end{equation*}
  \begin{equation*}
    \begin{mymatrix}{rrr}
      0 & 2 & 1
    \end{mymatrix}
    \begin{mymatrix}{rrr}
      1 & 2 & 0 \\
      0 & 3 & 1 \\
      -2 & 1 & 1
    \end{mymatrix}
    ~=~
    0 \begin{mymatrix}{rrr} 1 & 2 & 0 \end{mymatrix}
    + 2 \begin{mymatrix}{rrr} 0 & 3 & 1 \end{mymatrix}
    + 1 \begin{mymatrix}{rrr} -2 & 1 & 1 \end{mymatrix}
    ~=~
    \begin{mymatrix}{rrr} -2 & 7 & 3 \end{mymatrix}.
  \end{equation*}
  The resulting two row vectors form the rows of $AB$.  Thus,
  \begin{equation*}
    AB ~=~ \begin{mymatrix}{rrr}
      -1 & 9 & 3 \\
      -2 & 7 & 3
    \end{mymatrix}.
  \end{equation*}
  Once again this is the same answer as in
  Examples~\ref{exa:matrix-multiplication-columns} and
  {\ref{exa:matrix-multiplication}}. All three methods give the same
  result. But notice how in the row method, each row of $A$ provides
  instructions for taking a linear combinations of the rows of $B$.
\end{solution}

We finish this section by introducing an important square matrix
called the identity matrix.

\begin{definition}{Identity matrix}{identity-matrix}
  The \textbf{identity matrix}%
  \index{matrix!identity}%
  \index{identity matrix} of size $n\times n$ has ones along the
  diagonal, and zeros everywhere else. In other words, it is the
  matrix $\mat{\delta_{ij}}$ where $\delta_{ij}=1$ if $i=j$ and
  $\delta_{ij}=0$ otherwise. The identity matrix is always a square
  matrix. Here are some identity matrices of various sizes.
  \begin{equation*}
    \begin{mymatrix}{r}
      1
    \end{mymatrix},
    \quad
    \begin{mymatrix}{cc}
      1 & 0 \\
      0 & 1
    \end{mymatrix},
    \quad
    \begin{mymatrix}{ccc}
      1 & 0 & 0 \\
      0 & 1 & 0 \\
      0 & 0 & 1
    \end{mymatrix},
    \quad
    \begin{mymatrix}{cccc}
      1 & 0 & 0 & 0 \\
      0 & 1 & 0 & 0 \\
      0 & 0 & 1 & 0 \\
      0 & 0 & 0 & 1
    \end{mymatrix}.
  \end{equation*}
  When it is necessary to distinguish which size of identity matrix is
  being discussed, we will use the notation $I_n$ for the $n\times n$
  identity matrix.
\end{definition}

\begin{example}{Multiplying by the identity matrix}{identity-multiply}
  Calculate $AI$, where $I$ is the $2\times 2$ identity matrix and
  \begin{equation*}
    A ~=~ \begin{mymatrix}{rr}
      a_{11} & a_{12} \\
      a_{21} & a_{22} \\
      a_{31} & a_{32} \\
    \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  We need to calculate
  \begin{equation*}
    AI ~=~
    \begin{mymatrix}{rr}
      a_{11} & a_{12} \\
      a_{21} & a_{22} \\
      a_{31} & a_{32} \\
    \end{mymatrix}
    \begin{mymatrix}{rr}
      1 & 0 \\
      0 & 1
    \end{mymatrix}.
  \end{equation*}
  Using the column method, we find that the first column of $AI$ is
  \begin{equation*}
    \begin{mymatrix}{rr}
      a_{11} & a_{12} \\
      a_{21} & a_{22} \\
      a_{31} & a_{32} \\
    \end{mymatrix}
    \begin{mymatrix}{r}
      1 \\
      0
    \end{mymatrix}
    ~=~
    1 \begin{mymatrix}{rr}
      a_{11} \\
      a_{21} \\
      a_{31} \\
    \end{mymatrix}
    + 0 \begin{mymatrix}{rr}
      a_{12} \\
      a_{22} \\
      a_{32} \\
    \end{mymatrix}
    ~=~
    \begin{mymatrix}{rr}
      a_{11} \\
      a_{21} \\
      a_{31} \\
    \end{mymatrix},
  \end{equation*}
  which is exactly the same as the first column of $A$.
  Similarly, the second column of $AI$ is
  \begin{equation*}
    \begin{mymatrix}{rr}
      a_{11} & a_{12} \\
      a_{21} & a_{22} \\
      a_{31} & a_{32} \\
    \end{mymatrix}
    \begin{mymatrix}{r}
      0 \\
      1
    \end{mymatrix}
    ~=~
    0 \begin{mymatrix}{rr}
      a_{11} \\
      a_{21} \\
      a_{31} \\
    \end{mymatrix}
    + 1 \begin{mymatrix}{rr}
      a_{12} \\
      a_{22} \\
      a_{32} \\
    \end{mymatrix}
    ~=~
    \begin{mymatrix}{rr}
      a_{12} \\
      a_{22} \\
      a_{32} \\
    \end{mymatrix},
  \end{equation*}
  which is exactly the same as the second column of $A$. Therefore
  \begin{equation*}
    AI ~=~
    \begin{mymatrix}{rr}
      a_{11} & a_{12} \\
      a_{21} & a_{22} \\
      a_{31} & a_{32} \\
    \end{mymatrix}
    ~=~ A.
  \end{equation*}
\end{solution}

The calculation of the last example generalizes to matrices of all
sizes, and is summarized in the following proposition.

\begin{proposition}{Multiplying by the identity matrix}{identity-matrix}
  Let $A$ be any $m\times n$-matrix. Then
  \begin{equation*}
    I_mA = A = AI_n.
  \end{equation*}
\end{proposition}

We can also raise a square matrix to a power. For example, $A^5$ means
$A\cdot A\cdot A\cdot A\cdot A$.

\begin{example}{Raising a matrix to a power}{matrix-power}
  Compute $A^3$, where
  \begin{equation*}
    A = \begin{mymatrix}{rr}
      1 & 2 \\
      -2 & 3 \\
    \end{mymatrix}.
  \end{equation*}
\end{example}

\begin{solution}
  We have
  \begin{equation*}
    A^3 = A\cdot A\cdot A
    = \begin{mymatrix}{rr}
      1 & 2 \\
      -2 & 3 \\
    \end{mymatrix}
    \begin{mymatrix}{rr}
      1 & 2 \\
      -2 & 3 \\
    \end{mymatrix}
    \begin{mymatrix}{rr}
      1 & 2 \\
      -2 & 3 \\
    \end{mymatrix}
    =
    \begin{mymatrix}{rr}
      -3 & 8 \\
      -8 & 5 \\
    \end{mymatrix}
    \begin{mymatrix}{rr}
      1 & 2 \\
      -2 & 3 \\
    \end{mymatrix}
    =
    \begin{mymatrix}{rr}
      -19 & 18 \\
      -18 & -1 \\
    \end{mymatrix}.
  \end{equation*}
\end{solution}


\subsection{Conversions}
\noindent
\begin{itemize}
	\item From Cylindrical to Cartesian
		\begin{equation*}
			(r, \theta, z) = (r\cos{\theta}, r\sin{\theta}, z)
		\end{equation*}
	\item From Cartesian to Cylindrical
		\begin{equation*}
			(x, y, z)=\left(\sqrt{x^2 + y^2}, \tan^{-1}{\frac{y}{x}}, z\right)
		\end{equation*}
\end{itemize}


\section{Generative Learning: Naive Bayes}
\subsection{Assumptions}
The naive bayes model supposes that the features of each data point are all
independent:
\begin{equation}
  P(x|y)=P(x_1,x_2,...,x_n|y)=P(x_1|y)P(x_2|y)...P(x_n|y)=\prod_{i=1}^nP(x_i|y)
\end{equation}
\subsection{Estimation}
We can write the likelihood of the data as:
\begin{equation}
  \mathcal{L}(\phi_y, \phi_{j=l|y=k}) = \prod_{i=1}^m P(x^{(i)}, y^{(i)})
\end{equation}
with classes denoted by $k$ and features $x_j = l$ ($x_j$ takes on value $l$).
By maximizing the estimates, we get:
\begin{equation}
  \phi_y = P(y=k)=\frac{1}{m} \sum_{i=1}^m 1_{\{y^{(i)}=k\}}
\end{equation}
\begin{equation}
  \phi_{j=l|y=k} = P(x_j=l|y=k)= \frac{ \sum_{i=1}^m 1_{\{y^{(i)}=k \, \wedge \, x_j^{(i)} = l\}} }{ \sum_{i=1}^m 1_{\{y^{(i)}=k\}} }
\end{equation}
\subsubsection{Laplace Smoothing}
Laplace smoothing allows for unseen data to have a probability, and not automatically
destory the prediciton process by setting all classes to $0$. We can replace
our feature estimates with the following:
\begin{equation}
  \phi_j = \frac{ \sum_{i=1}^m 1_{\{x^{(i)}=j\}} +1} { m + k }
\end{equation}
Or more generically:
\begin{equation}
  \phi_{j=l|y=k} = \frac{ \sum_{i=1}^m 1_{\{x_j^{(i)}=l \wedge y^{(i)} = k\}} + 1 } { \sum_{i=1}^m 1_{\{y^{(i)}=k\}} + |K| }
\end{equation}
where $|K|$ is the number of classes
\subsection{Prediction}
To classify a point $x$, we find the class $y=k$ which maximizes the probability:
\begin{equation}
  \hat{y} = \underset{k}{\textrm{arg max}} \,\, P(y=k) \cdot\prod_{i=j}^n P(x_j | y=k)
\end{equation}


\section*{Ex.34.1-5}
\subsection*{Show that if an algorithm makes at most a constant number of calls to polynomial-time subroutines and performs an additional amount of work that also takes polynomial times, then it runs in polynomial time}

Each subroutine $i$ takes $O(n^{k_i}_i)$ time, and there are $m$ subroutines. So in total, the algorithms takes
$$
\sum^m_{i=1} O(n^{k_i}_i) = c_1n^{k_1}_1 + c_2n^{k_2}_2 + \ldots + c_m n^{k_m}_m \leq m\cdot c_{max}n^{k_{max}}_{max} = O(n^{k_{max}}_{max}),
$$
where the index $max$ represents the subroutine, that takes the longest time.
\\
Since both $m$ and $c$ are constants, the total running time is still (bounded by) $O(n^k)$.

\subsection*{Show that a polynomial number of calls to polynomial-time subroutines a may result in an exponential-time algorithm}

We show an example:
\\
Let the subroutine be a recursive function, where an integer $x$ is multiplied with $x-1$, then $x-2$, then $x-3$, and so on, $x-1$ times which is linearly many times. The outcome of the recursive function is:
$$
x\cdot (x-1)\cdot (x-2)\cdot (x-3)\cdots 1 = x!
$$
Note, that $x!\in \omega(e^x)$,
\\
It will take an exponential number of bits to represent this, so it must also take exponential time to perform.


\documentclass{article}
    % General document formatting
    \usepackage[margin=0.7in]{geometry}
    \usepackage[parfill]{parskip}
    \usepackage[utf8]{inputenc}
    \usepackage{amsmath}
    \usepackage{amssymb}
    \usepackage{amsthm}
    \usepackage{tikz}
    \usepackage{fancyhdr}

\pagestyle{fancy}
\fancyhf{}
\rhead{Edgar Jacob Rivera Rios - A01184125}

\renewcommand{\labelenumi}{\alph{enumi})}
\renewcommand{\labelenumii}{\roman{enumii})}

\begin{document}
\section*{1.4.1 Proof by simple induction}
\begin{enumerate}
    \item Use simple induction to show that for every positive integer $n$, $5^n - 1$ is divisible by 4\\
    $5^{n}-1$ is divisible by 4\\
    Base case:
    \begin{align*}
        5^{1}-1 &= 4j\\
        &= 4 = 4j\\
        5^{k}-1 &= 4j\\
        5^{k}&= 4j+1
    \end{align*}
    Induction step:
    \begin{align*}
        &= 5^{k+ 1}-1\\
        &= 5^{k} \cdot 5 - 1\\
        &=(4j+1) \cdot 5 - 1\\
        &=(20j + 5) -1\\
        &=20j + 4\\
        &=4\cdot(5j + 1)\\
        &\implies true\ for\ n = k + 1
    \end{align*}
    \item Use simple induction to show that for every positive integer $n$, $n^3- n$ is divisible by 3. (Hint: In the induction step, you will need to make use of the arithmetic fact that $(k+1)^3 = k^3 + 3k^2 + 3k +1$)\\
    $n^3- n$ is divisible by 3\\
    Base case:
    \begin{align*}
        1^3- 1 &= 3j\\
        &= 0 = 3j\\
        k^{3}-k &= 3j\\
        k^{3}&= 3j+k
    \end{align*}
    Induction step:
    \begin{align*}
        &= (k + 1)^{3}-(k +1)\\
        &= (k^3 + 3k^2 + 3k +1) - (k +1)\\
        &= k^3 + 3k^2 + 2k\\
        &= (3j+k) + 3k^2 + 2k\\
        &= 3j+ 3k^2 + 3k\\
        &=3\cdot(j + k^2 + k)\\
        &\implies true\ for\ n = k + 1
    \end{align*}
    \item Show by simple induction that for every natural number $n$, $\sum_{i=0}^{n} 2^{i} = 2^{n+1} - 1$\\
    $f(n) = 2^{n+1} - 1 $\\
    Base case:
    \begin{align*}
        f(1) &= 2^{1+1} - 1\\
        f(1) &= 4 - 1\\
        f(1) &= 3\\
        f(k) &= 2^{k+1} - 1
    \end{align*}
    Induction step:
    \begin{align*}
        f(k +1) &= 2^{k + 2} - 1\\
        f(k +1) &= 2^1 + 2^2 ... + 2^{k} + 2^{k +1}\\
        &=f(k) + 2^{k +1}\\
        &=2^{k+1} - 1 + 2^{k +1}\\
        &=2(2^{k+1}) - 1\\
        &=2^{1} \cdot 2^{k+1} - 1\\
        &=2^{k+1 +1} - 1\\
        &=2^{k+2} - 1\\
        &\implies true\ for\ n = k + 1
    \end{align*}
\end{enumerate}
\section*{1.4.2: Definition by simple recursion}
\begin{enumerate}
    \item  Let $f: N \rightarrow N$ be the function defined by putting $f (0) = 0$ and $f(n+1) = n$ for all $n \in N$.
    \begin{enumerate}
        \item Evaluate this function bottom-up for all arguments 0–5.
        \begin{align*}
            f(0) &= 0\\
            f(1) &= 0\\
            f(2) &= 1\\
            f(3) &= 2\\
            f(4) &= 3\\
            f(5) &= 4
        \end{align*}
        \item Explain what f does by expressing it in explicit terms (i.e. without a recursion).\\
        This function is not recursive, it only returns the antecesor of the current value
    \end{enumerate}
    \item Let $f : N^{+} \rightarrow N$ be the function that takes each positive integer $n$ to the greatest natural number $p$ with $2^{p} \leq n$. Define this function by a simple recursion. (Hint: You will need to divide the recursion step into two cases.)
    \begin{align*}
        &f(n) = 0& &, when\ n = 1\\
        &f(n) = f(n-1)& &, when\ n > 1, \log_{2}(n) \notin N\\
        &f(n) = f(n-1) + 1& &, when\ n > 1, \log_{2}(n) \in N
    \end{align*}
    \item Let $g: NXN \rightarrow N$ be defined by putting $g(m,0)= m$ for all $m \in N$ and $g(m,n+1)=f (g(m,n))$ where $f$ is the function defined in part (a) of this exercise.
    \begin{enumerate}
        \item Evaluate $g(3,4)$ top-down.
        \begin{align*}
            g(3, 4) &= f(g(3, 3))\\
            &= f(f(g(3, 2)))\\
            &= f(f(f(g(3, 1))))\\
            &= f(f(f(f(g(3, 0)))))\\
            &= f(f(f(f(f(3)))))\\
            &= f(f(f(f(2))))\\
            &= f(f(f(1)))\\
            &= f(f(0))\\
            &= f(0)\\
            &= 0\\
        \end{align*}
        \item Explain what $g$ does by expressing it in explicit terms (i.e. without a recursion).\\
        It substracts the right item from the left, but if the remain is negative, it returns 0
    \end{enumerate}
\end{enumerate}

\section*{1.4.3: Proof by cumulative induction}
\begin{enumerate}
    \item Use cumulative induction to show that any postage cost of four or more pence can be covered by two-pence and five-pence stamps.\\
    $2x + 5y = n$, when $n \geq 4$\\
    Base case:
    \begin{align*}
        n &= 4 &4 &= 2(2) + 5(0)
    \end{align*}
    Induction step:
    \begin{align*}
        hypothesis &\rightarrow \forall j < k,\ j = 2x +5y\\
        goal &\rightarrow k = 2x + 3y\\
        case (1) &\rightarrow k\ is\ multiple\ of\ two \implies y = 0\ and\ x \in N\\
        case (2) &\rightarrow k\ is\ multiple\ of\ five \implies x = 0\ and\ y \in N\\
        case (3) &\rightarrow k\ is\ not\ multiple\ of\ any \implies y \in N\ and\ x \in N\\
    \end{align*}
    \item Use cumulative induction to show that for every natural number $n$, $F(n) \leq 2^{n}-1$, where $F$ is the Fibonacci function.\\
    $F(n) \leq 2^{n} -1$, \\
    Base case:
    \begin{align*}
        n &\leq 1 &1 &\leq 2^{1} - 1
    \end{align*}
    Induction step:
    \begin{align*}
        hypothesis &\rightarrow \forall j < k,\ F(j) \leq 2^{j} - 1\\
        goal &\rightarrow F(k) \leq 2^{k} -1\\
        case (1) &\rightarrow F(k)\ is\ greater\ than\ 2^{k} -1 \implies 2^{k+1} -1 < F(j +1)\\
        &Impossible\ given\ that\ F\ grows\ at\ F(k)\ and\ 2^{k} -1\ grows\ at\ double\ rate\\
        case (2) &\rightarrow F(k)\ is\ less\ or\ equal\ than\ 2^{k} -1e \implies This\ must\ be\ true\ then
    \end{align*}
    \item Calculate F(5) top-down, and then again bottom-up, where again $F$ is the Fibonacci function\\
    Top-down:
    \begin{align*}
        F(5) &= F(4) + F(3) \\
        F(5) &= (F(3) + F(2)) + (F(2) + F(1)) \\
        F(5) &= ((F(2) + F(1)) + (F(1) + F(0))) + ((F(1) + F(0)) + 1)\\
        F(5) &= (((F(1) + F(0)) + 1) + (1 + 0)) + ((1 + 0) + 1)\\
        F(5) &= (((1+ 0) + 1) + (1 + 0)) + ((1 + 0) + 1)\\
        F(5) &= ((1 + 1) + (1)) + (1 + 1)\\
        F(5) &= (2 + 1) + (2)\\
        F(5) &= 3 + 2\\
        F(5) &= 5
    \end{align*}
    Bottom-up
    \begin{align*}
        F(0) &= 0 \\
        F(1) &= 1 \\
        F(2) &= F(1) + F(0) = 1 + 0 = 1\\
        F(3) &= F(2) + F(1) = 1 + 1 = 2\\
        F(4) &= F(3) + F(2) = 2 + 1 = 3\\
        F(5) &= F(4) + F(3) = 3 + 2 = 5
    \end{align*}
    \item Express each of the numbers 14, 15 and 16 as a sum of $3s$ and/or $8s$. Using this fact in your basis, show by cumulative induction that every positive integer $n \leq 14$ may be expressed as a sum of $3s$ and/or $8s$.\\
    $n = 3x + 8y$, when $n > 14$\\
    Base case:
    \begin{align*}
        n &= 14 & 14 = 3(2) + 8(1)\\
        n &= 15 & 15 = 3(5) + 8(0)\\
        n &= 16 & 16 = 3(0) + 8(2)\\
    \end{align*}
    Induction step:
    \begin{align*}
        hypothesis &\rightarrow \forall j < k,\ j = 3x + 8y\\
        goal &\rightarrow k = 3x + 8y\\
        case (1) &\rightarrow k\ is\ multiple\ of\ three \implies y = 0\ and\ x \in N\\
        case (2) &\rightarrow k\ is\ multiple\ of\ eigth \implies x = 0\ and\ y \in N\\
        case (3) &\rightarrow k\ is\ not\ multiple\ of\ any \implies y \in N\ and\ x \in N\\
    \end{align*}
    \item Show by induction that for every natural number $n$, $A(1,n) = n+2$, where $A$ is the Ackermann function.
    \[
    A(m,n) =
    \begin{cases}
        n + 1 &\quad if\ m=0\\
        A(m-1, 1) &\quad if\ m >0\ and\ n=0\\
        A(m-1, A(m, n - 1)) &\quad if\ m >0\ and\ n >0\\
    \end{cases}
    \]
    Base case:
    \begin{align*}
        n &= 0 & A(1, 0) &= A(0, 1)\\
        & & &=2\\
        & & &=(0) +2\\
        n &= 1 & A(1, 1) &= A(0, A(1, 0))\\
        & & &=A(0, 2)\\
        & & &=3\\
        & & &=(1) +2\\
    \end{align*}
    Induction step:
    \begin{align*}
        hypothesis &\rightarrow \forall j < k,\ A(1,j) = j + 2\\
        goal &\rightarrow A(1,k) = k + 2\\
        A(1, k + 1) &= (k + 1) + 2\\
        &= A(0, A(1, k))\\
        &= A(1, k) +1\\
        &= (k + 2) +1\\
        A(1, k + 1) &= k + 3\\
        &\therefore A(1,k) = k + 2
    \end{align*}
\end{enumerate}

\end{document}


\subsection{Gradient}

In a scalar field we can calculate the partial derivative at any point with respect to one input.

We may wish to consider these collectively. To do that we use the gradient operator.

We previously introduced the Del operator where:

\(\nabla =(\sum_{i=1}^n e_i\dfrac{\delta }{\delta x_i})\)

Where \(e\) are the basis vectors.

This on its own means nothing. It is similar to the partial differentiation function.

We now multiply Del by the function. This gives us:

\(\nabla f=(\sum_{i=1}^n e_i\dfrac{\delta f}{\delta x_i})\). This gives us a vector in the underlying vector space.

This is the gradient.


\renewcommand{\Pr}{\text{Pr}}
\newcommand{\Var}{\text{Var}}

\chapter{Probability}

Before we can discuss probablistic and randomized algorithms, we must
first develop the tools necessary to rigorously discuss probability.

We are concerned with the probability of sets of events, and for
simplicity we will simply call these sets events.

The probability of a event $A$ is $\Pr(A)$.  The probability of an
event $A$ given that event $B$ has occured is $\Pr(A | B)$.

The probability of events $A$ and $B$ both happening is

\begin{center}
\begin{math}
\Pr(A \cap B)
= \Pr(A|B)\Pr(B) = \Pr(A)\Pr(B|A)
\end{math}
\end{center}

which is the definition of \emph{conjunctive probability}.

The probability of either events $A$ or $B$ happening is

\begin{center}
\begin{math}
  \Pr(A \cup B)
  = \Pr(A) + \Pr(B) - \Pr(A \cap B)
\end{math}
\end{center}

which is the definition of \emph{disjunctive probability}.

If the outcome of event $A$ does not affect that of $B$, we say $A$
and $B$ are \emph{independent}.

If we have a set of independent events $S$, from the definition of
disjunctive probability we get

\begin{displaymath}
  %
  \Pr \left( \bigcup_{s \in S} s \right) = \sum_{s \in S} \Pr(s)
  %
\end{displaymath}

which is the \emph{linearity of independent probability}.

\section{Linearity of Expectation}

Often, we are interested in the outcome of some event with intrinsic
value, such as the result of rolling a die.  In this case, we may be
interested in the average value.  Since not all values are equally
likely, it is important to weight each value by its probability.  We
call such a weighted average the \emph{Expected Value}.

The expected value of a random variable $X$ is

\begin{displaymath}
  %
  E(X) = \sum_{x \in X} x \cdot \Pr(x)
  %
\end{displaymath}

which is the definition of \emph{expectation}.

If we have two events $X$ and $Y$, we have

\begin{center}
\begin{math}
  E(X + Y) = E(X) + E(Y)
\end{math}
\end{center}

which is the \emph{linearity of expectation}.

\section{Markov's Inequality}

We often wish to establish bounds on probability, for which one
important result is for any random variable $X$ and $a > 0$

\begin{center}
\begin{math}
  \Pr(X \geq a) \leq \frac{E(X)}{a}
\end{math}
\end{center}

which is \emph{Markov's Inequality}.

\section{Variance}

Given a random variable $X$ we have

\begin{center}
  \begin{math}
    \Var (X) = E(X^2) - E(X)^2
  \end{math}
\end{center}

which is the definition of \emph{variance}.

If we have an independent and identically distributed set of events
$X$ then

\begin{displaymath}
  \Var (\sum_{x \in X} x) = \sum_{x \in X} \Var (x)
\end{displaymath}

\section{Chebyshev's Inequality}

For any random variable $X$ and $a > 0$ we have

\begin{center}
  \begin{math}
    \Pr ( | X - E(X) | \geq a ) \leq \frac{\Var{X}}{a^2}
  \end{math}
\end{center}

which is \emph{Chebyshev's Inequality}.

\section{Chernoff Bounds}

Where $X$ is the sum of indicator random variables, for any $ \epsilon
> 0 $ we have

\begin{center}
  \begin{math}
    \Pr ( X \geq ( 1 + \epsilon ) E(X) ) \leq e^{ \frac{-\epsilon^2 E(X)}{3}}
  \end{math}
\end{center}

and

\begin{center}
  \begin{math}
    \Pr ( X \geq ( 1 - \epsilon ) E(X) ) \leq e^{ \frac{-\epsilon^2 E(X)}{2}}
  \end{math}
\end{center}

which are \emph{Chernoff Bounds}.


\section{SVD as an eigen problem}
\label{sec:svd-lanczos-eigen}

Aiming to calculate numerically the SVD factorizations, made
researchers reformulate that problem as the quite related
eigen decomposition (or eigenproblem). Such problem consists in
finding, for an square matrix $A$, the eigenvalues and
eigenvectors. If we arrange the eigenvectors in an orthogonal matrix
$Q$ and the eigenvalues in a diagonal matrix $\Lambda$, the eigen
problem can be restated as the following factorization: \\

\[
A = Q \Sigma \trans{Q}
\]
\hfill

In order to see the connection between the SVD and the eigenproblem,
we need to recall the gramian matrix $\trans{A}A$ from the theory
chapter. It was the gramian, which provided the matrix $V$ on the
first place; because the vectors
$\vec{v}$ were its eigenvectors (see \cref{cha:svd-theory}. Finding
the matrix $V$ then, can be thought as the eigenproblem for matrix
$\trans{A}A$; which can be stated as finding its diagonal
factorization $\suchthat$:

\begin{equation}
\label{eq:eigenprob-doc}
\trans{A}A = V\Sigma^2\trans{V}
\end{equation}
\hfill

But the same is true for matrix $U$, if we now consider the matrix
$A\trans{A}$, which can be diagonalized if one finds its eigenvectors
and place them into the matrix $U$ (the eigenvalues are the same as
the gramian):

\begin{equation}
\label{eq:eigenprob-term}
A\trans{A} = U\Sigma^2\trans{U}
\end{equation}
\hfill

It is the second eigenvalue problem equivalence, that is used for this
distributed algorithm of chapter \cref{cha:svd-dist}. Per the SVD
factorization $A = U\Sigma\trans{V}$, if we have the original matrix
$A$, plus the diagonal $\Sigma$ and the matrix $U$;  we can
reconstruct the matrix $V$ (if required):

\[
\trans{V} = \inv{S} \trans{U} A = P A
\]
\hfill

The matrix $P = \inv{S} \trans{U}$ is called the projection matrix, and
is used in LSI for ``folding-in'' new document vectors \vec{x}, by
calculating $P\vec{x}$; that is, the matrix $P$ is used as a
predictive (rather than descriptive) model, to predict where the
position of document \vec{x} will be in the latent space. \\

For this chapter though, we could use either eigenproblem from
\cref{eq:eigenprob-doc} or \cref{eq:eigenprob-term}. Actually, the literature
originally reported the former, perhaps due the early shape of the
matrices used for LSI (more terms than documents). Today's LSI
applications have much more documents than terms, but still these
early algorithms are useful, as we will see in \cref{cha:svd-dist}
(where the original matrix is split into several submatrices, which do
have the shape expected by Lanczos algorithm that we document here).


\section*{Exercises}

\begin{ex}
  Find the algebraic and geometric multiplicity of each eigenvalue of
  the following matrices. Which of the matrices are diagonalizable?
  \begin{equation*}
    (a)\quad
    A = \begin{mymatrix}{rr} 1 & 1 \\ -1 & 3 \end{mymatrix},
    \quad
    (b)\quad
    B = \begin{mymatrix}{rr} -7 & 8 \\ -4 & 5 \end{mymatrix},
  \end{equation*}
  \begin{equation*}
    (c)\quad
    C = \begin{mymatrix}{rrr}
      2 & -1 & -1 \\
      0 & 4 & 2 \\
      0 & -1 & 1 \\
    \end{mymatrix},
    \quad
    (d)\quad
    D = \begin{mymatrix}{rrr}
      3 & 1 & -1 \\
      0 & 2 & 1 \\
      0 & -1 & 4 \\
    \end{mymatrix},
    \quad
    (e)\quad
    E = \begin{mymatrix}{rrr}
      -2 & 0 & 1 \\
      -1 & -1 & 1 \\
      -2 & 1 & 0 \\
    \end{mymatrix}.
  \end{equation*}
  \begin{sol}
    \begin{enumerate}
    \item $\eigenvar=2$ has algebraic multiplicity 2 and geometric
      multiplicity 1. Since the sum of the geometric multiplicities of
      all eigenvalues is $1$, the matrix is not diagonalizable.
    \item $\eigenvar=1$ has algebraic and geometric
      multiplicity 1; $\eigenvar=-3$ has algebraic and geometric
      multiplicity 1. Since the sum of the geometric multiplicities is
      $2$, the matrix is diagonalizable.
    \item $\eigenvar=2$ has algebraic and geometric
      multiplicity 2, $\eigenvar=3$ has algebraic and geometric
      multiplicity 1. Since the sum of the geometric multiplicities is
      $3$, the matrix is diagonalizable.
    \item $\eigenvar=3$ has algebraic multiplicity 3 and geometric
      multiplicity 2. Since the sum of the geometric multiplicities of
      all eigenvalues is $2$, the matrix is not diagonalizable.
    \item $\eigenvar = -1$ has algebraic multiplicity 3 and geometric
      multiplicity 1. Since the sum of the geometric multiplicities of
      all eigenvalues is $1$, the matrix is not diagonalizable.
    \end{enumerate}
  \end{sol}
\end{ex}

\begin{ex}
  Determine which of the following matrices are diagonalizable.
  \begin{enumerate}
  \item $A$ is a $3\times 3$-matrix with eigenvalues $-1$ and $3$. The
    eigenvalue $-1$ has algebraic multiplicity $2$ and geometric
    multiplicity $1$. The eigenvalue $3$ has algebraic and geometric
    multiplicity $1$.
  \item $B$ is a $4\times 4$-matrix with eigenvalues $2$ and $-2$. The
    eigenvalue $2$ has algebraic and geometric multiplicity $1$. The
    eigenvalue $-2$ has algebraic and geometric multiplicity $3$.
  \item $C$ is a $5\times 5$-matrix with eigenvalues $1$ and $3$. The
    eigenvalue $1$ has algebraic and geometric multiplicity $2$, and
    the eigenvalue $3$ has algebraic and geometric multiplicity $1$.
  \end{enumerate}
  \begin{sol}
    \begin{enumerate}
    \item Not diagonalizable.
    \item Diagonalizable.
    \item Not diagonalizable.
    \end{enumerate}
  \end{sol}
\end{ex}