\savestack{\pcareconstruction}{\input{./modules/Module5/tikz_images/pca_reconstruction.tex}}

% Slide 37
\begin{frame}
  \myheading{Module 6.5 : PCA : Interpretation 2}
\end{frame}

%----------------------------------------------------------------------------------------------------------
% Slide 38
\begin{frame}
  \begin{overlayarea}{\textwidth}{\textheight}
    \onslide<1->{
      Given $n$ orthogonal linearly independent vectors $P=p_1,p_2,\cdots,p_n$ we can represent $x_i$ exactly as a linear combination of these vectors.
    }
    \onslide<2->{
      \begin{equation*}
        x_i= \sum_{j=1}^{n} \alpha_{ij} p_j \hspace{0.3cm}[\textnormal{we know how to estimate $\alpha_{ij}'s$ but we will come back to that later}]
      \end{equation*}
    }
    \onslide<3->{But we are interested only in the top-k dimensions (we want to get rid of noisy \& redundant dimensions)
      \begin{equation*}
        \hat{x}_i = \sum_{j=1}^{k} \alpha_{ik} p_k
      \end{equation*}
    }
    \onslide<4->{
      We want to select $p_{i}'s$ such that we minimise the reconstructed error
      \begin{equation*}
        e= \sum_{i=1}^{m} (x_i - \hat{x}_i)^T (x_i - \hat{x}_i)
      \end{equation*}
    }
  \end{overlayarea}
\end{frame}


%----------------------------------------------------------------------------------------------------------
% Slide 39
\begin{frame}[shrink=20]%change this to change the font
  \begin{columns}
    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{align*}
          \onslide<1->{  e &= \sum_{i=1}^{m} (x_i - \hat{x}_i)^T (x_i - \hat{x}_i) } \\
          \onslide<2->{
            &= \sum_{i=1}^{m} \left(\sum_{j=1}^{n} \alpha_{ij}p_j - \sum_{j=1}^{k} \alpha_{ij} p_j\right)^2} \\
          \onslide<3->{ &= \sum_{i=1}^{m} \left(\sum_{j=k+1}^{n}\alpha_{ij}p_j\right)^2 = \sum_{i=1}^{m} \left(\sum_{j=k+1}^{n}\alpha_{ij}p_j\right)^T\left(\sum_{j=k+1}^{n}\alpha_{ij}p_j\right)} \\
          \onslide<4->{ &= \sum_{i=1}^{m} \left(\alpha_{i,k+1}p_{k+1}+\alpha_{i,k+2}p_{k+2}+\hdots+\alpha_{i,n}p_n\right)^T\left(\alpha_{i,k+1}p_{k+1}+\alpha_{i,k+2}p_{k+2}+\hdots+\alpha_{i,n}p_n\right)} \\
          \onslide<5->{ &= \sum_{i=1}^{m} \sum_{j=k+1}^{n} \alpha_{ij}p_j^T p_j \alpha_{ij} + \sum_{i=1}^{m} \sum_{j=k+1}^{n} \sum_{L=k+1,L\neq k}^{n} \alpha_{ij}p_j^T p_L \alpha_{iL}} \\
          \onslide<6->{&= \sum_{i=1}^{m} \sum_{j=k+1}^{n} \alpha_{ij}^2 \qquad \qquad (\because p_j^T p_j =1, p_i^T p_j =0 \quad \forall i\neq j) }\\
          \onslide<7->{&= \sum_{i=1}^{m} \sum_{j=k+1}^{n} \left(x_i^T p_j\right)^2 } \\
      \end{align*}
    \end{overlayarea}
    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{tabular}{|l}
        {$\begin{aligned}
              \onslide<8->{&= \sum_{i=1}^{m} \sum_{j=k+1}^{n} \left(p_j^T x_i\right)\left(x_i^T p_j\right)} \\
              \onslide<9->{&= \sum_{j=k+1}^{n} p_j^T \left( \sum_{i=1}^{m} x_i x_i^T\right) p_j  } \\
              \onslide<10->{ &= \sum_{j=k+1}^{n} p_j^T m C p_j \qquad \left[\because \frac{1}{m} \sum_{i=1}^{m} x_i x_i^T = \frac{X^TX}{m} = C\right]}
          \end{aligned}$}
      \end{tabular}
    \end{overlayarea}
  \end{columns}
\end{frame}

%----------------------------------------------------------------------------------------------------------
% Slide 40
\begin{frame}
  \begin{overlayarea}{\textwidth}{\textheight}
    \onslide<1->{
      \vspace{1.3cm}

      We want to minimize $e$
      \begin{equation*}
        \min_{p_{k+1},p_{k+2},\cdots,p_{n}} \sum_{j=k+1}^{n} p_j^T m C p_j \qquad s.t.\quad p_j^Tp_j =1 \quad \forall j = k+1,k+2,\cdots,n
      \end{equation*}

    }
    \onslide<2->{
      The solution to the above problem is given by the eigen vectors corresponding to the smallest eigen values of $C$ (\textbf{Proof : refer Slide 26}). }

    \onslide<3->{
      \vspace{0.7cm}
      Thus we select $P=p_1,p_2,\cdots,p_n$ as eigen vectors of $C$ and retain only top-k eigen vectors to express the data [or discard the eigen vectors $k+1,\cdots,n$]
    }
  \end{overlayarea}
\end{frame}

%----------------------------------------------------------------------------------------------------------
% Slide 41
\begin{frame}
  %\begin{overlayarea}{\textwidth}{\textheight}
  %\begin{center}
  \begin{block}{Key Idea}
    Minimize the error in reconstructing $x_i$ after projecting the data on to a new basis.
  \end{block}
  %\end{center}
  %\end{overlayarea}
\end{frame}

%----------------------------------------------------------------------------------------------------------
% Slide 42
\begin{frame}
  \begin{center}
    \textit{Let's look at the \textbf{`Reconstruction Error'} in the context of our toy example}
  \end{center}
\end{frame}

%----------------------------------------------------------------------------------------------------------
% Slide 43
\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}<1->
    \begin{overlayarea}{\textwidth}{\textheight}
      \makebox[\textwidth][c]{\usebox{\pcareconstructioncontent}}
      % \input{./modules/Module5/tikz_images/pca_reconstruction.tex}
      \begin{itemize}\justifying
        \item<1-> $u_1=[1,1]$ and $u_2=[-1,1]$ are the new basis vectors
        \item<2-> Let us convert them to unit vectors $u_1=\begin{bmatrix}
                  \frac{1}{\sqrt{2}} & \frac{1}{\sqrt{2}}
                \end{bmatrix}$
              \& $u_2=\begin{bmatrix}
                  \frac{-1}{\sqrt{2}} & \frac{1}{\sqrt{2}}
                \end{bmatrix}$


      \end{itemize}
    \end{overlayarea}

    \column{0.5\textwidth}<3->
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}\justifying
        \item<3-> Consider the point $x=[3.3,3]$ in the original data
        \item<4-> $\alpha_1 = x^Tu_1= {6.3}/{\sqrt{2}}$ \\
              $\alpha_2 = x^Tu_2= {-0.3}/{\sqrt{2}}$
        \item<5-> the perfect reconstruction of $x$ is given by (using $n=2$ dimensions)
              \begin{equation*}
                x= \alpha_1 u_1 + \alpha_2 u_2 = \begin{bmatrix}3.3 & 3 \end{bmatrix}
              \end{equation*}
        \item<6-> But we are going to reconstruct it using fewer (only $k=1<n$ dimensions, ignoring the low variance $u_2$ dimension)
              \begin{equation*}
                \hat{x}=\alpha_1 u_1=\begin{bmatrix}3.15 & 3.15 \end{bmatrix}
              \end{equation*}
              (reconstruction with minimum error)
      \end{itemize}
    \end{overlayarea}
  \end{columns}
\end{frame}

%----------------------------------------------------------------------------------------------------------
% Slide 44
\begin{frame}
  \begin{overlayarea}{\textwidth}{\textheight}
    \begin{block}{Recap}
      \begin{itemize}\justifying
        \item<1-> The eigen vectors of a matrix with distinct eigenvalues are linearly independent
        \item<2-> The eigen vectors of a square symmetric matrix are orthogonal
        \item<3-> PCA exploits this fact by representing the data using a new basis comprising only the top-$k$ eigen vectors
        \item<4-> The $n-k$ dimensions which contribute very little to the reconstruction error are discarded
        \item<5-> \textbf{These are also the directions along which the variance is minimum}
      \end{itemize}
    \end{block}
  \end{overlayarea}
\end{frame}
