\savestack{\nnfruitclassexample}{\input{modules/Module3/tikz_images/nn_fruit_class_example.tex}}
\savestack{\nnimdbexample}{\input{modules/Module3/tikz_images/nn_imdb_example.tex}}
\savestack{\nn}{\input{modules/Module3/tikz_images/nn.tex}}


\begin{frame}
  \myheading{Module 4.3: Output Functions and Loss Functions}
\end{frame}

%Slide 11
\begin{frame}
  \begin{block}{We need to answer two questions}
    \begin{itemize}
      \justifying
      \item \alert<2>{How to choose the loss function $\mathscr{L}(\theta)$ ?} \color{black}
      \item How to compute $\nabla \theta$ which is composed of:\\
      $\nabla W_1, \nabla W_2, ..., \nabla W_{L-1} \in \mathbb{R}^{n \times n}, \nabla W_L \in \mathbb{R}^{n \times k}$\\
      $\nabla b_1, \nabla b_2, ..., \nabla b_{L-1} \in \mathbb{R}^n $ and $\nabla b_L \in \mathbb{R}^k$ ?
    \end{itemize}
  \end{block}
\end{frame}

%Slide 12
\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \vspace{0.3cm}
      \visible<3->{
        \makebox[\textwidth][c]{\usebox{\nnimdbexamplecontent}}
      }
    \end{overlayarea}

    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}[<+->]
      \justifying
        \item The choice of loss function depends on the problem at hand
        \item We will illustrate this with the help of two examples
        \item Consider our movie example again but this time we are interested in predicting ratings
        \item Here $y_i \in \mathbb{R}^3$
        \item The loss function should capture how much $\hat{y}_i$ deviates from $y_i$
        \item If $y_i \in \mathbb{R}^n$ then the squared error loss can capture this deviation
            \vspace{-0.2in}
            \begin{align*}
              \mathscr{L}(\theta) = \frac{1}{N}\sum_{i=1}^{N}\sum_{j=1}^{3}\left(\hat{y}_{ij} - y_{ij}\right)^2
            \end{align*}
      \end{itemize}
    \end{overlayarea}
  \end{columns}
\end{frame}

%Slide 13
\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \makebox[\textwidth][c]{\usebox{\nncontent}}
    %   \makebox[\textwidth][c]{\usebox{\nnimdbexamplecontent}}
    \end{overlayarea}


    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}[<+->]
      \justifying
        \item A related question: What should the output function `$O$' be if $y_i \in \mathbb{R}$?
        \item More specifically, can it be the logistic function?
        \item No, because it restricts $\hat{y}_i$ to a value between $0$ \& $1$ but we want $\hat{y}_i \in \mathbb{R}$
        \item So, in such cases it makes sense to have `$O$' as linear function
            \begin{align*}
              f(x) & = h_{L} = O(a_{L})             \\
                  &= W_{O}a_{L} + b_{O}
            \end{align*}

        \item $\hat{y}_i = f(x_i)$ is no longer bounded between 0 and 1
      \end{itemize}
    \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}[b]
    \visible<1-2>{\tiny Intentionally left blank}
\end{frame}

%Slide 14
\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \vspace{0.3cm}
      \makebox[\textwidth][c]{\usebox{\nnfruitclassexamplecontent}}
    \end{overlayarea}

    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}[<+->]
      \justifying
        \item Now let us consider another problem for which a different loss function would be appropriate
        \item Suppose we want to classify an image into $1$ of $k$ classes
        \item Here again we could use the squared error loss to capture the deviation
        \item But can you think of a better function?
      \end{itemize}
    \end{overlayarea}
  \end{columns}
\end{frame}

%Slide 15
\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \only<1-2>{
        \vspace{0.3cm}
        \makebox[\textwidth][c]{\usebox{\nnfruitclassexamplecontent}}
      }
      \only<3->{
        \makebox[\textwidth][c]{\usebox{\nncontent}}
      }
    \end{overlayarea}

    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}
        \justifying
        \item<1-> Notice that $y$ is a probability distribution
        \item<2-> Therefore we should also ensure that $\hat{y}$ is a probability distribution
        \item<3-> What choice of the output activation `$O$' will ensure this ?
            \vspace{-0.1in}
            \begin{align*}
              a_{L}               & = W_{L}h_{L-1} + b_{L}                                            \\
              \visible<4->{\hat{y}_j & = O(a_{L})_j = \frac{e^{a_{L, j}}}{\sum_{i=1}^{k} e^{a_{L,i}}}}
            \end{align*}
            % \vspace{-0.1in}
            \visible<4->{$O(a_{L})_j$ is the $j^{th}$ element of $\hat{y}$ and $a_{L,j}$ is the $j^{th}$ element of the vector $a_{L}$.}
        \item<5-> This function is called the \textit{softmax} function
      \end{itemize}
    \end{overlayarea}
  \end{columns}
\end{frame}

%Slide 16
\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \vspace{0.3cm}
      \makebox[\textwidth][c]{\usebox{\nnfruitclassexamplecontent}}
    \end{overlayarea}

    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}
        \justifying
        \item<1-> Now that we have ensured that both $y$ \& $\hat{y}$ are probability distributions can you think of a function which captures the difference between them?
        \item<2-> Cross-entropy \begin{equation*}
              \mathscr{L}(\theta) = - \sum_{c=1}^{k} y_c \log \hat{y}_c
            \end{equation*}
        \item<3-> Notice that \begin{align*}
              y_c                 & = 1 \hspace{0.5cm} \text{if $c=\ell$ (the true class label)} \\
                        & = 0 \hspace{0.5cm} \text{otherwise}                          \\
              \therefore~~ &\mathscr{L}(\theta)  = - \log \hat{y}_{\ell}
            \end{align*}
      \end{itemize}
    \end{overlayarea}
  \end{columns}
\end{frame}

%Slide 17
\begin{frame}
  \begin{columns}
    \column{0.35\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \vspace{0.3cm}
      \makebox[\textwidth][c]{\usebox{\nncontent}}
      % \makebox[\textwidth][c]{\usebox{\nnfruitclassexamplecontent}}
    \end{overlayarea}

    \column{0.65\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}[<+->]
      \justifying
        \item So, for classification problem (where you have to choose $1$ of $K$ classes), we use the following objective function
          \begin{align*}
                        \underset{\theta}{\text{minimize}} ~~~~~~\mathscr{L}(\theta) &= -\log \hat{y}_{\ell} \\ %\{- \log \hat{y}_{\ell}\}$ \\ 
              \text{or}~~~~~~\underset{\theta}{\text{maximize}} ~~ -\mathscr{L}(\theta) &=  \log \hat{y}_{\ell}    %\{- \log \hat{y}_{\ell}\}$ \\ 
          \end{align*}
          % \hspace{2cm}
          % \hspace{2cm} 
          %     or $\max \log \hat{y}_{\ell}$
        \item But wait! \\
            Is $\hat{y}_\ell$ a function of $\theta=[W_1,W_2,.,W_L,b_1,b_2,.,b_L]$?
        \item Yes, it is indeed a function of $\theta$
          \begin{center}
            $ \hat y_\ell = [O(W_{3}g(W_{2}g(W_{1}x+b_{1})+b_{2})+b_{3})]_\ell $
          \end{center}
        \item What does $\hat{y}_\ell$ encode? 
        \item It is the probability that $x$ belongs to the $\ell^{th}$ class (bring it as close to $1$).
        \item $\log \hat{y}_\ell$ is called the \textit{log-likelihood} of the data.
      \end{itemize}
    \end{overlayarea}
  \end{columns}
\end{frame}

%Slide 18
\begin{frame}
  \begin{overlayarea}{\textwidth}{\textheight}
    \begin{center}
      \begin{table}[]
        \centering
        \label{my-label}
        \renewcommand{\arraystretch}{2}
        \begin{tabular}{c|c|c|}
          \cline{2-3}
                              & \multicolumn{2}{        c|}{\textbf{Outputs}} \\ \cline{2-3} 
                              & Real Values             & Probabilities     \\ \hline
          \multicolumn{1}{|c|}{Output Activation} & \visible<2->{Linear}       & \alert<7>{\visible<3->{Softmax}}\\ \hline
          \multicolumn{1}{|c|}{Loss Function}     & \visible<4->{Squared Error}& \alert<7>{\visible<5->{Cross Entropy}}   \\ \hline
        \end{tabular}
        \renewcommand{\arraystretch}{1}
      \end{table}
    \end{center}

    \begin{itemize}
      \justifying
      \item<6-> Of course, there could be other loss functions depending on the problem at hand but the two loss functions that we just saw are encountered very often
      \item<7-> For the rest of this lecture we will focus on the case where the output activation is a softmax function and the loss function is cross entropy
    \end{itemize}
  \end{overlayarea}
\end{frame}