\begin{frame}
	\myheading{Module 16.3: Attention Mechanism}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.45\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \vspace{0.3cm}
  \centering
  \input{modules/Module3/tikz_images/translation_opt2_1} %Figure E
      \end{overlayarea}

    \column{0.55\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
\begin{itemize}
\justifying
\item<1->  Let us motivate the task of attention with the help of MT
\item<2-> The encoder reads the sentences only once and encodes it
\item <3-> At each timestep the decoder uses this embedding to produce a new word
\item <4-> Is this how humans translate a sentence ? \onslide<5->{Not really!}
\end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \vspace{0.5cm}

  o/p : {\only<3>{\color{red}}I} {\only<4>{\color{red}}am} {\only<5>{\color{red}}going} {\only<6>{\color{red}}home}\\
  \begin{tabular}{l}
  \onslide<3->{$t_1$ : [ {\only<3>{\color{red}}1}  0  0  0  0 ]\\}
  \hspace{-0.3cm}
  \onslide<4->{$t_2$ : [ 0  0  0  0  {\only<4>{\color{red}}1} ]\\}
  \hspace{-0.3cm}
  \onslide<5->{$t_3$ : [ 0  0  {\only<5>{\color{red}}0.5}  {\only<5>{\color{red}}0.5}  0 ]\\}
  \hspace{-0.3cm}
  \onslide<6->{$t_4$ : [ 0  {\only<6>{\color{red}}1}  0  0  0 ]\\}
  \end{tabular}

  i/p : {\only<3>{\color{red}}Main} {\only<6>{\color{red}}ghar} {\only<5>{\color{red}}ja} {\only<5>{\color{red}}raha} {\only<4>{\color{red}}hoon}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
\begin{itemize}
\justifying
\item<1->  Humans try to produce each word in the output by focusing only on certain words in the input
\item<2-> Essentially at each time step we come up with a distribution on the input words
\item <7-> This distribution tells us how much attention to pay to each input words at each time step
\item <8-> Ideally, at each time-step we should feed only this relevant information (i.e. encodings of relevant words) to the decoder
\end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\if 0
\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \vspace{0.3cm}
  \centering
   \input{modules/Module3/tikz_images/translation_opt2_2} %Figure E to be modified
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
\begin{itemize}
\justifying
\item<1->  Let us revisit the decoder that have seen so far
\end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}
\fi

\begin{frame}
  \begin{columns}
    \column{0.45\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \vspace{0.3cm}
  \centering
   \input{modules/Module3/tikz_images/translation_opt2_2} %Figure E to be modified
      \end{overlayarea}

    \column{0.55\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
\begin{itemize}
\justifying
\item<1->  Let us revisit the decoder that we have seen so far
\item<2-> We either feed in the encoder information only once(at $s_0$)
\item <3-> Or we feed the same encoder information at each time step
\item <4-> Now suppose an oracle told you which words to focus on at a given time-step $t$
\item <5-> Can you think of a smarter way of feeding information to the decoder?
\end{itemize}
      \end{overlayarea}
  \end{columns}

\end{frame}
\begin{frame}
  \begin{columns}
    \column{0.45\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \vspace{0.3cm}
  \centering
   \input{modules/Module3/tikz_images/mt_attn} %Figure E to be modified
      \end{overlayarea}

    \column{0.55\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \begin{itemize}\justifying
\item<2->  We could just take a weighted average of the corresponding word representations and feed it to the decoder
\item<3-> For example at timestep 3, we can just take a weighted average of the representations of `ja' and `raha'
\item<4-> Intuitively this should work better because we are not overloading the decoder with irrelevant information (about words that do not matter at this time step)
\item <5-> How do we convert this intuition into a model ?
  \end{itemize}

      \end{overlayarea}
  \end{columns}
\end{frame}


\begin{frame}
  \begin{columns}
    \column{0.45\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \vspace{0.3cm}
  \centering
   \input{modules/Module3/tikz_images/mt_attn_no_anim} %Figure E to be modified
      \end{overlayarea}

    \column{0.55\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \begin{itemize}
    \justifying
    \item<2-> Of course in practice we will not have this oracle
    \item<3-> The machine will have to learn this from the data
    \item <4-> To enable this we define a function
    %\vspace{-0.5cm}
    \begin{align*}
      e_{jt} = f_{ATT}(s_{t-1},c_j)
    \end{align*}
    \item <5-> This quantity captures the importance of the $j^{th}$ input word for decoding the $t^{th}$ output word (we will see the exact form of $f_{ATT}$ later)
    \item<6-> We can normalize these weights by using the softmax function
    \vspace{-0.5cm}
    \begin{align*}
      \alpha_{jt} =\frac{exp(e_{jt})}{\sum\limits_{j=1}^{M}exp(e_{jt})}
    \end{align*}
  \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}


\begin{frame}
  \begin{columns}
    \column{0.45\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \vspace{0.3cm}
  \centering
   \input{modules/Module3/tikz_images/mt_attn_no_anim} %Figure E to be modified
      \end{overlayarea}

    \column{0.55\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
\onslide<2->{
    \begin{align*}
      \alpha_{jt} =\frac{exp(e_{jt})}{\sum\limits_{j=1}^{M}exp(e_{jt})}
    \end{align*}
}
\begin{itemize}
\justifying
\item<3->  $\alpha_{jt}$ denotes the probability of focusing on the $j^{th}$ word to produce the $t^{th}$ output word
\item<4-> We are now trying to learn the $\alpha$'s instead of an oracle informing us about the $\alpha$'s
\item <5-> Learning would always involve some parameters
\item <6-> So let's define a parametric form for $\alpha$'s
\end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.45\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \vspace{0.3cm}
  \centering
  \input{modules/Module3/tikz_images/mt_attn_no_anim} %Figure E to be modified
      \end{overlayarea}

    \column{0.55\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
\begin{itemize}
\justifying
\item<2-> From now on we will refer to the decoder RNN's state at the $t$-th timestep as $s_t$ and the encoder RNN's state at the $j$-th time step as $c_j$
\item<3-> Given these new notations, one (among many) possible choice for $f_{ATT}$ is
\begin{align*}
e_{jt} = V^{T}_{att}\tanh(U_{att}s_{t-1} + W_{att} c_{j})
\end{align*}
\item<4-> $V_{att}\in \mathbb{R}^{d}$ , $U_{att}\in \mathbb{R}^{d \times d}$, $W_{att} \in \mathbb{R}^{d\times d}$ are additional parameters of the model
\item <5-> These parameters will be learned along with the other parameters of the encoder and decoder
\end{itemize}

      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.45\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \vspace{0.3cm}
  \centering
   \input{modules/Module3/tikz_images/mt_attn_no_anim} %Figure E to be modified
      \end{overlayarea}

    \column{0.55\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
\begin{itemize}
\justifying
\item<2->  Wait a minute !
\item<3-> This model would make a lot of sense if were given the true $\alpha$'s at training time
\begin{align*}
\alpha^{true}_{tj} = [0, 0, 0.5, 0.5, 0]
\end{align*}
\begin{align*}
\alpha^{pred}_{tj} = [0.1, 0.1, 0.35, 0.35, 0.1]
\end{align*}
\item <4-> We could then minimize $\mathscr{L}(\alpha^{true}, \alpha^{pred})$ in addition to $\mathscr{L}(\theta)$ as defined earlier
\item <5-> But in practice it is very hard to get $\alpha^{true}$
\end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}
\begin{frame}
  \begin{columns}
    \column{0.45\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \vspace{0.3cm}
  \centering
   \input{modules/Module3/tikz_images/mt_attn_no_anim} %Figure E to be modified
      \end{overlayarea}

    \column{0.55\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \begin{itemize}
\justifying
\item<1->  For example, in our translation example we would want someone to manually annotate the source words which contribute to every target word
\item<2-> It is hard to get such annotated data
\item <3-> Then how would this model work in the absence of such data ?
\end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.45\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \vspace{0.3cm}
  \centering
  \input{modules/Module3/tikz_images/mt_attn_no_anim}
      \end{overlayarea}

    \column{0.55\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \begin{itemize}
\justifying
\item<2->  It works because it is a better modeling choice
\item<3-> This is a more informed model
\item <4-> We are essentially asking the model to approach the problem in a better (more natural) way
\item <5-> Given enough data it should be able to learn these attention weights just as humans do
\item <6-> That's the hope (and hope is a good thing)
\item <7-> And in practice indeed these models work better than the vanilla encoder decoder models
\end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
Let us revisit the MT model that we saw earlier and answer the same set of questions again (data, encoder, decoder, loss, training algorithm)
\end{frame}


\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \centering
   \input{modules/Module3/tikz_images/mt_attn} %Figure J
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \footnotesize{ \begin{itemize}

          \justifying
            \item<1-> \textbf{Task:} Machine Translation
            \item<2-> \textbf{Data:} $\{x_i=source_i,\ y_i=target_i\}_{i=1}^{N}$
              \item<3-> \textbf{Encoder:}
              \vspace{-0.2cm}
                \begin{align*}
                h_t &= RNN(h_{t-1}, x_t)\\
                s_0 &= h_T
                \end{align*}
                \vspace{-0.6cm}

                \item<4-> \textbf{Decoder:}
                \vspace{-0.2cm}
                \begin{align*}
                  \onslide<5->{e_{jt} &= V_{attn}^{T}tanh(U_{attn}h_j + W_{attn}s_t)\\}
                  \onslide<6->{\alpha_{jt} &= softmax(e_{jt})\\}
                  \onslide<7->{c_t &= \sum_{j=1}^{T} \alpha_{jt}h_j\\}
                  \onslide<8->{s_t &= RNN(s_{t-1}, [e(\hat{y}_{t - 1}), c_{t}])\\}
                  \onslide<9->{\ell_t &= softmax(Vs_t + b)}
                \end{align*}
            \vspace{-0.5cm}
            \item<9-> \textbf{Parameters:} $U_{dec}$, $V$, $W_{dec}$, $U_{enc}$, $W_{enc}$, $b$, $U_{attn}$,
            $V_{attn}$
            \item<10-> \textbf{Loss} and \textbf{Algorithm} remains same

        \end{itemize}}

      \end{overlayarea}
  \end{columns}
\end{frame}
\begin{frame}
You can try adding an attention component to all the other encoder decoder models that we discussed earlier and answer the same set of questions  (data, encoder, decoder, loss, training algorithm)
\end{frame}

\begin{frame}
\begin{itemize}
\item<1-> Can we check if the attention model actually learns something meaningful ?
\item<2-> In other words does it really learn to focus on the most relevant words in the input at the $t$-th timestep ?
\item<3-> We can check this by plotting the attention weights as a heatmap (we will see some examples on the next slide)
\end{itemize}
\end{frame}

\begin{frame}
\begin{overlayarea}{\textwidth}{\textheight}
\vspace{-0.8cm}
\begin{columns}
\column{0.5 \textwidth}
\onslide<1->{
\begin{figure}
\includegraphics[scale=0.24]{images/att_visual1.png}
\caption{Example output of attention-based summarization system [\href{https://www.aclweb.org/anthology/D/D15/D15-1044.pdf}{Rush et al.  2015}.]  }
\end{figure}}

\column{0.5 \textwidth}
\onslide<2->{
\begin{figure}
\includegraphics[scale=0.24]{images/att_visual2.png}
\caption{Example output of attention-based neural machine translation model [\href{https://arxiv.org/pdf/1507.01053.pdf}{Cho et al.  2015}].}

\end{figure}}
\end{columns}
\begin{itemize}
\item<3-> The heat map shows a soft alignment between the input and the generated output.

\item<4-> Each cell in the heat mapsssss corresponds to $\alpha_{tj}$ (i.e., the importance of the $j^{th}$ input word for predicting the $t^{th}$ output word as determined by the model)
\end{itemize}
\end{overlayarea}
\end{frame}

\begin{frame}
    \begin{overlayarea}{\textwidth}{\textheight}
        \begin{figure}
            \includegraphics[scale=0.35]{images/videocaptioning.png}
            \caption{Example output of attention-based video captioning system [\href{https://www.cv-foundation.org/openaccess/content_iccv_2015/papers/Yao_Describing_Videos_by_ICCV_2015_paper.pdf}{Yao et al.  2015}.]  }
        \end{figure}
    \end{overlayarea}
\end{frame}
