\begin{frame}
	\myheading{Module 16.5: Hierarchical Attention}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \vspace{0.5cm}
      \begin{block}{Context}
      \begin{itemize}\justifying
        \item[] U: Can you suggest a good movie?
        \item[] B: Yes, sure. How about Logan?
        \item[] U: Okay, who is the lead actor?
      \end{itemize}


      \end{block}

  \begin{block}{Response}
    \begin{itemize}\justifying
      \item[] B: Hugh Jackman, of course

    \end{itemize}
  \end{block}

    \end{overlayarea}

    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}\justifying
        \item<1-> Consider a dialog between a user ($u$) and a bot ($B$)
        \item<2-> The dialog contains a sequence of utterances between the user and the bot
        \item<3-> Each utterance in turn is a sequence of words
        \item<4->Thus what we have here is a ``sequence of sequences'' as input
        \item<5->Can you think of an encoder for such a sequence of sequences?
      \end{itemize}
    \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \vspace{0.1cm}
\input{modules/Module5/tikz_images/HRNN1}
    \end{overlayarea}

    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}\justifying
        \item<1-> We could think of a two level hierarchical RNN encoder
        \item<2-> The first level RNN operates on the sequence of words in each utterance and gives us a representation
        \item<3-> We now have a sequence of utterance representations (red vectors in the image)
        \item<4->We can now have another RNN which encodes this sequence and gives a single representations for the sequences of utterances
        \item<5->The decoder can then produce an output sequence conditioned on this utterance
      \end{itemize}
    \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
        \vspace{0.2cm}

\begin{tabular}{|l|}
\hline
Politics is the process of making decisions\\ applying to all members of each group.\\
More narrowly, it refers to achieving and \dots \\
\hline
\end{tabular}
  \vspace{0.2cm}
  \centering
\input{modules/Module5/tikz_images/HRNN2}
    \end{overlayarea}

    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}\justifying
        \item<1-> Let us look at another example
        \item<2-> Consider the task of document classification or summarization
        \item<3-> A document is a sequence of sentences
        \item<4-> Each sentence in turn is a sequence of words
        \item<5-> We can again use a hierarchical RNN to model this
      \end{itemize}
    \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.50\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
        \vspace{0.2cm}

        \begin{tabular}{|l|}
          \hline
          Politics is the process of making decisions\\ applying to all members of each group.\\
          More narrowly, it refers to achieving and \dots \\
          \hline
        \end{tabular}
        \vspace{0.2cm}
        \centering
        \input{modules/Module5/tikz_images/HRNN2_mod}
    \end{overlayarea}

    \column{0.50\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}\justifying
        \small \item<1-> \textbf{Data:} $\{ Document_i, class_i\}_{i=1}^N$
                \item<2-> \textbf{Word level (1) encoder:}
                \vspace{-0.1in}
                \begin{align*}
                  h^{1}_{ij} &= RNN(h^{1}_{ij-1}, w_{ij}) \\
                  s_i &= h^{1}_{iT_i} \quad \textit{[$T$ is length of sentence i]}
                \end{align*}
                \item<3-> \textbf{Sentence level (2) encoder:}
                \vspace{-0.1in}
                \begin{align*}
                  h^{2}_{i} &= RNN(h^{2}_{i-1}, s_i) \\
                  s &= h^{2}_{K}  \quad [\textit{$K$ is number of sentences]}
                \end{align*}
                \item<4-> \textbf{Decoder:}
                \vspace{-0.1in}
                \begin{align*}
                  P(y|document) = softmax (Vs + b)
                \end{align*}
                \item<5-> \textbf{Params:} $W_{enc}^1$, $U_{enc}^1$, $W_{enc}^2$, $U_{enc}^2$, $V, b$
                \item<6-> \textbf{Loss:} Cross Entropy
                \item<7-> \textbf{Algorithm:} Gradient Descent with backpropagation
      \end{itemize}
    \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{figure}
        \includegraphics<3->[scale=0.4]{images/HAN.png}
        \visible<3->{\caption{Hierarchical Attention Network [\href{https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf}{Yang et al.}] }}
      \end{figure}
    \end{overlayarea}

    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}\justifying
        \item<1-> How would you model attention in such a hierarchical encoder decoder model ?
        \item<2-> We need attention at two levels
        \item<3-> First we need to attend to important (most informative) words in a sentence
        \item<4-> Then we need to attend to important (most informative) sentences in a document

        \item<5-> Let us see how to model this
      \end{itemize}
    \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
            \begin{figure}
              \includegraphics[scale=0.4]{images/HAN.png}
              \caption{Hierarchical Attention Network [\href{https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf}{Yang et al.}]  }
            \end{figure}
    \end{overlayarea}

    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}
        \justifying
        \item<1-> \textbf{Data:} $\{ Document_i, class_i\}_{i=1}^N$
        \item<2-> \textbf{Word level (1) encoder:}
                  \vspace{-0.1in}
                  \scriptsize
                   \begin{align*}
                                      h_{ij} &= RNN(h_{ij-1}, w_{ij}) \\
                                      u_{ij} &= tanh(W_{w}h_{ij}+b_{w})\\
                                      \alpha_{ij} &= \frac{exp(u^{T}_{ij}u_{w})} { \sum_{t}exp(u^{T}_{it}u_{w})}\\
                                      s_{i} &= \sum_{j} \alpha_{ij} h_{ij}
                   \end{align*}
        \item<3-> \textbf{Sentence level (2) encoder:}
                    \scriptsize
                    \begin{align*}
                                          h_{i} &= RNN(h_{i-1}, s_i) \\
                                          u_{i} &= tanh(W_{s}h_{i} + b_{s}) \\
                                          \alpha_{i} &= \frac{exp(u_{i}^{T}u_{s})}{ \sum_{i}exp(u_{i}^{T}u_{s})} \\
                                          s &= \sum_{i} \alpha_{i}h_{i}
                                        \end{align*}
      \end{itemize}
    \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
            \begin{figure}
              \includegraphics[scale=0.4]{images/HAN.png}
              \caption{Hierarchical Attention Network [\href{https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf}{Yang et al.}.]  }
            \end{figure}
    \end{overlayarea}

    \column{0.5\textwidth}
    \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}\justifying
        \item<1-> \textbf{Decoder:}
                      \begin{align*}
                                  P(y|document) = softmax(Vs + b)
                     \end{align*}
        \item<2-> \textbf{Parameters:} $W_{w},W_{s},V,b_{w},b_{s},b,u_{w},u_{s}$
        \item<3-> \textbf{Loss:} cross entropy
        \item<4-> \textbf{Algorithm:} Gradient Descent and backpropagation
      \end{itemize}
    \end{overlayarea}
  \end{columns}
\end{frame}
