\begin{frame}
	\myheading{Module 16.1: Introduction to Encoder Decoder Models}
\end{frame}



\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
	\vspace{0.3cm}
	\centering
	\input{modules/Module1/tikz_images/rnn1.tex}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
 \begin{itemize}
        	\justifying
            \item<1->  We will start by revisiting the problem of language modeling
            \item<2-> Informally, given `$t-i$' words we are interested in predicting the $t^{th}$ word
            \item<3-> More formally, given $y_1, y_2, ..., y_{t-1}$ we want to find
            \begin{align*}
            y^* = argmax\ P(y_t | y_1,y_2,...,y_{t-1})
            \end{align*}
     	    \item<4-> Let us see how we model $P(y_t | y_1,y_2...y_{t-1})$ using a RNN
     	    \item<5-> We will refer to $P(y_t | y_1,y_2...y_{t-1})$ by shorthand notation: $P(y_t |y_1^{t-1})$
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}


\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
	\vspace{0.3cm}
	\centering
	\input{modules/Module1/tikz_images/rnn11.tex}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
\begin{itemize}
        	\justifying
            \item<1-> We are interested in
            \begin{align*}
              P(y_t =j | y_1,y_2...y_{t-1})
            \end{align*}
            where $j \in V$ and $V$ is the set of all vocabulary words
            \item<2-> Using an RNN we compute this as
            \begin{align*}
            P(y_t=j|y_1^{t-1}) = softmax(Vs_t +c)_j
            \end{align*}
            \item<3-> In other words we compute
            \begin{align*}
            P(y_t=j|y_1^{t-1}) &= P(y_t=j|s_t) \\
            &=softmax(Vs_t +c)_j
            \end{align*}
     	    \item<4-> Notice that the recurrent connections ensure that $s_t$ has information about $y_1^{t-1}$
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
	\vspace{0.3cm}
	\centering
	\input{modules/Module1/tikz_images/rnn11.tex}
	\onslide<1->{

	\vspace{0.5cm}
	Data:
	\begin{tabular}{|l|}
	\hline
	India, officially the Republic\\
	of India, is a country in South \\
	Asia. It is the seventh-largest\\
	 country by area, .....  \\
	\hline
	\end{tabular}}

      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
\begin{itemize}
        	\justifying
          \item <1-> \textbf{Data:} All sentences from any large corpus (say wikipedia)
          \item<2-> \textbf{Model:} \onslide<3->\begin{align*} s_t &= \sigma(Ws_{t-1} + Ux_t +b) \\ P(y_t=j|y_1^{t-1}) &= softmax(Vs_t +c)_j\end{align*}
          \item<4-> \textbf{Parameters:} $U, V, W, b, c$
          \item<5-> \textbf{Loss:}\onslide<6-> \begin{align*} \mathscr{L}(\theta) &= \sum_{t=1}^{T} \mathscr{L}_t(\theta)\\
           \mathscr{L}_t(\theta) &= -\log P(y_t = \ell_t | y_1^{t-1})
          \end{align*}
          where $\ell_t$ is the true word at time step $t$

          %   \item<1-> We are essentially making a modeling choice by saying
          %   \begin{align*}
          %   P(y_k=j|y_1^{k-1}) = softmax(Vh_i +b)_j
          %   \end{align*}
          %   \item<2-> The parameters of this distribution $P(y_k=j|y_1^{k-1})$ are U, V, W which can be estimated from the data
          %   \item<3-> Data: All sentences from any large corpus (say wikipedia)
     	    % \item<5-> Every sentence is one training instance
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
	\vspace{0.3cm}
	\centering
	\input{modules/Module1/tikz_images/test.tex}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
 \begin{itemize}
        	\justifying
            \item<1-> What is the input at each time step?
            \item<3-> It is simply the word that we predicted at the previous time step
            \item<8-> In general
 		\vspace{-0.5cm}
            \begin{align*}
            s_t=RNN(s_{t-1},x_t)
            \end{align*}
            \item<9-> Let $j$ be the index of the word which has been assigned the max probability at time step $t-1$
 		\vspace{-0.5cm}
            \begin{align*}
	 	x_t = e(v_j)
            \end{align*}
     	    \item<10-> $x_t$ is essentially a one-hot vector ($e(v_j)$)representing the $j^{th}$ word in the vocabulary
            \item<11-> In practice, instead of one hot representation we use a pre-trained word embedding of the $j^{th}$ word
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}


\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
  \vspace{0.3cm}
  \centering
	\input{modules/Module1/tikz_images/rnn11.tex}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
\begin{itemize}
          \justifying
            \item<1-> Notice that $s_0$ is not computed but just randomly initialized
            \item<2-> We learn it along with the other parameters of RNN (or LSTM or GRU)
            \item<3-> We will return back to this later
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{overlayarea}{\textwidth}{\textheight}
  \begin{columns}
    \begin{column}{0.3\textwidth}
  \vspace{0.3cm}
  \onslide<2->{

       \begin{align*}
    s_t &= \sigma(U\ \textcolor{red}{x_t} +W\textcolor{red}{s_{t-1}} + b)
      \color{white}{s_{t}}\\
         \color{white}{h_{t}}\\
              \color{white}{h_{t}}
            \end{align*}
       \begin{align*}
       \onslide<3->{s_t} & \onslide<3->{= \text{RNN}(\ \textcolor{red}{s_{t-1}},\textcolor{red}{x_t})}
       \end{align*}
  }
    \end{column}
    \begin{column}{0.36\textwidth}
  \vspace{0.3cm}
  \onslide<4->{

    \begin{align*}
            \tilde{s_t} &= \sigma(W (o_{t} \odot \ \textcolor{red}{s_{t-1}}) + U\textcolor{red}{x_{t}} + b) \\
              s_{t} &= i_t \odot \ \textcolor{red}{s_{t-1}} + (1 - i_t) \odot \tilde{s_t} \\
         \color{white}{h_{t}}
          \end{align*}
          \begin{align*}
          \onslide<5->{s_t} & \onslide<5->{=\text{GRU}(\ \textcolor{red}{s_{t-1}}, \textcolor{red}{x_t})}
          \end{align*}
  }
    \end{column}
    \begin{column}{0.33\textwidth}
  \vspace{0.3cm}
  \onslide<6->{

     \begin{align*}
              \tilde{s_t} &= \sigma(W\ \textcolor{red}{h_{t-1}} + U\textcolor{red}{x_{t}} + b) \\
              s_{t} &= f_t \odot \ \textcolor{red}{s_{t-1}} + i_t \odot \tilde{s_t} \\
              h_{t} &= o_{t} \odot \sigma(s_{t})
              \end{align*}
              \begin{align*}
              \onslide<7->{h_t, s_t} & \onslide<7->{=\text{LSTM}(\ \textcolor{red}{h_{t-1}}, \textcolor{red}{s_{t-1}},\textcolor{red}{x_t})}
          %   rnn_{out} &= h_t
          \end{align*}  }
      \end{column}
    \end{columns}
    \vspace{0.5cm}
\only<1->{
\begin{itemize}
    \item Before moving on we will see a compact way of writing the function computed by RNN, GRU and LSTM
    \item<8-> We will use these notations going forward
\end{itemize}
  }
    \end{overlayarea}
\end{frame}



\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
	\vspace{0.3cm}
	\centering
	\input{modules/Module1/tikz_images/rnn3.tex}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
 \begin{itemize}
        	\justifying
            \item<1-> So far we have seen how to model the conditional probability distribution $P(y_t|y_1^{t-1})$
            \item<2-> More informally, we have seen how to generate a sentence given previous words
            \item<3-> What if we want to generate a sentence given an image?
            \item<11-> We are now interested in $P(y_t|y_1^{t-1},I)$ instead of $P(y_t|y_1^{t-1})$ where $I$ is an image
            \item<12-> Notice that $P(y_t|y_1^{t-1},I)$ is again a conditional distribution
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
	\vspace{0.3cm}
	\centering
	\input{modules/Module1/tikz_images/test1.tex}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
\begin{itemize}
        	\justifying
            \item<1-> Earlier we modeled $P(y_t|y_1^{t-1})$ as
             \begin{align*}
            P(y_t|y_1^{t-1}) = P(y_{t}=j|s_t)
            \end{align*}
            \item<2-> Where $s_t$ was a state capturing all the previous words
            \item<3-> We could now model $P(y_t=j|y_1^{t-1},I)$ as $P(y_{t}=j|s_t,f_{c_7}( I ))$
            \item<4-> where $fc_7(I)$ is the representation obtained from the $fc_7$ layer of an image
        \end{itemize}
      \end{overlayarea}
  \end{columns}

\end{frame}


  \begin{frame}
  \begin{block}{}
  \begin{itemize}
    \justifying
    \item<1-> There are many ways of making $P(y_{t}=j)$ conditional on $f_{c_7}(I)$
    \item<2-> Let us see two such options
  \end{itemize}
  \end{block}
  \end{frame}


\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
	\vspace{0.3cm}
	\hspace{-0.3cm}
	\centering
	\input{modules/Module1/tikz_images/test2.tex}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \vspace{2cm}
\begin{itemize}
        	\justifying
            \item<1-> \textbf{Option 1:} Set $s_0 = f_{c_7}(I)$
            \item<2-> Now $s_0$ and hence all subsequent $s_t$'s depend on $f_{c_7}(I)$
	\item<3-> We can thus say that $P(y_{t}=j)$ depends on $f_{c_7}(I)$
	\item<4-> In other words, we are computing $P(y_{t}=j|s_t,f_{c_7}(I))$
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
	\vspace{0.3cm}
	\centering
	\input{modules/Module1/tikz_images/test3.tex}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
\begin{itemize}
        	\justifying
            \item<1-> \textbf{Option 2:} Another more explicit way of doing this is to compute
            \begin{align*}
            s_t = RNN(s_{t-1},[x_t,f_{c_7}(I))]
            \end{align*}
            \item<2-> In other words we are explicitly using $f_{c_7}(I)$ to compute  $s_t$ and hence \\ $P(y_{t}=j)$
            \item<3-> You could think of other ways of conditioning $P(y_{t}=j)$ on $f_{c_7}$
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.55\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
	\vspace{0.3cm}
	\centering
	\input{modules/Module1/tikz_images/rnn81.tex}
      \end{overlayarea}

    \column{0.45\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
\begin{itemize}
        	\justifying
            \item<1-> Let us look at the full architecture
            \item<2-> A CNN is first used to \textbf{encode} the image
            \item<3-> A RNN is then used to decode (generate) a sentence from the encoding
            \item<4-> This is a typical \textbf{encoder decoder architecture}
            \item<5-> Both the encoder and decoder use a neural network
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}


\begin{frame}
  \begin{columns}
    \column{0.55\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
	\vspace{0.3cm}
	\centering
	\input{modules/Module1/tikz_images/rnn812.tex}
      \end{overlayarea}

    \column{0.45\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
\begin{itemize}
        	\justifying
            \item<1-> Let us look at the full architecture
            \item<1-> A CNN is first used to \textbf{encode} the image
            \item<1-> A RNN is then used to decode (generate) a sentence from the encoding
            \item<1-> This is a typical \textbf{encoder decoder architecture}
            \item<1-> Both the encoder and decoder use a neural network
            \item<1-> Alternatively, the encoder's output can be fed to every step of the decoder
        \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}
