% \savestack{\lstmi}{\input{modules/Module2/tikz_images/lstm_i.tex}}
% \savestack{\lstmfull}{\input{modules/Module2/tikz_images/lstm_full.tex}}
% \savestack{\rnnone}{\input{modules/Module2/tikz_images/rnn_1.tex}}
% \savestack{\rnntwo}{\input{modules/Module2/tikz_images/rnn_2.tex}}
% \savestack{\rnnthree}{\input{modules/Module2/tikz_images/rnn_3.tex}}
% \savestack{\lstmstilde}{\input{modules/Module2/tikz_images/lstm_stilde.tex}}

\savestack{\movie}{\input{modules/Module2/tikz_images/movie.tex}}
\savestack{\lstmintrosigmoid}{\input{modules/Module2/tikz_images/lstm_intro_sigmoid.tex}}
\savestack{\lstmintrosigmoids}{\input{modules/Module2/tikz_images/lstm_intro_sigmoid_0s.tex}}
\savestack{\lstmselwriteone}{\input{modules/Module2/tikz_images/lstm_sel_write1.tex}}
\savestack{\lstmselwritetwo}{\input{modules/Module2/tikz_images/lstm_sel_write2.tex}}
\savestack{\lstmselread}{\input{modules/Module2/tikz_images/lstm_sel_read.tex}}
\savestack{\lstmselforgetone}{\input{modules/Module2/tikz_images/lstm_sel_forget1.tex}}
\savestack{\lstmselforgettwo}{\input{modules/Module2/tikz_images/lstm_sel_forget2.tex}}
\savestack{\lstmfull}{\input{modules/Module2/tikz_images/lstm_full.tex}}
% \savestack{\srrnn}{\input{modules/Module2/tikz_images/sr_rnn.tex}}

\begin{frame}
  \myheading{Module 15.2: Long Short Term Memory(LSTM) and Gated Recurrent Units(GRUs)}
\end{frame}

\begin{frame}
\begin{block}{Questions}
\begin{itemize}
\item<1-> \textcolor<3->{red}{Can we give a concrete example where RNNs also need to selectively read, write and forget ?}
\item<2-> How do we convert this intuition into mathematical equations ? \onslide<3->{We will see this over the next few slides}
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.4\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \makebox[\textwidth][c]{\usebox{\moviecontent}}
      \end{overlayarea}

    \column{0.6\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}\justifying
      \item<1-> Consider the task of predicting the sentiment (positive/negative) of a review
      \item<2-> RNN reads the document from left to right and after every word updates the state
      \item<3-> By the time we reach the end of the document the information obtained from the first few words is completely lost
      \item<4-> Ideally we want to
        \begin{itemize}\justifying
          \item<5-> \textbf{forget} the information added by stop words (a, the, etc.)
          \item<6-> \textbf{selectively read} the information added by previous sentiment bearing words (awesome, amazing, etc.)
          \item<7-> \textbf{selectively write} new information from the current word to the state
        \end{itemize}
      \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
\begin{block}{Questions}
\begin{itemize}
\item Can we give a concrete example where RNNs also need to selectively read, write and forget ?
\item \textcolor<1->{red}{How do we convert this intuition into mathematical equations ?}
\end{itemize}
\end{block}
\end{frame}


\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \makebox[\textwidth][c]{\usebox{\moviecontent}}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}\justifying
      \item<1-> Recall that the blue colored vector ($s_t$) is called the state of the RNN
      \item<2-> It has a finite size ($s_t \in \mathbb{R}^n$) and is used to store all the information upto timestep $t$
      \item<3-> This state is analogous to the whiteboard and sooner or later it will get overloaded and the information from the initial states will get morphed beyond recognition
      \item<4-> \textbf{Wishlist:} selective write, selective read and selective forget to ensure that this finite sized state vector is used effectively
      \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}


\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \begin{center}
          \input{modules/Module2/tikz_images/lstm_intro_text.tex}
        \end{center}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}\justifying
      \item<1-> Just to be clear, we have computed a state $s_{t-1}$ at timestep $t-1$ and now we want to overload it with new information ($x_t$) and compute a new state ($s_t$)
      \item<2-> While doing so we want to make sure that we use selective write, selective read and selective forget so that only important information is retained in $s_t$
      \item<3-> We will now see how to implement these items from our wishlist
      \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}


\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \only<1>{\makebox[\textwidth][c]{\usebox{\lstmintrosigmoidcontent}}}
        \only<2>{\makebox[\textwidth][c]{\usebox{\lstmintrosigmoidcontent}}}
        \only<3>{\makebox[\textwidth][c]{\usebox{\lstmintrosigmoidcontent}}}
        \only<4->{\makebox[\textwidth][c]{\usebox{\lstmintrosigmoidscontent}}}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \textbf{Selective Write}
      \begin{itemize}
      \item<1-> Recall that in RNNs we use $s_{t-1}$ to compute $s_{t}$\\
      \onslide<2-> {
      $s_t = \sigma(Ws_{t-1} + Ux_{t})~\textit{(ignoring bias)}$
      }
      \item<3-> But now instead of passing $s_{t-1}$ as it is to $s_{t}$ we want to pass (write) only some portions of it to the next state
      %\item<4-> In other words, $s_{t-1} \in \mathbb{R}^{n}$ and we may want to selectively pass (write) only some portions of it to the next state
      \item<4-> In the strictest case our decisions could be binary (for example, retain 1st and 3rd entries and delete the rest of the entries)
      \item<5-> But a more sensible way of doing this would be to assign a value between 0 and 1 which determines what fraction of the current state to pass on to the next state
      \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}


\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \makebox[\textwidth][c]{\usebox{\lstmselwriteonecontent}}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \textbf{Selective Write}
      \begin{itemize}\justifying
      \item<1-> We introduce a vector $o_{t-1}$ which decides what fraction of each element of $s_{t-1}$ should be passed to the next state
      \item<2-> Each element of $o_{t-1}$ gets multiplied with the corresponding element of $s_{t-1}$
      \item<3-> Each element of $o_{t-1}$ is restricted to be between 0 and 1
      \item<4-> But how do we compute $o_{t-1}$? How does the RNN know what fraction of the state to pass on?
      \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}


\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \makebox[\textwidth][c]{\usebox{\lstmselwriteonecontent}}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \textbf{Selective Write}
      \begin{itemize}\justifying
      \item<1-> Well the RNN has to learn $o_{t-1}$ along with the other parameters ($W, U, V$)
      \item<2-> We compute $o_{t-1}$ and $h_{t-1}$ as
      \begin{align*}
        \onslide<3->{o_{t-1} &= \sigma(W_oh_{t-2} + U_ox_{t-1} + b_o)\\}
        \onslide<4->{h_{t-1} &= o_{t-1} \odot \sigma(s_{t-1})}
      \end{align*}
      \item<5-> The parameters $W_o, U_o, b_o$ need to be learned along with the existing parameters $W, U, V$
      \item<6-> The sigmoid (logistic) function ensures that the values are between 0 and 1
      \item<7-> $o_{t}$ is called the output gate as it decides how much to pass (write) to the next time step
      \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        %\vspace{0.4}
        % \makebox[\textwidth][c]{\usebox{\srrnncontent}}
        % \makebox[\textwidth][c]{\usebox{\lstmstildecontent}}
        % \include{sr_rnn}
        \makebox[\textwidth][c]{\usebox{\lstmselwritetwocontent}}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \textbf{Selective Read}
      \begin{itemize}\justifying
      \item<1-> We will now use $h_{t-1}$ to compute the new state at the next time step
      \item<2-> We will also use $x_t$ which is the new input at time step t
      \begin{align*}
        \tilde{s_t} = \sigma(Wh_{t-1} + Ux_{t} + b)
      \end{align*}
      \item<3-> Note that $W, U$ and $b$ are similar to the parameters that we used in RNN (for simplicity we have not shown the bias b in the figure)
      \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}


\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        % \makebox[\textwidth][c]{\usebox{\lstmicontent}}
        \makebox[\textwidth][c]{\usebox{\lstmselreadcontent}}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \textbf{Selective Read}
      \begin{itemize}\justifying
        \item<1-> $\tilde{s_t}$ thus captures all the information from the previous state ($h_{t-1}$) and the current input $x_{t}$ %We want to combine this new cell state (information) $\tilde{s_t}$ with the running cell state (information) $s_t$
        \item<2-> However, we may not want to use all this new information and only selectively \textbf{read} from it before constructing the new cell state $s_t$ %Again, we might want to selectively \textbf{read} from this new information $\tilde{s_t}$
        \item<3-> To do this we introduce another gate called the input gate
        \begin{align*}
          \onslide<4->{i_t = \sigma(W_ih_{t-1} + U_ix_{t} + b_i)}
        \end{align*}
        \item<5-> and use $i_t \odot \tilde{s_t}$ as the selectively read state information
      \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{columns}
    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
        \makebox[\textwidth][c]{\usebox{\lstmselreadcontent}}
        % \makebox[\textwidth][c]{\usebox{\lstmicontent}}
      \end{overlayarea}

    \column{0.5\textwidth}
      \begin{overlayarea}{\textwidth}{\textheight}
      \begin{itemize}\justifying
        \item<1-> So far we have the following
        \begin{align*}
          \onslide<2->{&\textbf{Previous state:} \\ &s_{t-1} \\}
          \onslide<3->{&\textbf{Output gate:} \\ &o_{t-1} = \sigma(W_oh_{t-2} + U_ox_{t-1} + b_o)\\}
          \onslide<4->{&\textbf{Selectively Write:} \\ &h_{t-1} = o_{t-1} \odot \sigma(s_{t-1})\\}
          \onslide<5->{&\textbf{Current (temporary) state:} \\ &\tilde{s_t} = \sigma(Wh_{t-1} + Ux_{t} + b)\\}
          \onslide<6->{&\textbf{Input gate:} \\ &i_t = \sigma(W_ih_{t-1} + U_ix_{t} + b_i)\\}
          \onslide<7->{&\textbf{Selectively Read:} \\ &i_t \odot \tilde{s_t}}
        \end{align*}
      \end{itemize}
      \end{overlayarea}
  \end{columns}
\end{frame}

\begin{frame}
  \begin{overlayarea}{\textwidth}{0.4\textheight}
    \only<2-4>{\makebox[\textwidth][l]{\usebox{\lstmselforgetonecontent}}}
    \only<5-> {\makebox[\textwidth][l]{\usebox{\lstmselforgettwocontent}}}
  \end{overlayarea}

  \begin{overlayarea}{\textwidth}{0.6\textheight}
    \begin{columns}
      \column{0.5\textwidth}
        \textbf{Selective Forget}
        \begin{itemize}\justifying
          \item<1-> How do we combine $s_{t-1}$ and $\tilde{s_t}$ to get the new state
          \item<2-> Here is one simple (but effective) way of doing this:
          \begin{align*}
            \onslide<3->{s_{t} = s_{t-1} + i_t \odot \tilde{s_t}}
          \end{align*}
        \end{itemize}

      \column{0.5\textwidth}
        \begin{itemize}\justifying
          \item<4-> But we may not want to use the whole of $s_{t-1}$ but forget some parts of it
          \item<5-> To do this we introduce the forget gate
          \begin{align*}
            \onslide<6->{f_t &= \sigma(W_fh_{t-1} + U_fx_{t} + b_f) \\}
            \onslide<7->{s_{t} &= f_t \odot s_{t-1} + i_t \odot \tilde{s_t}}
          \end{align*}
        \end{itemize}
    \end{columns}
  \end{overlayarea}
\end{frame}

\begin{frame}
  \begin{overlayarea}{\textwidth}{0.4\textheight}
    \makebox[\textwidth][l]{\usebox{\lstmfullcontent}}
  \end{overlayarea}

  \begin{overlayarea}{\textwidth}{0.6\textheight}
    \begin{itemize}\justifying
      \item<1-> We now have the full set of equations for LSTMs
      \item<2-> The green box together with the selective write operations following it, show all the computations which happen at timestep $t$
    \end{itemize}
    \begin{columns}
      \column{0.5\textwidth}
        \begin{overlayarea}{0.5\textwidth}{\textheight}
        \onslide<2->{\textbf{Gates:}}
          \begin{align*}
            \onslide<3->{o_t &= \sigma(W_oh_{t-1} + U_ox_{t} + b_o) \\}
            \onslide<4->{i_t &= \sigma(W_ih_{t-1} + U_ix_{t} + b_i) \\}
            \onslide<5->{f_t &= \sigma(W_fh_{t-1} + U_fx_{t} + b_f)}
          \end{align*}
        \end{overlayarea}
      \column{0.5\textwidth}
        \begin{overlayarea}{0.5\textwidth}{\textheight}
        \onslide<2->{\textbf{States:}}
          \begin{align*}
            \onslide<6->{\tilde{s_t} &= \sigma(Wh_{t-1} + Ux_{t} + b) \\}
            \onslide<7->{s_{t} &= f_t \odot s_{t-1} + i_t \odot \tilde{s_t} \\}
            \onslide<8->{h_{t} &= o_{t} \odot \sigma(s_{t})}
            \onslide<9->{\text{ and } rnn_{out} &= h_t}
          \end{align*}
        \end{overlayarea}
    \end{columns}
  \end{overlayarea}
\end{frame}

% \begin{frame}
%   \begin{columns}
%     \column{0.5\textwidth}
%       \begin{overlayarea}{\textwidth}{\textheight}
%         \makebox[\textwidth][c]{\usebox{\lstmfullcontent}}
%       \end{overlayarea}

%     \column{0.5\textwidth}
%       \begin{overlayarea}{\textwidth}{\textheight}
%       \begin{itemize}\justifying
%         \item<1-> We now have the full set of equations for LSTMs
%       \end{itemize}
%       \onslide<2->{\textbf{Gates:}}
%         \begin{align*}
%           \onslide<3->{o_t &= \sigma(W_oh_{t-1} + U_ox_{t} + b_o) \\}
%           \onslide<4->{i_t &= \sigma(W_ih_{t-1} + U_ix_{t} + b_i) \\}
%           \onslide<5->{f_t &= \sigma(W_fh_{t-1} + U_fx_{t} + b_f)}
%         \end{align*}
%       \onslide<2->{\textbf{States:}}
%         \begin{align*}
%           \onslide<6->{\tilde{s_t} &= \sigma(Wh_{t-1} + Ux_{t} + b) \\}
%           \onslide<7->{s_{t} &= f_t \odot s_{t-1} + i_t \odot \tilde{s_t} \\}
%           \onslide<8->{h_{t} &= o_{t} \odot \sigma(s_{t})\\}
%           \onslide<9->{rnn_{out} &= h_t}
%         \end{align*}

%       \end{overlayarea}
%   \end{columns}
% \end{frame}

\begin{frame}
      \begin{overlayarea}{\textwidth}{\textheight}
      \begin{block} {Note}
      \begin{itemize}\justifying
        \item LSTM has many variants which include different number of gates and also different arrangement of gates
        \item The one which we just saw is one of the most popular variants of LSTM
        \item Another equally popular variant of LSTM is Gated Recurrent Unit which we will see next
      \end{itemize}
      \end{block}
      \end{overlayarea}
\end{frame}

\begin{frame}
  \begin{overlayarea}{\textwidth}{0.4\textheight}
    \begin{center}
      % \makebox[\textwidth][c]{\usebox{\grufullcontent}}
      \input{modules/Module2/tikz_images/gru_full.tex}
    \end{center}
  \end{overlayarea}
  \begin{overlayarea}{\textwidth}{0.6\textheight}
    \begin{columns}
      \column{0.5\textwidth}

        % \begin{overlayarea}{\textwidth}{\textheight}
        % \begin{overlayarea}{\textwidth}{\textheight}
        % \end{overlayarea}
        % \begin{itemize}\justifying
          \visible<1-> The full set of equations for GRUs
        % \end{itemize}
        \onslide<2->{\textbf{Gates:}}
          \vspace{-4mm}
          \begin{align*}
            \onslide<3->{o_t &= \sigma(W_os_{t-1} + U_ox_{t} + b_o) \\}
            \onslide<5->{i_t &= \sigma(W_is_{t-1} + U_ix_{t} + b_i) }
          \end{align*}
        \onslide<2->{\textbf{States:}}
          \vspace{-4mm}
          \begin{align*}
            \onslide<4->{\tilde{s_t} &= \sigma(W (o_{t} \odot s_{t-1}) + Ux_{t} + b) \\}
            \onslide<6->{s_{t} &= \alert<7>{(1-i_t)} \odot s_{t-1} + \alert<7>{i_t} \odot \tilde{s_t} }
          \end{align*}
      \column{0.5\textwidth}

        \begin{itemize}\justifying
          \item<7-> No explicit forget gate (\alert<7>{the forget gate and input gates are tied})
           \item<8-> The gates depend directly on  $s_{t-1}$ and not the intermediate $h_{t-1}$ as in the case of LSTMs
        \end{itemize}


        % \end{overlayarea}
    \end{columns}
  \end{overlayarea}

\end{frame}
